From b9667b9eda97001f2ac8173d0e41182f80f81083 Mon Sep 17 00:00:00 2001 From: Atlas Date: Sun, 16 Aug 2026 01:41:06 -0500 Subject: [PATCH 01/29] wip: bank native daily verb implementation --- gr2/python_cli/add.py | 51 +++++++++ gr2/python_cli/app.py | 79 ++++++++++++++ gr2/python_cli/commit.py | 83 ++++++++++++++ gr2/python_cli/push.py | 155 ++++++++++++++++++++++++++ gr2/tests/test_add.py | 108 +++++++++++++++++++ gr2/tests/test_commit.py | 137 +++++++++++++++++++++++ gr2/tests/test_push.py | 228 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 841 insertions(+) create mode 100644 gr2/python_cli/add.py create mode 100644 gr2/python_cli/commit.py create mode 100644 gr2/python_cli/push.py create mode 100644 gr2/tests/test_add.py create mode 100644 gr2/tests/test_commit.py create mode 100644 gr2/tests/test_push.py diff --git a/gr2/python_cli/add.py b/gr2/python_cli/add.py new file mode 100644 index 0000000..1352038 --- /dev/null +++ b/gr2/python_cli/add.py @@ -0,0 +1,51 @@ +"""Native single-repository staging for the Python gr2 CLI.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from .gitops import git + + +class AddError(Exception): + pass + + +@dataclass(frozen=True) +class AddResult: + requested_paths: tuple[str, ...] + staged_files: tuple[str, ...] + + +def stage_files(repo: Path, paths: list[str]) -> AddResult: + """Stage only the requested pathspecs and report their observed index rows. + + ``git add --all`` is deliberate. A deleted tracked path does not exist on + disk, so a filesystem existence precheck turns a valid deletion into a + false "missing" error. Git's index is the authority for pathspec validity. + """ + if not paths: + raise AddError("at least one path is required") + + requested = tuple(paths) + try: + staged = git(repo, "add", "--all", "--", *requested) + except OSError as exc: + raise AddError(f"failed to launch git add in {repo}: {exc}") from exc + if staged.returncode != 0: + detail = (staged.stderr or staged.stdout).strip() + raise AddError(detail or f"git add failed with exit {staged.returncode}") + + try: + observed = git(repo, "diff", "--cached", "--name-only", "-z", "--", *requested) + except OSError as exc: + raise AddError( + f"git add succeeded but staged-path evidence was unavailable: {exc}" + ) from exc + if observed.returncode != 0: + detail = (observed.stderr or observed.stdout).strip() + raise AddError(detail or "git add succeeded but staged-path evidence was unavailable") + + staged_files = tuple(sorted(name for name in observed.stdout.split("\0") if name)) + return AddResult(requested_paths=requested, staged_files=staged_files) diff --git a/gr2/python_cli/app.py b/gr2/python_cli/app.py index be72012..095fa30 100644 --- a/gr2/python_cli/app.py +++ b/gr2/python_cli/app.py @@ -12,9 +12,12 @@ from gr2.prototypes import lane_workspace_prototype as lane_proto from gr2.prototypes import repo_maintenance_prototype as repo_proto +from . import add as add_ops from . import branch as branch_ops +from . import commit as commit_ops from . import execops, failures, migration, spec_apply, syncops from . import pr as pr_ops +from . import push as push_ops from .events import EventType, emit_after_outcome from .gitops import ( branch_exists, @@ -574,6 +577,82 @@ def branch_cmd( typer.echo(f"Switched to branch '{name}'") +@app.command("add") +def add_cmd( + paths: list[str] = typer.Argument(..., help="Paths or pathspecs to stage"), + repo_path: Path | None = typer.Option( + None, + "--repo-path", + help="Repo to operate on (defaults to cwd; gr2 verbs are single-repo)", + ), +) -> None: + """Stage paths in one repository, including tracked deletions.""" + target = (repo_path or Path.cwd()).resolve() + try: + result = add_ops.stage_files(target, paths) + except add_ops.AddError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + if result.staged_files: + typer.echo(f"Staged {len(result.staged_files)} path(s): {', '.join(result.staged_files)}") + else: + typer.echo("No changes staged for the requested paths") + + +@app.command("commit") +def commit_cmd( + message: str = typer.Option(..., "--message", "-m", help="Commit message"), + amend: bool = typer.Option(False, "--amend", help="Amend the current commit"), + repo_path: Path | None = typer.Option( + None, + "--repo-path", + help="Repo to operate on (defaults to cwd; gr2 verbs are single-repo)", + ), +) -> None: + """Create or amend one commit from the staged index.""" + target = (repo_path or Path.cwd()).resolve() + try: + receipt = commit_ops.create_commit(target, message, amend=amend) + except commit_ops.CommitError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + action = "Amended" if receipt.amended else "Committed" + typer.echo(f"{action} {receipt.commit_sha}") + + +@app.command("push") +def push_cmd( + remote: str | None = typer.Option(None, "--remote", help="Configured remote to push"), + set_upstream: bool = typer.Option(False, "--set-upstream", "-u"), + force_with_lease: bool = typer.Option( + False, + "--force-with-lease", + help="Replace the remote ref only if its observed value still matches", + ), + repo_path: Path | None = typer.Option( + None, + "--repo-path", + help="Repo to operate on (defaults to cwd; gr2 verbs are single-repo)", + ), +) -> None: + """Push one branch and verify its immutable remote commit.""" + target = (repo_path or Path.cwd()).resolve() + try: + receipt = push_ops.push_current_branch( + target, + remote=remote, + set_upstream=set_upstream, + force_with_lease=force_with_lease, + ) + except push_ops.PushError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + typer.echo( + f"Pushed {receipt.branch} to {receipt.remote} at {receipt.remote_sha} " + "(remote ref verified)" + ) + + @exec_app.command("status") def exec_status( workspace_root: Path, diff --git a/gr2/python_cli/commit.py b/gr2/python_cli/commit.py new file mode 100644 index 0000000..b4c3518 --- /dev/null +++ b/gr2/python_cli/commit.py @@ -0,0 +1,83 @@ +"""Native single-repository commit creation for the Python gr2 CLI.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from .gitops import git + + +class CommitError(Exception): + pass + + +class NothingToCommitError(CommitError): + pass + + +@dataclass(frozen=True) +class CommitReceipt: + commit_sha: str + message: str + amended: bool + + +def _head_sha(repo: Path, *, required: bool) -> str | None: + try: + result = git(repo, "rev-parse", "--verify", "HEAD") + except OSError as exc: + raise CommitError(f"failed to resolve HEAD in {repo}: {exc}") from exc + sha = result.stdout.strip() if result.returncode == 0 else "" + if required and not sha: + raise CommitError("git commit succeeded but the resulting HEAD could not be verified") + return sha or None + + +def _staged_changes_exist(repo: Path) -> bool: + try: + probe = git(repo, "diff", "--cached", "--quiet", "--exit-code") + except OSError as exc: + raise CommitError(f"failed to inspect the staged index in {repo}: {exc}") from exc + if probe.returncode == 0: + return False + if probe.returncode == 1: + return True + detail = (probe.stderr or probe.stdout).strip() + raise CommitError(detail or f"staged-index probe failed with exit {probe.returncode}") + + +def create_commit(repo: Path, message: str, *, amend: bool = False) -> CommitReceipt: + """Create a commit and return the immutable commit ID observed afterward. + + A normal commit first asks the index whether a staged diff exists. It never + classifies "nothing to commit" by localized or version-dependent prose. + Amend is allowed without a new staged diff because changing only the prior + commit message is a valid amend operation. + """ + if not message: + raise CommitError("commit message must not be empty") + if not amend and not _staged_changes_exist(repo): + raise NothingToCommitError("no staged changes to commit") + + head_before = _head_sha(repo, required=False) + + args = ["commit"] + if amend: + args.append("--amend") + args.extend(["-m", message]) + try: + committed = git(repo, *args) + except OSError as exc: + raise CommitError(f"failed to launch git commit in {repo}: {exc}") from exc + if committed.returncode != 0: + detail = (committed.stderr or committed.stdout).strip() + raise CommitError(detail or f"git commit failed with exit {committed.returncode}") + + commit_sha = _head_sha(repo, required=True) + assert commit_sha is not None + if commit_sha == head_before: + raise CommitError( + f"git commit returned success but HEAD did not advance from {head_before}" + ) + return CommitReceipt(commit_sha=commit_sha, message=message, amended=amend) diff --git a/gr2/python_cli/push.py b/gr2/python_cli/push.py new file mode 100644 index 0000000..b90ebc9 --- /dev/null +++ b/gr2/python_cli/push.py @@ -0,0 +1,155 @@ +"""Native single-repository push with explicit remote and arrival evidence.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from .gitops import git + + +class PushError(Exception): + pass + + +class PushEvidenceError(PushError): + """The push was acknowledged, but its remote result cannot be verified.""" + + +@dataclass(frozen=True) +class PushReceipt: + remote: str + branch: str + local_sha: str + remote_sha: str + set_upstream: bool + force_with_lease: bool + + +def _current_branch(repo: Path) -> str: + try: + result = git(repo, "branch", "--show-current") + except OSError as exc: + raise PushError(f"failed to determine the current branch in {repo}: {exc}") from exc + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + raise PushError(detail or "failed to determine the current branch") + branch = result.stdout.strip() + if not branch: + raise PushError("cannot push from detached HEAD") + return branch + + +def _remote_names(repo: Path) -> tuple[str, ...]: + try: + result = git(repo, "remote") + except OSError as exc: + raise PushError(f"failed to enumerate remotes in {repo}: {exc}") from exc + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + raise PushError(detail or "failed to enumerate git remotes") + return tuple(line.strip() for line in result.stdout.splitlines() if line.strip()) + + +def _select_remote(repo: Path, branch: str, explicit: str | None) -> str: + remotes = _remote_names(repo) + if explicit is not None: + if explicit not in remotes: + raise PushError(f"remote '{explicit}' is not configured in {repo}") + return explicit + + for key in (f"branch.{branch}.pushRemote", "remote.pushDefault", f"branch.{branch}.remote"): + try: + configured = git(repo, "config", "--get", key) + except OSError as exc: + raise PushError(f"failed to inspect configured push destination {key}: {exc}") from exc + if configured.returncode == 0 and configured.stdout.strip(): + remote = configured.stdout.strip() + if remote not in remotes: + raise PushError(f"{key} names unavailable remote '{remote}'") + return remote + if configured.returncode not in {0, 1}: + detail = (configured.stderr or configured.stdout).strip() + raise PushError(detail or f"failed to inspect configured push destination {key}") + + if len(remotes) == 1: + return remotes[0] + if not remotes: + raise PushError("no git remote is configured; pass --remote after adding one") + raise PushError( + f"multiple remotes are configured ({', '.join(remotes)}); pass --remote or configure the branch upstream" + ) + + +def _head_sha(repo: Path) -> str: + try: + result = git(repo, "rev-parse", "--verify", "HEAD") + except OSError as exc: + raise PushError(f"failed to resolve HEAD in {repo}: {exc}") from exc + sha = result.stdout.strip() if result.returncode == 0 else "" + if not sha: + raise PushError("cannot push because HEAD is not a commit") + return sha + + +def _remote_branch_sha(repo: Path, remote: str, branch: str) -> str: + ref = f"refs/heads/{branch}" + try: + result = git(repo, "ls-remote", "--heads", remote, ref) + except OSError as exc: + raise PushEvidenceError( + f"push was acknowledged but remote receipt evidence was unavailable: {exc}" + ) from exc + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + raise PushEvidenceError( + detail or "push was acknowledged but the remote branch could not be queried" + ) + rows = [line.split() for line in result.stdout.splitlines() if line.strip()] + matches = [parts[0] for parts in rows if len(parts) == 2 and parts[1] == ref] + if len(matches) != 1: + raise PushEvidenceError( + f"push was acknowledged but remote '{remote}' did not provide exactly one receipt for {ref}" + ) + return matches[0] + + +def push_current_branch( + repo: Path, + *, + remote: str | None = None, + set_upstream: bool = False, + force_with_lease: bool = False, +) -> PushReceipt: + """Push the current branch and verify that the remote ref equals HEAD.""" + branch = _current_branch(repo) + selected_remote = _select_remote(repo, branch, remote) + local_sha = _head_sha(repo) + + args = ["push"] + if set_upstream: + args.append("--set-upstream") + if force_with_lease: + args.append("--force-with-lease") + args.extend([selected_remote, branch]) + try: + pushed = git(repo, *args) + except OSError as exc: + raise PushError(f"failed to launch git push in {repo}: {exc}") from exc + if pushed.returncode != 0: + detail = (pushed.stderr or pushed.stdout).strip() + raise PushError(detail or f"git push failed with exit {pushed.returncode}") + + remote_sha = _remote_branch_sha(repo, selected_remote, branch) + if remote_sha != local_sha: + raise PushEvidenceError( + f"push was acknowledged but remote {selected_remote}/{branch} is {remote_sha}, expected {local_sha}" + ) + return PushReceipt( + remote=selected_remote, + branch=branch, + local_sha=local_sha, + remote_sha=remote_sha, + set_upstream=set_upstream, + force_with_lease=force_with_lease, + ) diff --git a/gr2/tests/test_add.py b/gr2/tests/test_add.py new file mode 100644 index 0000000..321250f --- /dev/null +++ b/gr2/tests/test_add.py @@ -0,0 +1,108 @@ +"""Executable contract for the native single-repository ``gr2 add`` verb.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest +from typer.testing import CliRunner + + +def _git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repo, + check=check, + capture_output=True, + text=True, + ) + + +def _init_repo(path: Path) -> Path: + path.mkdir() + _git(path, "init", "-b", "main") + _git(path, "config", "user.name", "Test") + _git(path, "config", "user.email", "test@example.com") + (path / "keep.txt").write_text("before\n") + (path / "delete.txt").write_text("delete me\n") + _git(path, "add", ".") + _git(path, "commit", "-m", "initial") + return path + + +def _cached_names(repo: Path) -> list[str]: + output = _git(repo, "diff", "--cached", "--name-only", "-z").stdout + return [name for name in output.split("\0") if name] + + +def test_stage_files_stages_added_modified_and_deleted_paths(tmp_path: Path) -> None: + from gr2.python_cli.add import stage_files + + repo = _init_repo(tmp_path / "repo") + (repo / "keep.txt").write_text("after\n") + (repo / "delete.txt").unlink() + (repo / "new.txt").write_text("new\n") + + result = stage_files(repo, ["keep.txt", "delete.txt", "new.txt"]) + + assert result.staged_files == ("delete.txt", "keep.txt", "new.txt") + assert _cached_names(repo) == ["delete.txt", "keep.txt", "new.txt"] + + +def test_deleted_path_is_staged_instead_of_misclassified_as_missing(tmp_path: Path) -> None: + from gr2.python_cli.add import stage_files + + repo = _init_repo(tmp_path / "repo") + (repo / "delete.txt").unlink() + + result = stage_files(repo, ["delete.txt"]) + + assert result.staged_files == ("delete.txt",) + assert ( + _git(repo, "diff", "--cached", "--diff-filter=D", "--name-only").stdout.strip() + == "delete.txt" + ) + + +def test_never_existing_path_refuses_without_touching_the_index(tmp_path: Path) -> None: + from gr2.python_cli.add import AddError, stage_files + + repo = _init_repo(tmp_path / "repo") + + with pytest.raises(AddError): + stage_files(repo, ["never-existed.txt"]) + + assert _cached_names(repo) == [] + + +def test_result_is_scoped_to_requested_paths_not_the_whole_index(tmp_path: Path) -> None: + from gr2.python_cli.add import stage_files + + repo = _init_repo(tmp_path / "repo") + (repo / "keep.txt").write_text("already staged\n") + _git(repo, "add", "keep.txt") + (repo / "new.txt").write_text("requested\n") + + result = stage_files(repo, ["new.txt"]) + + assert result.staged_files == ("new.txt",) + assert _cached_names(repo) == ["keep.txt", "new.txt"] + + +def test_add_cli_defaults_to_the_cwd_repository_only( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli.app import app + + repo_a = _init_repo(tmp_path / "a") + repo_b = _init_repo(tmp_path / "b") + (repo_a / "new.txt").write_text("a\n") + (repo_b / "new.txt").write_text("b\n") + monkeypatch.chdir(repo_a) + + result = CliRunner().invoke(app, ["add", "new.txt"]) + + assert result.exit_code == 0, result.output + assert _cached_names(repo_a) == ["new.txt"] + assert _cached_names(repo_b) == [] diff --git a/gr2/tests/test_commit.py b/gr2/tests/test_commit.py new file mode 100644 index 0000000..93e1b20 --- /dev/null +++ b/gr2/tests/test_commit.py @@ -0,0 +1,137 @@ +"""Executable contract for the native single-repository ``gr2 commit`` verb.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest +from typer.testing import CliRunner + + +def _git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repo, + check=check, + capture_output=True, + text=True, + ) + + +def _init_repo(path: Path) -> Path: + path.mkdir() + _git(path, "init", "-b", "main") + _git(path, "config", "user.name", "Test") + _git(path, "config", "user.email", "test@example.com") + (path / "tracked.txt").write_text("initial\n") + _git(path, "add", ".") + _git(path, "commit", "-m", "initial") + return path + + +def test_create_commit_returns_the_commit_it_actually_created(tmp_path: Path) -> None: + from gr2.python_cli.commit import create_commit + + repo = _init_repo(tmp_path / "repo") + (repo / "tracked.txt").write_text("changed\n") + _git(repo, "add", "tracked.txt") + + receipt = create_commit(repo, "native commit") + + actual_head = _git(repo, "rev-parse", "HEAD").stdout.strip() + assert receipt.commit_sha == actual_head + assert receipt.message == "native commit" + assert _git(repo, "show", "-s", "--format=%s", "HEAD").stdout.strip() == "native commit" + + +def test_no_staged_changes_refuses_structurally_before_commit(tmp_path: Path) -> None: + from gr2.python_cli.commit import NothingToCommitError, create_commit + + repo = _init_repo(tmp_path / "repo") + head_before = _git(repo, "rev-parse", "HEAD").stdout.strip() + + with pytest.raises(NothingToCommitError): + create_commit(repo, "must not exist") + + assert _git(repo, "rev-parse", "HEAD").stdout.strip() == head_before + + +def test_commit_outcome_uses_exit_status_not_command_prose( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli import commit as commit_ops + + repo = _init_repo(tmp_path / "repo") + calls: list[tuple[str, ...]] = [] + + def fake_git(_repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + calls.append(args) + if args[:3] == ("diff", "--cached", "--quiet"): + return subprocess.CompletedProcess(["git", *args], 1, "", "nothing to commit") + if args[0] == "commit": + return subprocess.CompletedProcess(["git", *args], 0, "", "nothing to commit") + if args == ("rev-parse", "--verify", "HEAD"): + sha = "b" * 40 if calls.count(args) == 1 else "a" * 40 + return subprocess.CompletedProcess(["git", *args], 0, sha + "\n", "") + raise AssertionError(args) + + monkeypatch.setattr(commit_ops, "git", fake_git) + + receipt = commit_ops.create_commit(repo, "status wins") + + assert receipt.commit_sha == "a" * 40 + assert any(args[0] == "commit" for args in calls) + + +def test_acknowledged_commit_without_a_new_head_refuses( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli import commit as commit_ops + + repo = _init_repo(tmp_path / "repo") + + def fake_git(_repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + if args[:3] == ("diff", "--cached", "--quiet"): + return subprocess.CompletedProcess(["git", *args], 1, "", "") + if args[0] == "commit": + return subprocess.CompletedProcess(["git", *args], 0, "", "") + if args == ("rev-parse", "--verify", "HEAD"): + return subprocess.CompletedProcess(["git", *args], 0, "a" * 40 + "\n", "") + raise AssertionError(args) + + monkeypatch.setattr(commit_ops, "git", fake_git) + + with pytest.raises(commit_ops.CommitError, match="HEAD did not advance"): + commit_ops.create_commit(repo, "no fruit") + + +def test_amend_without_new_staged_changes_is_allowed(tmp_path: Path) -> None: + from gr2.python_cli.commit import create_commit + + repo = _init_repo(tmp_path / "repo") + + receipt = create_commit(repo, "amended message", amend=True) + + assert receipt.amended is True + assert _git(repo, "show", "-s", "--format=%s", "HEAD").stdout.strip() == "amended message" + + +def test_commit_cli_defaults_to_the_cwd_repository_only( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli.app import app + + repo_a = _init_repo(tmp_path / "a") + repo_b = _init_repo(tmp_path / "b") + for repo in (repo_a, repo_b): + (repo / "tracked.txt").write_text("changed\n") + _git(repo, "add", "tracked.txt") + before_b = _git(repo_b, "rev-parse", "HEAD").stdout.strip() + monkeypatch.chdir(repo_a) + + result = CliRunner().invoke(app, ["commit", "-m", "cwd only"]) + + assert result.exit_code == 0, result.output + assert _git(repo_a, "show", "-s", "--format=%s", "HEAD").stdout.strip() == "cwd only" + assert _git(repo_b, "rev-parse", "HEAD").stdout.strip() == before_b diff --git a/gr2/tests/test_push.py b/gr2/tests/test_push.py new file mode 100644 index 0000000..12ceb42 --- /dev/null +++ b/gr2/tests/test_push.py @@ -0,0 +1,228 @@ +"""Executable contract for the native single-repository ``gr2 push`` verb.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest +from typer.testing import CliRunner + + +def _git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repo, + check=check, + capture_output=True, + text=True, + ) + + +def _init_repo(path: Path) -> Path: + path.mkdir() + _git(path, "init", "-b", "main") + _git(path, "config", "user.name", "Test") + _git(path, "config", "user.email", "test@example.com") + (path / "tracked.txt").write_text("initial\n") + _git(path, "add", ".") + _git(path, "commit", "-m", "initial") + return path + + +def _bare(path: Path) -> Path: + subprocess.run(["git", "init", "--bare", str(path)], check=True, capture_output=True, text=True) + return path + + +def _commit(repo: Path, text: str) -> str: + (repo / "tracked.txt").write_text(text + "\n") + _git(repo, "add", "tracked.txt") + _git(repo, "commit", "-m", text) + return _git(repo, "rev-parse", "HEAD").stdout.strip() + + +def test_push_uses_the_configured_non_origin_remote_and_verifies_arrival(tmp_path: Path) -> None: + from gr2.python_cli.push import push_current_branch + + repo = _init_repo(tmp_path / "repo") + remote = _bare(tmp_path / "remote.git") + _git(repo, "remote", "add", "upstream", str(remote)) + + receipt = push_current_branch(repo, set_upstream=True) + + remote_head = _git(remote, "rev-parse", "refs/heads/main").stdout.strip() + assert receipt.remote == "upstream" + assert receipt.branch == "main" + assert receipt.local_sha == remote_head + assert receipt.remote_sha == remote_head + assert _git(repo, "config", "--get", "branch.main.remote").stdout.strip() == "upstream" + + +def test_multiple_remotes_without_a_configured_or_explicit_choice_refuse(tmp_path: Path) -> None: + from gr2.python_cli.push import PushError, push_current_branch + + repo = _init_repo(tmp_path / "repo") + _git(repo, "remote", "add", "one", str(_bare(tmp_path / "one.git"))) + _git(repo, "remote", "add", "two", str(_bare(tmp_path / "two.git"))) + + with pytest.raises(PushError, match="multiple remotes"): + push_current_branch(repo) + + +def test_branch_push_remote_takes_precedence_over_fetch_remote(tmp_path: Path) -> None: + from gr2.python_cli.push import push_current_branch + + repo = _init_repo(tmp_path / "repo") + fetcher = _bare(tmp_path / "fetcher.git") + pusher = _bare(tmp_path / "pusher.git") + _git(repo, "remote", "add", "fetcher", str(fetcher)) + _git(repo, "remote", "add", "pusher", str(pusher)) + _git(repo, "config", "branch.main.remote", "fetcher") + _git(repo, "config", "branch.main.pushRemote", "pusher") + + receipt = push_current_branch(repo) + + assert receipt.remote == "pusher" + assert _git(pusher, "rev-parse", "refs/heads/main").stdout.strip() == receipt.local_sha + assert _git(fetcher, "show-ref", "--verify", "refs/heads/main", check=False).returncode != 0 + + +def test_explicit_remote_selects_one_of_multiple_remotes(tmp_path: Path) -> None: + from gr2.python_cli.push import push_current_branch + + repo = _init_repo(tmp_path / "repo") + one = _bare(tmp_path / "one.git") + two = _bare(tmp_path / "two.git") + _git(repo, "remote", "add", "one", str(one)) + _git(repo, "remote", "add", "two", str(two)) + + receipt = push_current_branch(repo, remote="two") + + assert receipt.remote == "two" + assert _git(two, "rev-parse", "refs/heads/main").stdout.strip() == receipt.local_sha + assert _git(one, "show-ref", "--verify", "refs/heads/main", check=False).returncode != 0 + + +def test_detached_head_refuses_before_any_push(tmp_path: Path) -> None: + from gr2.python_cli.push import PushError, push_current_branch + + repo = _init_repo(tmp_path / "repo") + remote = _bare(tmp_path / "remote.git") + _git(repo, "remote", "add", "upstream", str(remote)) + _git(repo, "checkout", "--detach", "HEAD") + + with pytest.raises(PushError, match="detached HEAD"): + push_current_branch(repo) + + assert _git(remote, "show-ref", "--verify", "refs/heads/main", check=False).returncode != 0 + + +def test_divergent_push_refuses_without_rewriting_the_remote(tmp_path: Path) -> None: + from gr2.python_cli.push import PushError, push_current_branch + + repo = _init_repo(tmp_path / "repo") + remote = _bare(tmp_path / "remote.git") + _git(repo, "remote", "add", "upstream", str(remote)) + push_current_branch(repo, set_upstream=True) + + other = tmp_path / "other" + subprocess.run( + ["git", "clone", str(remote), str(other)], check=True, capture_output=True, text=True + ) + _git(other, "config", "user.name", "Other") + _git(other, "config", "user.email", "other@example.com") + remote_advanced = _commit(other, "remote advance") + _git(other, "push", "origin", "main") + + _commit(repo, "local divergence") + with pytest.raises(PushError): + push_current_branch(repo) + + assert _git(remote, "rev-parse", "refs/heads/main").stdout.strip() == remote_advanced + + +def test_acknowledged_push_without_matching_receipt_evidence_refuses( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli import push as push_ops + + repo = _init_repo(tmp_path / "repo") + + def fake_git(_repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + if args == ("branch", "--show-current"): + return subprocess.CompletedProcess(["git", *args], 0, "main\n", "") + if args[:2] == ("config", "--get"): + if args[2] == "branch.main.remote": + return subprocess.CompletedProcess(["git", *args], 0, "upstream\n", "") + return subprocess.CompletedProcess(["git", *args], 1, "", "") + if args == ("remote",): + return subprocess.CompletedProcess(["git", *args], 0, "upstream\n", "") + if args == ("rev-parse", "--verify", "HEAD"): + return subprocess.CompletedProcess(["git", *args], 0, "a" * 40 + "\n", "") + if args[:2] == ("push", "upstream"): + return subprocess.CompletedProcess(["git", *args], 0, "", "") + if args[:3] == ("ls-remote", "--heads", "upstream"): + return subprocess.CompletedProcess(["git", *args], 0, "", "") + raise AssertionError(args) + + monkeypatch.setattr(push_ops, "git", fake_git) + + with pytest.raises(push_ops.PushEvidenceError, match="did not provide"): + push_ops.push_current_branch(repo) + + +def test_force_with_lease_is_threaded_to_git_without_raw_force( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli import push as push_ops + + repo = _init_repo(tmp_path / "repo") + push_args: tuple[str, ...] | None = None + + def fake_git(_repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + nonlocal push_args + if args == ("branch", "--show-current"): + return subprocess.CompletedProcess(["git", *args], 0, "main\n", "") + if args == ("remote",): + return subprocess.CompletedProcess(["git", *args], 0, "upstream\n", "") + if args[:2] == ("config", "--get"): + if args[2] == "branch.main.remote": + return subprocess.CompletedProcess(["git", *args], 0, "upstream\n", "") + return subprocess.CompletedProcess(["git", *args], 1, "", "") + if args == ("rev-parse", "--verify", "HEAD"): + return subprocess.CompletedProcess(["git", *args], 0, "a" * 40 + "\n", "") + if args[0] == "push": + push_args = args + return subprocess.CompletedProcess(["git", *args], 0, "", "") + if args[:3] == ("ls-remote", "--heads", "upstream"): + row = "a" * 40 + "\trefs/heads/main\n" + return subprocess.CompletedProcess(["git", *args], 0, row, "") + raise AssertionError(args) + + monkeypatch.setattr(push_ops, "git", fake_git) + + push_ops.push_current_branch(repo, force_with_lease=True) + + assert push_args is not None + assert "--force-with-lease" in push_args + assert "--force" not in push_args + + +def test_cli_exposes_force_with_lease_but_never_raw_force(tmp_path: Path) -> None: + from gr2.python_cli.app import app + + repo = _init_repo(tmp_path / "repo") + remote = _bare(tmp_path / "remote.git") + _git(repo, "remote", "add", "upstream", str(remote)) + runner = CliRunner() + + safe = runner.invoke( + app, + ["push", "--repo-path", str(repo), "--set-upstream", "--force-with-lease"], + ) + unsafe = runner.invoke(app, ["push", "--repo-path", str(repo), "--force"]) + + assert safe.exit_code == 0, safe.output + assert unsafe.exit_code != 0 + assert "No such option: --force" in unsafe.output From 1592c3e9aacbf04e8bc929f088985a9d2de18054 Mon Sep 17 00:00:00 2001 From: Atlas Date: Mon, 17 Aug 2026 04:42:34 -0500 Subject: [PATCH 02/29] test(gr2): bind daily verb CLI forwarding --- gr2/tests/test_add.py | 21 +++++++++++++ gr2/tests/test_commit.py | 46 ++++++++++++++++++++++++++++ gr2/tests/test_push.py | 65 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+) diff --git a/gr2/tests/test_add.py b/gr2/tests/test_add.py index 321250f..b2e5acb 100644 --- a/gr2/tests/test_add.py +++ b/gr2/tests/test_add.py @@ -106,3 +106,24 @@ def test_add_cli_defaults_to_the_cwd_repository_only( assert result.exit_code == 0, result.output assert _cached_names(repo_a) == ["new.txt"] assert _cached_names(repo_b) == [] + + +def test_add_cli_honors_explicit_repo_path_over_cwd( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli.app import app + + cwd_repo = _init_repo(tmp_path / "cwd") + requested_repo = _init_repo(tmp_path / "requested") + (cwd_repo / "new.txt").write_text("cwd\n") + (requested_repo / "new.txt").write_text("requested\n") + monkeypatch.chdir(cwd_repo) + + result = CliRunner().invoke( + app, + ["add", "--repo-path", str(requested_repo), "new.txt"], + ) + + assert result.exit_code == 0, result.output + assert _cached_names(cwd_repo) == [] + assert _cached_names(requested_repo) == ["new.txt"] diff --git a/gr2/tests/test_commit.py b/gr2/tests/test_commit.py index 93e1b20..941bc0b 100644 --- a/gr2/tests/test_commit.py +++ b/gr2/tests/test_commit.py @@ -135,3 +135,49 @@ def test_commit_cli_defaults_to_the_cwd_repository_only( assert result.exit_code == 0, result.output assert _git(repo_a, "show", "-s", "--format=%s", "HEAD").stdout.strip() == "cwd only" assert _git(repo_b, "rev-parse", "HEAD").stdout.strip() == before_b + + +def test_commit_cli_honors_explicit_repo_path_over_cwd( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli.app import app + + cwd_repo = _init_repo(tmp_path / "cwd") + requested_repo = _init_repo(tmp_path / "requested") + for repo in (cwd_repo, requested_repo): + (repo / "tracked.txt").write_text("changed\n") + _git(repo, "add", "tracked.txt") + cwd_head = _git(cwd_repo, "rev-parse", "HEAD").stdout.strip() + monkeypatch.chdir(cwd_repo) + + result = CliRunner().invoke( + app, + ["commit", "--repo-path", str(requested_repo), "-m", "requested only"], + ) + + assert result.exit_code == 0, result.output + assert _git(cwd_repo, "rev-parse", "HEAD").stdout.strip() == cwd_head + assert ( + _git(requested_repo, "show", "-s", "--format=%s", "HEAD").stdout.strip() + == "requested only" + ) + + +def test_commit_cli_threads_amend_without_new_staged_changes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli.app import app + + repo = _init_repo(tmp_path / "repo") + monkeypatch.chdir(tmp_path) + + result = CliRunner().invoke( + app, + ["commit", "--repo-path", str(repo), "--amend", "-m", "amended through cli"], + ) + + assert result.exit_code == 0, result.output + assert ( + _git(repo, "show", "-s", "--format=%s", "HEAD").stdout.strip() + == "amended through cli" + ) diff --git a/gr2/tests/test_push.py b/gr2/tests/test_push.py index 12ceb42..688efb3 100644 --- a/gr2/tests/test_push.py +++ b/gr2/tests/test_push.py @@ -226,3 +226,68 @@ def test_cli_exposes_force_with_lease_but_never_raw_force(tmp_path: Path) -> Non assert safe.exit_code == 0, safe.output assert unsafe.exit_code != 0 assert "No such option: --force" in unsafe.output + assert _git(repo, "config", "--get", "branch.main.remote").stdout.strip() == "upstream" + + +def test_push_cli_honors_explicit_repo_path_and_remote( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli.app import app + + repo = _init_repo(tmp_path / "repo") + one = _bare(tmp_path / "one.git") + two = _bare(tmp_path / "two.git") + _git(repo, "remote", "add", "one", str(one)) + _git(repo, "remote", "add", "two", str(two)) + monkeypatch.chdir(tmp_path) + + result = CliRunner().invoke( + app, + ["push", "--repo-path", str(repo), "--remote", "two"], + ) + + assert result.exit_code == 0, result.output + assert _git(two, "rev-parse", "refs/heads/main").stdout.strip() == _git( + repo, "rev-parse", "HEAD" + ).stdout.strip() + assert _git(one, "show-ref", "--verify", "refs/heads/main", check=False).returncode != 0 + + +def test_push_cli_threads_force_with_lease_when_remote_has_advanced( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gr2.python_cli.app import app + from gr2.python_cli.push import push_current_branch + + repo = _init_repo(tmp_path / "repo") + remote = _bare(tmp_path / "remote.git") + _git(repo, "remote", "add", "upstream", str(remote)) + push_current_branch(repo, set_upstream=True) + + local_sha = _commit(repo, "local divergence") + other = tmp_path / "other" + subprocess.run( + ["git", "clone", str(remote), str(other)], check=True, capture_output=True, text=True + ) + _git(other, "config", "user.name", "Other") + _git(other, "config", "user.email", "other@example.com") + remote_advanced = _commit(other, "remote advance") + _git(other, "push", "origin", "main") + _git(repo, "fetch", "upstream", "main") + assert _git(repo, "rev-parse", "upstream/main").stdout.strip() == remote_advanced + monkeypatch.chdir(tmp_path) + + result = CliRunner().invoke( + app, + [ + "push", + "--repo-path", + str(repo), + "--remote", + "upstream", + "--force-with-lease", + ], + ) + + assert result.exit_code == 0, result.output + assert _git(remote, "rev-parse", "refs/heads/main").stdout.strip() == local_sha From 4f0170940cef2720d7e17faccc4370a729964956 Mon Sep 17 00:00:00 2001 From: Atlas Date: Wed, 19 Aug 2026 04:47:07 -0500 Subject: [PATCH 03/29] fix: make PR merge refusals observable --- src/cli/commands/pr/merge.rs | 11 +++- src/cli/mod.rs | 1 + src/cli/outcome.rs | 103 +++++++++++++++++++++++++++++++++++ src/core/repo.rs | 79 ++++++++++++++++++++++++++- src/main.rs | 14 ++++- tests/cli_tests.rs | 39 +++++++++++++ tests/test_pr_merge.rs | 87 ++++++++++++++++++++++++++--- 7 files changed, 319 insertions(+), 15 deletions(-) create mode 100644 src/cli/outcome.rs diff --git a/src/cli/commands/pr/merge.rs b/src/cli/commands/pr/merge.rs index 1033e9c..76f28c9 100644 --- a/src/cli/commands/pr/merge.rs +++ b/src/cli/commands/pr/merge.rs @@ -1,9 +1,13 @@ //! PR merge command implementation use super::create::has_commits_ahead; +use crate::cli::outcome::CliOutcomeError; use crate::cli::output::Output; use crate::core::manifest::Manifest; -use crate::core::repo::{get_manifest_repo_info, require_explicit_multi_repo_scope, RepoInfo}; +use crate::core::repo::{ + get_manifest_repo_info, require_explicit_multi_repo_scope, validate_repo_filters_known, + RepoInfo, +}; use crate::git::{get_current_branch, open_repo, path_exists}; use crate::platform::traits::{HostingPlatform, PlatformError}; use crate::platform::{get_platform_adapter, CheckState, MergeMethod, StatusCheckResult}; @@ -240,6 +244,9 @@ pub async fn run_pr_merge( manifest: &Manifest, opts: &MergeOptions<'_>, ) -> anyhow::Result<()> { + validate_repo_filters_known(manifest, opts.repo_filter.as_deref()) + .map_err(|error| CliOutcomeError::refusal(error.to_string()))?; + if !opts.json { Output::header("Merging pull requests..."); println!(); @@ -660,7 +667,7 @@ pub async fn run_pr_merge( .collect::>() .join("|") ); - return Ok(()); + return Err(CliOutcomeError::reported_refusal("merge refused by readiness gate").into()); } // Auto-merge flow: enable auto-merge and return early diff --git a/src/cli/mod.rs b/src/cli/mod.rs index b4be59b..56509c3 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -6,6 +6,7 @@ pub mod args; pub mod commands; pub mod context; pub mod dispatch; +pub mod outcome; pub mod output; pub mod output_sink; pub mod repo_iter; diff --git a/src/cli/outcome.rs b/src/cli/outcome.rs new file mode 100644 index 0000000..2ecc71c --- /dev/null +++ b/src/cli/outcome.rs @@ -0,0 +1,103 @@ +//! Process-level outcome classification for CLI commands. +//! +//! `anyhow::Result` distinguishes success from failure, but not a command that +//! was operationally unable to run from one that deliberately refused an act. +//! Callers need that distinction without parsing rendered prose. + +use thiserror::Error; + +/// Exit code used when the command understood the request but refused the act. +pub const EXIT_REFUSED: u8 = 2; + +/// An error whose process status and rendering behavior are part of the CLI contract. +#[derive(Debug, Error)] +#[error("{message}")] +pub struct CliOutcomeError { + exit_code: u8, + already_reported: bool, + message: String, +} + +impl CliOutcomeError { + /// Refuse before any command-specific diagnostic has been rendered. + pub fn refusal(message: impl Into) -> Self { + Self { + exit_code: EXIT_REFUSED, + already_reported: false, + message: message.into(), + } + } + + /// Refuse after the command has already rendered the actionable diagnostic. + pub fn reported_refusal(message: impl Into) -> Self { + Self { + exit_code: EXIT_REFUSED, + already_reported: true, + message: message.into(), + } + } + + pub fn exit_code(&self) -> u8 { + self.exit_code + } + + pub fn already_reported(&self) -> bool { + self.already_reported + } +} + +/// Resolve the process code without requiring callers to know concrete error types. +pub fn exit_code_for_error(error: &anyhow::Error) -> u8 { + error + .downcast_ref::() + .map(CliOutcomeError::exit_code) + .unwrap_or(1) +} + +/// Whether the command already printed the diagnostic that explains this failure. +pub fn error_was_reported(error: &anyhow::Error) -> bool { + error + .downcast_ref::() + .map(CliOutcomeError::already_reported) + .unwrap_or(false) +} + +/// Render an unreported error with the same Debug shape used by the previous +/// `Result<(), anyhow::Error>` process entry point. +pub fn render_unreported_error(error: &anyhow::Error) -> String { + format!("Error: {error:?}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn refusal_is_distinct_from_operational_failure() { + let refused = anyhow::Error::from(CliOutcomeError::refusal("not ready")); + let operational = anyhow::anyhow!("network failed"); + + assert_eq!(exit_code_for_error(&refused), EXIT_REFUSED); + assert_eq!(exit_code_for_error(&operational), 1); + } + + #[test] + fn only_reported_refusals_suppress_duplicate_rendering() { + let pending = anyhow::Error::from(CliOutcomeError::refusal("bad selector")); + let rendered = anyhow::Error::from(CliOutcomeError::reported_refusal("not ready")); + + assert!(!error_was_reported(&pending)); + assert!(error_was_reported(&rendered)); + } + + #[test] + fn unreported_errors_keep_the_prior_debug_chain_shape() { + let error = anyhow::anyhow!("inner failure").context("outer context"); + let rendered = render_unreported_error(&error); + + assert!( + rendered.starts_with("Error: outer context\n\nCaused by:\n inner failure"), + "unexpected rendering: {rendered:?}" + ); + } +} diff --git a/src/core/repo.rs b/src/core/repo.rs index 280dcb1..ae86f86 100644 --- a/src/core/repo.rs +++ b/src/core/repo.rs @@ -374,9 +374,35 @@ pub fn validate_repo_filters_known( .map(|name| format!("'{}'", name)) .collect::>() .join(", "); + let mut suggestions = missing + .iter() + .flat_map(|filter| { + manifest.repos.iter().filter_map(|(manifest_name, config)| { + let matches_remote_name = config + .url + .as_deref() + .and_then(parse_git_url) + .is_some_and(|parsed| parsed.repo == filter.as_str()); + let matches_path_name = std::path::Path::new(&config.path) + .file_name() + .is_some_and(|name| name == std::ffi::OsStr::new(filter.as_str())); + + (matches_remote_name || matches_path_name) + .then(|| format!("'{}' ({})", manifest_name, config.path)) + }) + }) + .collect::>(); + suggestions.sort(); + suggestions.dedup(); + let suggestion = if suggestions.is_empty() { + String::new() + } else { + format!(" Did you mean {}?", suggestions.join(" or ")) + }; anyhow::bail!( - "repo filter {} not found in local manifest. If it was added upstream, run `gr sync --repo manifest` or plain `gr sync` first, then retry.", - names + "repo filter {} not found in local manifest.{} If it was added upstream, run `gr sync --repo manifest` or plain `gr sync` first, then retry.", + names, + suggestion ); } @@ -1288,4 +1314,53 @@ repos: .is_ok() ); } + + #[test] + fn unknown_manifest_name_suggests_the_matching_remote_repo_name() { + let manifest = Manifest::parse( + r#" +repos: + synapt: + url: https://github.com/example/recall.git + path: ./synapt +"#, + ) + .unwrap(); + + let err = validate_repo_filters_known(&manifest, Some(&["recall".to_string()])) + .expect_err("a remote repository name is not a manifest selector"); + + assert!( + err.to_string() + .contains("Did you mean 'synapt' (./synapt)?"), + "the refusal should bridge from the remote name to the manifest selector: {err}" + ); + } + + #[test] + fn remote_name_hint_lists_every_matching_manifest_key_deterministically() { + let manifest = Manifest::parse( + r#" +repos: + grip-next: + url: https://github.com/example/grip.git + path: ./next + revision: next + grip-stable: + url: https://github.com/example/grip.git + path: ./stable + revision: stable +"#, + ) + .unwrap(); + + let err = validate_repo_filters_known(&manifest, Some(&["grip".to_string()])) + .expect_err("a remote repository name is not a manifest selector"); + + assert!( + err.to_string() + .contains("Did you mean 'grip-next' (./next) or 'grip-stable' (./stable)?"), + "every matching manifest key should be listed in stable order: {err}" + ); + } } diff --git a/src/main.rs b/src/main.rs index 5edd64c..1d73dae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,9 +2,11 @@ use clap::Parser; use gitgrip::cli::args::Cli; +use gitgrip::cli::outcome::{error_was_reported, exit_code_for_error, render_unreported_error}; +use std::process::ExitCode; #[tokio::main] -async fn main() -> anyhow::Result<()> { +async fn main() -> ExitCode { let cli = Cli::parse(); // Initialize tracing — `--verbose` enables debug logging for gitgrip @@ -23,5 +25,13 @@ async fn main() -> anyhow::Result<()> { let verbose = cli.verbose; let json = cli.json; - gitgrip::cli::dispatch::dispatch_command(cli.command, quiet, verbose, json).await + match gitgrip::cli::dispatch::dispatch_command(cli.command, quiet, verbose, json).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + if !error_was_reported(&error) { + eprintln!("{}", render_unreported_error(&error)); + } + ExitCode::from(exit_code_for_error(&error)) + } + } } diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index 678700b..5be90a7 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -251,6 +251,45 @@ fn test_checkout_add_errors_when_filters_match_no_repos() { )); } +#[test] +fn test_pr_merge_unknown_repo_is_a_process_level_refusal() { + let ws = WorkspaceBuilder::new().add_repo("app").build(); + + let mut cmd = Command::cargo_bin("gr").unwrap(); + cmd.current_dir(&ws.workspace_root) + .arg("pr") + .arg("merge") + .arg("--repo") + .arg("missing") + .arg("--method") + .arg("merge") + .arg("--yes") + .assert() + .code(2) + .stderr(predicate::str::contains( + "repo filter 'missing' not found in local manifest", + )); +} + +#[test] +fn test_pr_merge_known_repo_with_no_open_pr_is_still_success() { + let ws = WorkspaceBuilder::new().add_repo("app").build(); + + let mut cmd = Command::cargo_bin("gr").unwrap(); + cmd.current_dir(&ws.workspace_root) + .arg("pr") + .arg("merge") + .arg("--repo") + .arg("app") + .arg("--method") + .arg("merge") + .arg("--yes") + .assert() + .success() + .stdout(predicate::str::contains("No open PRs found")) + .stdout(predicate::str::contains("Repositories checked: 1")); +} + #[test] fn test_checkout_add_rejects_create_and_base_flags() { let ws = WorkspaceBuilder::new().add_repo("app").build(); diff --git a/tests/test_pr_merge.rs b/tests/test_pr_merge.rs index 040ea65..3cf186e 100644 --- a/tests/test_pr_merge.rs +++ b/tests/test_pr_merge.rs @@ -431,10 +431,11 @@ async fn test_pr_merge_repo_filter_excludes_non_target() { } // ── Repo Filter: No Matching Repos ──────────────────────────── -// When --repo names a repo that doesn't exist, all repos are filtered out. +// An explicit selector names something the caller believes exists. Empty is +// therefore a refusal, not a successful operation over an empty set. #[tokio::test] -async fn test_pr_merge_repo_filter_no_match_finds_no_prs() { +async fn test_pr_merge_repo_filter_no_match_is_a_usage_refusal() { let ws = WorkspaceBuilder::new().add_repo("app").build(); let manifest = ws.load_manifest(); @@ -463,10 +464,11 @@ async fn test_pr_merge_repo_filter_no_match_finds_no_prs() { ) .await; + let err = result.expect_err("an unknown explicit --repo selector must fail"); + assert_eq!(gitgrip::cli::outcome::exit_code_for_error(&err), 2); assert!( - result.is_ok(), - "repo filter with no matches should succeed with 'no PRs found': {:?}", - result.err() + err.to_string().contains("nonexistent"), + "the refusal must name the selector that matched nothing: {err}" ); } @@ -825,10 +827,11 @@ async fn test_skip_gate_approval_does_not_also_waive_checks() { ) .await; - assert!( - result.is_ok(), - "should report, not error: {:?}", - result.err() + let err = result.expect_err("a live readiness gate must refuse the merge"); + assert_eq!( + gitgrip::cli::outcome::exit_code_for_error(&err), + 2, + "readiness refusal must be distinguishable from operational failure" ); let requests = server.received_requests().await.unwrap(); @@ -841,6 +844,72 @@ async fn test_skip_gate_approval_does_not_also_waive_checks() { ); } +#[tokio::test] +async fn test_readiness_refusal_reaches_the_process_exit_status() { + let (server, _adapter) = setup_github_mock().await; + + let ws = WorkspaceBuilder::new().add_repo("app").build(); + let mut manifest = ws.load_manifest(); + + git_helpers::create_branch(&ws.repo_path("app"), "feat/test"); + git_helpers::commit_file( + &ws.repo_path("app"), + "feature.txt", + "feature", + "Add feature", + ); + + point_repo_at_mock(&mut manifest, "app", &server); + let manifest_yaml = serde_yaml::to_string(&manifest).unwrap(); + std::fs::write( + ws.workspace_root.join(".gitgrip/spaces/main/gripspace.yml"), + manifest_yaml, + ) + .unwrap(); + + mock_list_prs(&server, vec![(42, "feat/test")]).await; + mock_get_pr(&server, 42, "open", false).await; + mock_pr_reviews(&server, 42, vec![("COMMENTED", "alice")]).await; + mock_check_runs(&server, "feat/test", vec![("CI", "in_progress", None)]).await; + mock_merge_pr(&server, 42, true).await; + + let output = tokio::process::Command::new(assert_cmd::cargo::cargo_bin!("gr")) + .current_dir(&ws.workspace_root) + .env("GITHUB_TOKEN", "test") + .args([ + "pr", + "merge", + "--skip-gate", + "approval", + "--method", + "merge", + "--yes", + ]) + .output() + .await + .unwrap(); + + assert_eq!( + output.status.code(), + Some(2), + "readiness refusal must survive dispatch and main; stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + !String::from_utf8_lossy(&output.stderr).contains("Error: merge refused"), + "main must not append a generic error after the command rendered the actionable refusal" + ); + + let requests = server.received_requests().await.unwrap(); + assert!( + !requests + .iter() + .any(|r| r.method == Method::PUT && r.url.path().ends_with("/merge")), + "a refusal with exit 2 must still send no merge request" + ); +} + /// Waiving `approval` on a PR whose only failing gate IS approval must proceed. /// /// The negative control for the test above. Without it, a `--skip-gate` that From 447557d5a7b3d5b5387bda30c758ce384ad9bd4f Mon Sep 17 00:00:00 2001 From: Atlas Date: Wed, 19 Aug 2026 09:39:20 -0500 Subject: [PATCH 04/29] fix(link): refuse stale gripspace sources Manual link application now fetches every referenced branch-backed gripspace and refuses before writing when behind upstream. Materialization records the requested revision so a detached HEAD is accepted only when it matches an explicit tag or commit pin. Rev-less or branch-configured detachment refuses rather than claiming freshness from an unproven state. Tests cover source enumeration, stale refusal and recovery, detached refusal, and explicit-SHA acceptance. --- src/cli/commands/link.rs | 273 +++++++++++++++++++++++++++++++++- src/core/gripspace.rs | 52 +++++++ tests/link_apply_freshness.rs | 228 ++++++++++++++++++++++++++++ 3 files changed, 550 insertions(+), 3 deletions(-) create mode 100644 tests/link_apply_freshness.rs diff --git a/src/cli/commands/link.rs b/src/cli/commands/link.rs index f8d74a9..9927c26 100644 --- a/src/cli/commands/link.rs +++ b/src/cli/commands/link.rs @@ -2,12 +2,15 @@ //! //! Manages copyfile and linkfile entries. +use crate::cli::outcome::CliOutcomeError; use crate::cli::output::Output; +use crate::core::gripspace::requested_gripspace_revision; use crate::core::manifest::Manifest; use crate::core::manifest_paths; use crate::core::repo::RepoInfo; use crate::files::{process_composefiles, resolve_file_source}; -use crate::git::path_exists; +use crate::git::{fetch_remote, path_exists}; +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; /// Check if a source path contains glob characters (`*`, `?`, `[`). @@ -118,6 +121,7 @@ pub fn run_link( if status { show_link_status(workspace_root, manifest, json)?; } else if apply { + ensure_gripspace_sources_current(workspace_root, manifest)?; apply_links(workspace_root, manifest, false)?; } else { // Default: show status @@ -127,6 +131,152 @@ pub fn run_link( Ok(()) } +/// Names of gripspace clones whose bytes can reach a manual link application. +/// +/// Resolution rewrites inherited composefile parts to their materialized +/// directory name. Copyfile and linkfile sources retain the explicit +/// `gripspace::` spelling. Collect both forms so the freshness +/// check and the writer cover the same source classes. +fn referenced_gripspaces(manifest: &Manifest) -> BTreeSet { + let mut names = BTreeSet::new(); + let Some(config) = &manifest.manifest else { + return names; + }; + + for source in config + .copyfile + .iter() + .flatten() + .map(|entry| entry.src.as_str()) + .chain( + config + .linkfile + .iter() + .flatten() + .map(|entry| entry.src.as_str()), + ) + { + if let Some(rest) = source.strip_prefix("gripspace:") { + if let Some((name, _)) = rest.split_once(':') { + names.insert(name.to_string()); + } + } + } + + for name in config + .composefile + .iter() + .flatten() + .flat_map(|compose| &compose.parts) + .filter_map(|part| part.gripspace.as_ref()) + { + names.insert(name.clone()); + } + + names +} + +/// Refuse a manual apply when a branch-backed gripspace is behind upstream. +/// +/// A successful local composition proves that the files are internally +/// usable. It does not prove that the source clone is current. Fetch first, +/// then compare graph position. A detached HEAD is accepted only when the +/// materializer recorded an explicit tag or commit revision and HEAD still +/// resolves to that pin. Detachment by itself is not evidence of a pin. +fn ensure_gripspace_sources_current( + workspace_root: &Path, + manifest: &Manifest, +) -> anyhow::Result<()> { + let spaces_dir = manifest_paths::spaces_dir(workspace_root); + + for name in referenced_gripspaces(manifest) { + let path = spaces_dir.join(&name); + let repo = git2::Repository::open(&path).map_err(|error| { + anyhow::anyhow!( + "Cannot verify gripspace source '{}' at {}: {}", + name, + path.display(), + error + ) + })?; + fetch_remote(&repo, "origin").map_err(|error| { + anyhow::anyhow!("Cannot verify gripspace source '{name}' against origin: {error}") + })?; + + let head = repo.head()?; + let local_oid = head + .target() + .ok_or_else(|| anyhow::anyhow!("Cannot resolve HEAD for gripspace source '{name}'"))?; + if !head.is_branch() { + let requested_rev = requested_gripspace_revision(&path).map_err(|error| { + CliOutcomeError::refusal(format!( + "Cannot verify detached gripspace source '{name}': {error}" + )) + })?; + let Some(rev) = requested_rev else { + return Err(CliOutcomeError::refusal(format!( + "Gripspace source '{name}' is unexpectedly detached without an explicit revision pin. Run `gr sync` before `gr link --apply`." + )) + .into()); + }; + + let remote_branch = format!("refs/remotes/origin/{rev}"); + if repo.find_reference(&remote_branch).is_ok() { + return Err(CliOutcomeError::refusal(format!( + "Gripspace source '{name}' is detached from configured branch '{rev}'. Run `gr sync` before `gr link --apply`." + )) + .into()); + } + + let pinned_oid = repo + .revparse_single(&rev) + .and_then(|object| object.peel_to_commit()) + .map(|commit| commit.id()) + .map_err(|error| { + CliOutcomeError::refusal(format!( + "Cannot verify gripspace source '{name}' at configured revision '{rev}': {error}" + )) + })?; + if pinned_oid != local_oid { + return Err(CliOutcomeError::refusal(format!( + "Gripspace source '{name}' does not match configured revision '{rev}'. Run `gr sync` before `gr link --apply`." + )) + .into()); + } + continue; + } + + let branch = head + .shorthand() + .ok_or_else(|| anyhow::anyhow!("Cannot identify branch for gripspace source '{name}'"))? + .to_string(); + + let remote_ref = format!("refs/remotes/origin/{branch}"); + let remote_oid = repo + .find_reference(&remote_ref) + .and_then(|reference| { + reference + .target() + .ok_or_else(|| git2::Error::from_str(&format!("{remote_ref} has no target"))) + }) + .map_err(|error| { + anyhow::anyhow!( + "Cannot verify gripspace source '{name}' against origin/{branch}: {error}" + ) + })?; + let (_, behind) = repo.graph_ahead_behind(local_oid, remote_oid)?; + + if behind > 0 { + return Err(CliOutcomeError::refusal(format!( + "Gripspace source '{name}' is behind origin/{branch} by {behind} commit(s). Run `gr sync` before `gr link --apply`." + )) + .into()); + } + } + + Ok(()) +} + fn show_link_status(workspace_root: &Path, manifest: &Manifest, json: bool) -> anyhow::Result<()> { if !json { Output::header("File Link Status"); @@ -774,12 +924,45 @@ pub fn apply_links(workspace_root: &Path, manifest: &Manifest, quiet: bool) -> a mod tests { use super::*; use crate::core::manifest::{ - CloneStrategy, CopyFileConfig, LinkFileConfig, ManifestRepoConfig, ManifestSettings, - MergeStrategy, RepoConfig, + CloneStrategy, CopyFileConfig, GripspaceConfig, LinkFileConfig, ManifestRepoConfig, + ManifestSettings, MergeStrategy, RepoConfig, }; use std::collections::HashMap; + use std::process::Command as GitCommand; use tempfile::TempDir; + fn git(dir: &Path, args: &[&str]) -> String { + let output = GitCommand::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {args:?} failed in {}: {}", + dir.display(), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + fn source_repo(path: &Path) -> String { + std::fs::create_dir_all(path).unwrap(); + git(path, &["init", "-q", "-b", "main"]); + git(path, &["config", "user.email", "test@example.com"]); + git(path, &["config", "user.name", "test"]); + std::fs::write(path.join("SOURCE.md"), "version one\n").unwrap(); + git(path, &["add", "SOURCE.md"]); + git(path, &["commit", "-qm", "initial"]); + git(path, &["rev-parse", "HEAD"]) + } + + fn advance_source(path: &Path) { + std::fs::write(path.join("SOURCE.md"), "version two\n").unwrap(); + git(path, &["add", "SOURCE.md"]); + git(path, &["commit", "-qm", "advance"]); + } + fn create_test_manifest( copyfiles: Option>, linkfiles: Option>, @@ -825,6 +1008,90 @@ mod tests { } } + #[test] + fn freshness_sources_cover_every_gripspace_backed_link_shape() { + let manifest = Manifest::parse_raw( + r#" +version: 2 +repos: {} +manifest: + url: "" + copyfile: + - src: gripspace:copy-space:file.txt + dest: copy.txt + - src: local.txt + dest: local-copy.txt + linkfile: + - src: gripspace:link-space:file.txt + dest: link.txt + composefile: + - dest: composed.txt + parts: + - gripspace: compose-space + src: file.txt + - gripspace: copy-space + src: another.txt + - src: local.txt +"#, + ) + .unwrap(); + + assert_eq!( + referenced_gripspaces(&manifest), + BTreeSet::from([ + "compose-space".to_string(), + "copy-space".to_string(), + "link-space".to_string(), + ]) + ); + } + + #[test] + fn detached_source_requires_recorded_pin_provenance() { + let fixture = TempDir::new().unwrap(); + let workspace = fixture.path().join("workspace"); + let spaces = manifest_paths::spaces_dir(&workspace); + std::fs::create_dir_all(&spaces).unwrap(); + let origin = fixture.path().join("source-space"); + let pinned_sha = source_repo(&origin); + + let unpinned = GripspaceConfig { + url: origin.display().to_string(), + rev: None, + }; + let clone = crate::core::gripspace::ensure_gripspace(&spaces, &unpinned).unwrap(); + git(&clone, &["checkout", "--detach", "HEAD"]); + assert!(git(&clone, &["branch", "--show-current"]).is_empty()); + advance_source(&origin); + + let manifest = Manifest::parse_raw( + r#" +version: 2 +repos: {} +manifest: + url: "" + copyfile: + - src: gripspace:source-space:SOURCE.md + dest: OUTPUT.md +"#, + ) + .unwrap(); + let error = ensure_gripspace_sources_current(&workspace, &manifest).unwrap_err(); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("unexpectedly detached"), "{diagnostic}"); + assert!(diagnostic.contains("gr sync"), "{diagnostic}"); + + // Positive control: the same detached object is accepted when the + // materializer records that exact commit as the requested revision. + let pinned = GripspaceConfig { + url: origin.display().to_string(), + rev: Some(pinned_sha), + }; + let pinned_clone = crate::core::gripspace::ensure_gripspace(&spaces, &pinned).unwrap(); + assert!(git(&pinned_clone, &["branch", "--show-current"]).is_empty()); + ensure_gripspace_sources_current(&workspace, &manifest).unwrap(); + } + #[test] fn test_show_link_status_no_links() { let temp = TempDir::new().unwrap(); diff --git a/src/core/gripspace.rs b/src/core/gripspace.rs index 118f154..1638e4a 100644 --- a/src/core/gripspace.rs +++ b/src/core/gripspace.rs @@ -25,6 +25,53 @@ use std::process::Command; /// Maximum depth for recursive gripspace includes const MAX_GRIPSPACE_DEPTH: usize = 5; +/// Local git-config key recording the revision contract used to materialize a +/// gripspace clone. An empty value means the manifest omitted `rev` and the +/// clone is expected to remain branch-backed. +const REQUESTED_REV_CONFIG_KEY: &str = "gitgrip.requestedGripspaceRev"; + +fn record_requested_revision( + gripspace_path: &Path, + rev: Option<&str>, +) -> Result<(), ManifestError> { + let repo = git2::Repository::open(gripspace_path).map_err(|error| { + ManifestError::GripspaceError(format!( + "Failed to open gripspace revision metadata: {error}" + )) + })?; + repo.config() + .and_then(|mut config| config.set_str(REQUESTED_REV_CONFIG_KEY, rev.unwrap_or(""))) + .map_err(|error| { + ManifestError::GripspaceError(format!( + "Failed to record gripspace revision metadata: {error}" + )) + }) +} + +/// Revision contract recorded when this gripspace was resolved. +/// +/// `Ok(None)` is a recorded rev-less, branch-backed source. A missing key is +/// an error rather than an implicit pin because absence cannot prove why a +/// detached checkout exists. +pub fn requested_gripspace_revision( + gripspace_path: &Path, +) -> Result, ManifestError> { + let repo = git2::Repository::open(gripspace_path).map_err(|error| { + ManifestError::GripspaceError(format!( + "Failed to open gripspace revision metadata: {error}" + )) + })?; + let value = repo + .config() + .and_then(|config| config.get_string(REQUESTED_REV_CONFIG_KEY)) + .map_err(|error| { + ManifestError::GripspaceError(format!( + "Gripspace revision provenance is unavailable: {error}. Run `gr sync` before applying links." + )) + })?; + Ok((!value.is_empty()).then_some(value)) +} + /// Extract a gripspace name from its URL. /// /// Takes the last path component without `.git` suffix. @@ -311,6 +358,7 @@ pub fn ensure_gripspace( if let Some(ref rev) = config.rev { checkout_rev(&gripspace_path, rev)?; } + record_requested_revision(&gripspace_path, config.rev.as_deref())?; return Ok(gripspace_path); } @@ -333,6 +381,8 @@ pub fn ensure_gripspace( checkout_rev(&gripspace_path, rev)?; } + record_requested_revision(&gripspace_path, config.rev.as_deref())?; + Ok(gripspace_path) } @@ -387,6 +437,8 @@ pub fn update_gripspace( } } + record_requested_revision(gripspace_path, config.rev.as_deref())?; + Ok(()) } diff --git a/tests/link_apply_freshness.rs b/tests/link_apply_freshness.rs new file mode 100644 index 0000000..24cad99 --- /dev/null +++ b/tests/link_apply_freshness.rs @@ -0,0 +1,228 @@ +//! `gr link --apply` must not certify stale gripspace-derived output. +//! +//! The command composes from clones under `.gitgrip/spaces`. A valid local +//! file proves only that composition can run. It does not prove that the clone +//! still reflects its upstream. This test advances the upstream after a known +//! current baseline and requires the manual apply path to refuse before it +//! rewrites the destination from stale bytes. + +use assert_cmd::Command; +use std::path::{Path, PathBuf}; +use std::process::Command as StdCommand; +use tempfile::TempDir; + +fn git(dir: &Path, args: &[&str]) -> String { + let output = StdCommand::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?} failed in {}: {}", + args, + dir.display(), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +fn repo(root: &Path, name: &str, files: &[(&str, &str)]) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).unwrap(); + git(&dir, &["init", "-q", "-b", "main"]); + git(&dir, &["config", "user.email", "test@example.com"]); + git(&dir, &["config", "user.name", "test"]); + for (path, body) in files { + let full = dir.join(path); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(full, body).unwrap(); + } + git(&dir, &["add", "-A"]); + git(&dir, &["commit", "-qm", "initial"]); + dir +} + +fn advance(repo: &Path, body: &str) -> String { + std::fs::write(repo.join("SECTION.md"), body).unwrap(); + git(repo, &["add", "SECTION.md"]); + git(repo, &["commit", "-qm", "advance source"]); + git(repo, &["rev-parse", "HEAD"]) +} + +#[test] +fn link_apply_refuses_a_gripspace_source_that_is_behind_upstream() { + let fixture = TempDir::new().unwrap(); + let root = fixture.path(); + + let source = repo( + root, + "source-space", + &[ + ("SECTION.md", "version-one-content\n"), + ("gripspace.yml", "version: 2\nrepos: {}\n"), + ], + ); + let dummy = repo(root, "dummy-repo", &[("README.md", "dummy\n")]); + let manifest = repo( + root, + "workspace-manifest", + &[( + "gripspace.yml", + &format!( + r#"version: 2 +gripspaces: + - url: "{}" + rev: main +manifest: + url: "{}" + revision: main + composefile: + - dest: OUT.md + parts: + - gripspace: source-space + src: SECTION.md +repos: + dummy-repo: + url: "{}" + path: ./dummy-repo + revision: main +"#, + source.display(), + root.join("workspace-manifest").display(), + dummy.display(), + ), + )], + ); + + let workspace = root.join("workspace"); + let init = Command::cargo_bin("gr") + .unwrap() + .args([ + "init", + manifest.to_str().unwrap(), + "--path", + workspace.to_str().unwrap(), + "--no-interactive", + ]) + .output() + .unwrap(); + assert!( + init.status.success(), + "init precondition failed:\nstdout={}\nstderr={}", + String::from_utf8_lossy(&init.stdout), + String::from_utf8_lossy(&init.stderr) + ); + + // Establish a known-current composition through the same sync path the + // issue uses before advancing the source. `gr init` currently treats a + // link-application failure as a warning, so its zero exit alone is not a + // sufficient fixture precondition. + let baseline_sync = Command::cargo_bin("gr") + .unwrap() + .arg("sync") + .current_dir(&workspace) + .output() + .unwrap(); + assert!( + baseline_sync.status.success(), + "baseline sync precondition failed:\nmanifest={}\nstdout={}\nstderr={}", + std::fs::read_to_string(workspace.join(".gitgrip/spaces/main/gripspace.yml")) + .unwrap_or_else(|error| format!("")), + String::from_utf8_lossy(&baseline_sync.stdout), + String::from_utf8_lossy(&baseline_sync.stderr) + ); + assert_eq!( + std::fs::read_to_string(workspace.join("OUT.md")).unwrap_or_else(|error| { + panic!( + "baseline composition did not produce OUT.md: {error}\nstdout={}\nstderr={}", + String::from_utf8_lossy(&baseline_sync.stdout), + String::from_utf8_lossy(&baseline_sync.stderr) + ) + }), + "version-one-content\n" + ); + + advance(&source, "version-two-content\n"); + let sync = Command::cargo_bin("gr") + .unwrap() + .arg("sync") + .current_dir(&workspace) + .output() + .unwrap(); + assert!( + sync.status.success(), + "sync precondition failed:\nstdout={}\nstderr={}", + String::from_utf8_lossy(&sync.stdout), + String::from_utf8_lossy(&sync.stderr) + ); + assert_eq!( + std::fs::read_to_string(workspace.join("OUT.md")).unwrap(), + "version-two-content\n" + ); + + let upstream_head = advance(&source, "version-three-content\n"); + let local_head = git( + &workspace.join(".gitgrip/spaces/source-space"), + &["rev-parse", "HEAD"], + ); + assert_ne!( + local_head, upstream_head, + "fixture must be stale before apply" + ); + + let apply = Command::cargo_bin("gr") + .unwrap() + .args(["link", "--apply"]) + .current_dir(&workspace) + .output() + .unwrap(); + assert_eq!( + apply.status.code(), + Some(2), + "a stale source must be a deliberate refusal, not success:\nstdout={}\nstderr={}", + String::from_utf8_lossy(&apply.stdout), + String::from_utf8_lossy(&apply.stderr) + ); + let diagnostic = format!( + "{}{}", + String::from_utf8_lossy(&apply.stdout), + String::from_utf8_lossy(&apply.stderr) + ); + assert!( + diagnostic.contains("source-space"), + "diagnostic must name the stale source" + ); + assert!( + diagnostic.contains("gr sync"), + "diagnostic must state the recovery path" + ); + assert_eq!( + std::fs::read_to_string(workspace.join("OUT.md")).unwrap(), + "version-two-content\n", + "refusal must happen before stale bytes rewrite the destination" + ); + + // Positive control: after the named recovery path, the same command runs + // and the destination reflects the advanced source. + let sync = Command::cargo_bin("gr") + .unwrap() + .arg("sync") + .current_dir(&workspace) + .output() + .unwrap(); + assert!(sync.status.success()); + let apply = Command::cargo_bin("gr") + .unwrap() + .args(["link", "--apply"]) + .current_dir(&workspace) + .output() + .unwrap(); + assert!(apply.status.success()); + assert_eq!( + std::fs::read_to_string(workspace.join("OUT.md")).unwrap(), + "version-three-content\n" + ); +} From e84c4cac9a8fea170e9e18482a901edfe6ddd4cd Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 09:23:37 -0500 Subject: [PATCH 05/29] feat(gr2): propagation state machine prototype on synthetic repos Prove the propagation state contract before any daemon touches a real clone: observed -> fetched -> planned -> applied -> verified -> acknowledged, with refused, partial, and unverifiable reachable from any state. Every transition names the observation that established it and every receipt names the exact source and destination revisions. All identifiers are opaque; the allowed directions are a required input with no default. The tests build a bare source remote and three destinations (clean replica, dirty authoring clone, diverged authoring clone) and prove, each as its own witness: kill-between-states then replay applies exactly once (reflog and journal both measured); the cursor advances only on acknowledged; a moved expected base refuses with the observed base recorded and nothing merged or forced; dirty and diverged authoring clones are refused and left byte-for-byte untouched; an unreadable destination after the apply verb is unverifiable, never collapsed into a neighbour, and resolves on replay; acknowledged replays as a no-op returning the original outcome while refused starts a new attempt. Also carries the born-red outbox witness: read_events() advances the consumer cursor before the caller performs its effect, so a consumer that fails after reading loses the event. Marked xfail(strict=True) so the marker fails the suite the moment acknowledgment moves after the effect. Co-Authored-By: Claude --- gr2/prototypes/README.md | 40 + gr2/prototypes/propagation_state_machine.py | 1141 +++++++++++++++++++ gr2/tests/test_propagation_state_machine.py | 660 +++++++++++ 3 files changed, 1841 insertions(+) create mode 100644 gr2/prototypes/propagation_state_machine.py create mode 100644 gr2/tests/test_propagation_state_machine.py diff --git a/gr2/prototypes/README.md b/gr2/prototypes/README.md index 691f440..0fc193b 100644 --- a/gr2/prototypes/README.md +++ b/gr2/prototypes/README.md @@ -406,3 +406,43 @@ This keeps the cache discussion grounded in evidence: - cache remains the optimization - the prototype should tell us whether the optimization is material enough to justify building it into `apply` + +## Propagation State Machine (Prototype 0) + +Before any daemon is allowed to move changes between workspaces, the state +contract for one propagation is proven on synthetic repositories: + +```bash +python3 -m pytest gr2/tests/test_propagation_state_machine.py -q +``` + +`gr2/prototypes/propagation_state_machine.py` drives one change through +`observed -> fetched -> planned -> applied -> verified -> acknowledged`, with +`refused`, `partial`, and `unverifiable` reachable from any of them. Every +transition names the observation that established it, and every receipt names +the exact source and destination revisions. Everything it carries is an opaque +identifier; it holds no notion of agent or workspace identity and no policy +content (the allowed directions are a required input with no default). + +The tests build a bare source remote and three destinations (a clean replica, a +dirty authoring clone, an authoring clone that is ahead of the source) and +prove, each as its own witness: + +- killing the sink between each pair of states and replaying applies the change + exactly once (measured from the destination's own reflog and the journal) +- the cursor advances only on `acknowledged` +- a moved expected base refuses with the observed base in the receipt; nothing + is merged, forced, or retried against the new base +- a dirty or diverged authoring clone is refused and left byte-for-byte + untouched (refs, HEAD, index, worktree, reflog) +- a destination that cannot be read back after the apply verb is recorded as + `unverifiable`, never collapsed into `refused` or `applied`, and resolves on + replay +- an acknowledged operation replays as a no-op returning the original outcome; + a refused one starts a new attempt, because a refusal describes a moment + +It is also the home of a born-red witness for the existing event outbox: +`read_events()` advances the consumer cursor before the caller performs its +effect, so a consumer that fails after reading loses the event. The test is +marked `xfail(strict=True)` and turns the marker into a failure the moment +acknowledgment moves after the effect. diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py new file mode 100644 index 0000000..06c67c4 --- /dev/null +++ b/gr2/prototypes/propagation_state_machine.py @@ -0,0 +1,1141 @@ +"""Prototype 0: a propagation state machine over git repositories. + +This prototype proves the state contract for moving one change from a source +repository to one or more destination clones, before any daemon is allowed near +a real authoring clone. It is deliberately neutral: every identifier it carries +(source, destination, layer, artifact class, policy hash) is an opaque string +the caller resolves; the machine interprets none of them and holds no notion of +who an agent or a workspace is. + +The one rule the contract enforces: + + Every state transition names the observation that establishes it, and that + observation must be capable of returning a different answer. + +States, in order, each with the observation that establishes it: + + observed a source change was detected at a named revision + fetched the object is present in the sink's mirror and its digest was + recomputed from the object itself + planned a plan exists, bound to the destination base it was computed + against, with every gate recorded individually + applied the destination itself reports the intended revision (read + back from the destination, never from the verb's exit status) + verified the declared postcondition was checked by name and holds + acknowledged the source cursor advances, only now + +and three more reachable from any of them: + + refused a named refusal condition fired, and which one is recorded + partial some declared targets verified and others did not, both listed + unverifiable the effect may or may not have happened and no observation + available to the sink can tell; never collapsed into either + neighbour, because one invites a double apply and the other + invites work on an effect that may not exist + +Idempotency and compare-and-swap: + +* every operation has an id derived from (coordinate, source revision, expected + base, digest); the journal is keyed so a replay resumes the same operation +* every destination transition is compare-and-swap on the expected base; a + moved base refuses and reports what was observed, it never merges or forces +* the apply step tolerates exactly one benign discrepancy: a destination that + already reports the intended revision is recorded as applied without running + the verb again, which is how a sink killed mid-apply replays without applying + twice + +Replay rules the journal implements: + +* an acknowledged operation replays as a no-op returning the original outcome +* a refused operation is terminal for that attempt only: a later run at the same + source revision starts a new attempt, because a refusal describes a moment + (a dirty worktree, a moved base) and the author may have changed it +* anything else resumes from the last recorded state without skipping a state + +Everything under ``state_dir`` belongs to the sink: ``journal.jsonl`` (append +only, one row per transition or note, fsynced), ``cursors/`` (one small file per +coordinate, written only on acknowledged), and ``mirror.git`` (a bare mirror the +sink fetches into, so destinations are read but never written before apply). + +Fault injection for tests: ``kill_after`` raises :class:`SinkKilled` right after +the named state is journaled (or right after the apply verb ran, before the read +back, with ``KILL_AFTER_APPLY_VERB``); ``after_apply_verb`` runs a callable +between the verb and the read back so a destination can be made unreadable. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +from collections.abc import Callable, Iterable +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from enum import StrEnum +from pathlib import Path + +KILL_AFTER_APPLY_VERB = "apply-verb" + +_GATE_IDS = ( + "policy.direction", + "destination.base-unmoved", + "destination.clean", + "destination.fast-forward", +) +_POSTCONDITION = "head-is-intended-after-and-tree-matches-digest" + +# The prototype runs git without the user's global or system configuration so +# that signing, hook paths, and identity settings on the host cannot reach the +# synthetic repositories. Callers may pass their own environment. +_ISOLATED_GIT_ENV = { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", +} + + +class SinkKilled(RuntimeError): + """The sink process died at a fault-injection point.""" + + +class DestinationUnreadable(RuntimeError): + """A read against the destination returned an error instead of an answer.""" + + +class Direction(StrEnum): + DOWN = "down" + UP = "up" + ACROSS = "across" + + +class Operation(StrEnum): + APPLY = "apply" + CONTRIBUTE = "contribute" + OBSERVE = "observe" + + +class DestinationKind(StrEnum): + REPLICA = "replica" + AUTHORING = "authoring" + + +class State(StrEnum): + OBSERVED = "observed" + FETCHED = "fetched" + PLANNED = "planned" + APPLIED = "applied" + VERIFIED = "verified" + ACKNOWLEDGED = "acknowledged" + REFUSED = "refused" + PARTIAL = "partial" + UNVERIFIABLE = "unverifiable" + + +_ORDERED = ( + State.OBSERVED, + State.FETCHED, + State.PLANNED, + State.APPLIED, + State.VERIFIED, + State.ACKNOWLEDGED, +) + + +@dataclass(frozen=True) +class Coordinate: + source: str + destination: str + layer: str + direction: Direction + operation: Operation + artifact_class: str + + def key(self) -> str: + return "|".join( + ( + self.source, + self.destination, + self.layer, + str(self.direction), + str(self.operation), + self.artifact_class, + ) + ) + + def as_dict(self) -> dict[str, str]: + return { + "source": self.source, + "destination": self.destination, + "layer": self.layer, + "direction": str(self.direction), + "operation": str(self.operation), + "artifact_class": self.artifact_class, + } + + +@dataclass(frozen=True) +class Destination: + destination_id: str + path: Path + kind: DestinationKind + + +@dataclass(frozen=True) +class Policy: + """Opaque policy inputs. The caller resolves them; the machine only applies them. + + ``allowed_directions`` has no default on purpose: a permissive default would + be the decision, whoever wrote it. + """ + + policy_hash: str + allowed_directions: frozenset[Direction] + + +@dataclass(frozen=True) +class GateResult: + gate_id: str + result: str + detail: str + result_hash: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True) +class Transition: + state: State + observation: dict[str, object] + timestamp: str + operation_id: str | None + + def as_dict(self) -> dict[str, object]: + return { + "state": str(self.state), + "observation": self.observation, + "timestamp": self.timestamp, + "operation_id": self.operation_id, + } + + +@dataclass(frozen=True) +class Observation: + source_rev: str + cursor: str | None + is_new: bool + + +@dataclass(frozen=True) +class Receipt: + pending_id: str + attempt: int + operation_id: str | None + coordinate: Coordinate + source_rev: str + expected_base: str + observed_base: str | None + after: str | None + digest: str | None + plan_hash: str | None + policy_hash: str + gate_results: tuple[GateResult, ...] + state: State + postcondition_checked: str | None + postcondition_holds: bool | None + timestamp: str + refusal_reason: str | None + detail: str + transitions: tuple[Transition, ...] + replayed: bool + + def as_dict(self) -> dict[str, object]: + return { + "pending_id": self.pending_id, + "attempt": self.attempt, + "operation_id": self.operation_id, + "coordinate": self.coordinate.as_dict(), + "source_rev": self.source_rev, + "expected_base": self.expected_base, + "observed_base": self.observed_base, + "after": self.after, + "digest": self.digest, + "plan_hash": self.plan_hash, + "policy_hash": self.policy_hash, + "gate_results": [g.as_dict() for g in self.gate_results], + "state": str(self.state), + "postcondition_checked": self.postcondition_checked, + "postcondition_holds": self.postcondition_holds, + "timestamp": self.timestamp, + "refusal_reason": self.refusal_reason, + "detail": self.detail, + "transitions": [t.as_dict() for t in self.transitions], + "replayed": self.replayed, + } + + +@dataclass(frozen=True) +class PlanOutcome: + state: State + reached: tuple[str, ...] + not_reached: tuple[tuple[str, str], ...] + receipts: tuple[Receipt, ...] + + +# --------------------------------------------------------------------------- helpers + + +def _now() -> str: + return datetime.now(UTC).isoformat() + + +def _sha256_json(obj: object) -> str: + return hashlib.sha256( + json.dumps(obj, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def operation_id_for(coordinate_key: str, source_rev: str, expected_base: str, digest: str) -> str: + return _sha256_json( + { + "coordinate": coordinate_key, + "source_rev": source_rev, + "expected_base": expected_base, + "digest": digest, + } + ) + + +def pending_id_for(coordinate_key: str, source_rev: str, attempt: int) -> str: + return _sha256_json( + {"coordinate": coordinate_key, "source_rev": source_rev, "attempt": attempt} + ) + + +def _git(repo: Path, *args: str, env: dict[str, str] | None = None) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + env={**os.environ, **(env or _ISOLATED_GIT_ENV)}, + ) + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, proc.args, proc.stdout, proc.stderr) + return proc.stdout.strip() + + +def tree_digest(repo: Path, rev: str, env: dict[str, str] | None = None) -> str: + """The content digest of a revision: its tree id, recomputed from the object store.""" + return _git(repo, "rev-parse", f"{rev}^{{tree}}", env=env) + + +# --------------------------------------------------------------------------- journal + + +class Journal: + """Append-only record of transitions and notes, plus per-coordinate cursors.""" + + def __init__(self, state_dir: Path) -> None: + self.state_dir = state_dir + self.path = state_dir / "journal.jsonl" + self.cursors_dir = state_dir / "cursors" + + # -- rows + + def _rows(self) -> list[dict[str, object]]: + if not self.path.exists(): + return [] + rows: list[dict[str, object]] = [] + for line in self.path.read_text().splitlines(): + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return rows + + def _write(self, row: dict[str, object]) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a") as handle: + handle.write(json.dumps(row, separators=(",", ":")) + "\n") + handle.flush() + os.fsync(handle.fileno()) + + def append( + self, + *, + pending_id: str, + attempt: int, + operation_id: str | None, + coordinate_key: str, + source_rev: str, + state: State, + observation: dict[str, object], + ) -> Transition: + transition = Transition( + state=state, observation=observation, timestamp=_now(), operation_id=operation_id + ) + self._write( + { + "kind": "transition", + "pending_id": pending_id, + "attempt": attempt, + "operation_id": operation_id, + "coordinate_key": coordinate_key, + "source_rev": source_rev, + **transition.as_dict(), + } + ) + return transition + + def note( + self, + *, + pending_id: str, + attempt: int, + coordinate_key: str, + source_rev: str, + note: str, + data: dict[str, object] | None = None, + ) -> None: + self._write( + { + "kind": "note", + "pending_id": pending_id, + "attempt": attempt, + "coordinate_key": coordinate_key, + "source_rev": source_rev, + "note": note, + "data": data or {}, + "timestamp": _now(), + } + ) + + def rows_for(self, coordinate_key: str, source_rev: str) -> list[dict[str, object]]: + return [ + r + for r in self._rows() + if r.get("coordinate_key") == coordinate_key and r.get("source_rev") == source_rev + ] + + def find(self, coordinate_key: str, source_rev: str) -> list[list[Transition]]: + """Every attempt at (coordinate, source revision), oldest first, each as its transitions.""" + attempts: dict[int, list[Transition]] = {} + for row in self.rows_for(coordinate_key, source_rev): + if row.get("kind") != "transition": + continue + attempts.setdefault(int(row["attempt"]), []).append( + Transition( + state=State(str(row["state"])), + observation=dict(row["observation"]), # type: ignore[arg-type] + timestamp=str(row["timestamp"]), + operation_id=row.get("operation_id"), # type: ignore[arg-type] + ) + ) + return [attempts[k] for k in sorted(attempts)] + + def notes(self, coordinate_key: str, source_rev: str, note: str) -> int: + return sum( + 1 + for r in self.rows_for(coordinate_key, source_rev) + if r.get("kind") == "note" and r.get("note") == note + ) + + # -- cursors + + def cursor_path(self, coordinate_key: str) -> Path: + return self.cursors_dir / (hashlib.sha256(coordinate_key.encode()).hexdigest() + ".json") + + def cursor(self, coordinate_key: str) -> str | None: + path = self.cursor_path(coordinate_key) + if not path.exists(): + return None + try: + data = json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + return None + value = data.get("source_rev") + return str(value) if value else None + + def advance_cursor(self, coordinate_key: str, source_rev: str, pending_id: str) -> None: + self.cursors_dir.mkdir(parents=True, exist_ok=True) + path = self.cursor_path(coordinate_key) + tmp = path.with_suffix(".tmp") + tmp.write_text( + json.dumps( + { + "coordinate_key": coordinate_key, + "source_rev": source_rev, + "pending_id": pending_id, + "advanced_at": _now(), + }, + indent=2, + ) + ) + with tmp.open("rb") as handle: + os.fsync(handle.fileno()) + tmp.rename(path) + + +# ------------------------------------------------------------------------ the machine + + +@dataclass +class _Op: + """The in-flight operation, reconstructed from the journal on resume.""" + + coordinate: Coordinate + destination: Destination + source_rev: str + attempt: int + pending_id: str + previous_cursor: str | None + expected_base: str | None = None + digest: str | None = None + operation_id: str | None = None + observed_base: str | None = None + intended_after: str | None = None + plan_hash: str | None = None + gate_results: tuple[GateResult, ...] = () + transitions: list[Transition] = field(default_factory=list) + + @property + def key(self) -> str: + return self.coordinate.key() + + +class Propagator: + def __init__( + self, + source_remote: Path, + branch: str, + state_dir: Path, + policy: Policy, + *, + kill_after: State | str | None = None, + after_apply_verb: Callable[[], None] | None = None, + git_env: dict[str, str] | None = None, + ) -> None: + self.source_remote = source_remote + self.branch = branch + self.state_dir = state_dir + self.policy = policy + self.kill_after = kill_after + self.after_apply_verb = after_apply_verb + self.git_env = git_env or _ISOLATED_GIT_ENV + self.journal = Journal(state_dir) + self.mirror = state_dir / "mirror.git" + + # -- observation of the source + + def observe_source_at(self, cursor: str | None) -> Observation: + out = subprocess.run( + ["git", "ls-remote", str(self.source_remote), f"refs/heads/{self.branch}"], + capture_output=True, + text=True, + env={**os.environ, **self.git_env}, + ) + if out.returncode != 0 or not out.stdout.strip(): + raise RuntimeError( + f"source {self.source_remote} has no branch {self.branch}: {out.stderr.strip()}" + ) + source_rev = out.stdout.split()[0] + return Observation(source_rev=source_rev, cursor=cursor, is_new=(source_rev != cursor)) + + def observe_source(self, coordinate: Coordinate) -> Observation: + return self.observe_source_at(self.journal.cursor(coordinate.key())) + + # -- destination reads (never writes before apply) + + def read_head(self, destination: Destination) -> str: + try: + return _git(destination.path, "rev-parse", "HEAD", env=self.git_env) + except (subprocess.CalledProcessError, OSError) as exc: + raise DestinationUnreadable(f"{destination.destination_id}: {exc}") from exc + + def _porcelain(self, destination: Destination) -> str: + return _git(destination.path, "status", "--porcelain", env=self.git_env) + + # -- driver + + def run(self, coordinate: Coordinate, destination: Destination) -> Receipt | None: + key = coordinate.key() + observation = self.observe_source(coordinate) + if not observation.is_new: + return None + source_rev = observation.source_rev + + attempts = self.journal.find(key, source_rev) + if attempts: + last = attempts[-1] + last_state = last[-1].state + if last_state is State.ACKNOWLEDGED: + # terminal: the original outcome, no verb, and the cursor repaired if it lagged + attempt = len(attempts) + pending_id = pending_id_for(key, source_rev, attempt) + self.journal.note( + pending_id=pending_id, + attempt=attempt, + coordinate_key=key, + source_rev=source_rev, + note="replayed-terminal", + data={"state": str(last_state)}, + ) + self.journal.advance_cursor(key, source_rev, pending_id) + return self._receipt(coordinate, source_rev, attempt, replayed=True) + if last_state is State.REFUSED: + op = self._fresh( + coordinate, + destination, + source_rev, + observation.cursor, + attempt=len(attempts) + 1, + ) + else: + op = self._resume( + coordinate, + destination, + source_rev, + observation.cursor, + attempt=len(attempts), + transitions=last, + ) + else: + op = self._fresh(coordinate, destination, source_rev, observation.cursor, attempt=1) + + self._drive(op) + return self._receipt(coordinate, source_rev, op.attempt, replayed=False) + + def run_all(self, targets: Iterable[tuple[Coordinate, Destination]]) -> PlanOutcome | None: + receipts: list[Receipt] = [] + for coordinate, destination in targets: + receipt = self.run(coordinate, destination) + if receipt is not None: + receipts.append(receipt) + if not receipts: + return None + reached = tuple(r.coordinate.destination for r in receipts if r.state is State.ACKNOWLEDGED) + not_reached = tuple( + (r.coordinate.destination, r.refusal_reason or r.detail or f"state={r.state}") + for r in receipts + if r.state is not State.ACKNOWLEDGED + ) + if not not_reached: + state = State.ACKNOWLEDGED + elif not reached: + state = State.REFUSED + else: + state = State.PARTIAL + return PlanOutcome( + state=state, reached=reached, not_reached=not_reached, receipts=tuple(receipts) + ) + + # -- attempt construction + + def _fresh( + self, + coordinate: Coordinate, + destination: Destination, + source_rev: str, + cursor: str | None, + *, + attempt: int, + ) -> _Op: + return _Op( + coordinate=coordinate, + destination=destination, + source_rev=source_rev, + attempt=attempt, + pending_id=pending_id_for(coordinate.key(), source_rev, attempt), + previous_cursor=cursor, + ) + + def _resume( + self, + coordinate: Coordinate, + destination: Destination, + source_rev: str, + cursor: str | None, + *, + attempt: int, + transitions: list[Transition], + ) -> _Op: + op = self._fresh(coordinate, destination, source_rev, cursor, attempt=attempt) + op.transitions = list(transitions) + for t in transitions: + obs = t.observation + if t.state is State.OBSERVED: + op.expected_base = str(obs["expected_base"]) + elif t.state is State.FETCHED: + op.digest = str(obs["digest"]) + op.operation_id = t.operation_id + elif t.state is State.PLANNED: + op.observed_base = str(obs["observed_base"]) + op.intended_after = str(obs["intended_after"]) + op.plan_hash = str(obs["plan_hash"]) + op.gate_results = tuple( + GateResult(**g) + for g in obs["gate_results"] # type: ignore[arg-type] + ) + return op + + # -- the state machine + + def _drive(self, op: _Op) -> None: + last = op.transitions[-1].state if op.transitions else None + if last is None: + self._observe(op) + last = State.OBSERVED + if last is State.OBSERVED: + self._fetch(op) + last = State.FETCHED + if last is State.FETCHED: + if not self._plan(op): + return # refused at plan; journaled + last = State.PLANNED + if last in (State.PLANNED, State.UNVERIFIABLE): + if not self._apply(op): + return # refused or unverifiable; journaled + last = State.APPLIED + if last is State.APPLIED: + if not self._verify(op): + return # postcondition did not hold; journaled as a note, state stays applied + last = State.VERIFIED + if last is State.VERIFIED: + self._acknowledge(op) + + def _record(self, op: _Op, state: State, observation: dict[str, object]) -> None: + transition = self.journal.append( + pending_id=op.pending_id, + attempt=op.attempt, + operation_id=op.operation_id, + coordinate_key=op.key, + source_rev=op.source_rev, + state=state, + observation=observation, + ) + op.transitions.append(transition) + if self.kill_after == state: + raise SinkKilled(f"killed after {state}") + + def _note(self, op: _Op, note: str, data: dict[str, object] | None = None) -> None: + self.journal.note( + pending_id=op.pending_id, + attempt=op.attempt, + coordinate_key=op.key, + source_rev=op.source_rev, + note=note, + data=data, + ) + + def _observe(self, op: _Op) -> None: + op.expected_base = self.read_head(op.destination) + self._record( + op, + State.OBSERVED, + { + "source_rev": op.source_rev, + "source_branch": self.branch, + "previous_cursor": op.previous_cursor, + "expected_base": op.expected_base, + "established_by": ( + "ls-remote on the source reported a revision the cursor did not name" + ), + }, + ) + + def _fetch(self, op: _Op) -> None: + if not self.mirror.exists(): + self.mirror.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["git", "init", "-q", "--bare", str(self.mirror)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **self.git_env}, + ) + _git( + self.mirror, + "fetch", + "-q", + str(self.source_remote), + f"+refs/heads/{self.branch}:refs/remotes/source/{self.branch}", + env=self.git_env, + ) + # presence is asserted against the object itself, not the fetch's exit status + _git(self.mirror, "cat-file", "-e", f"{op.source_rev}^{{commit}}", env=self.git_env) + op.digest = tree_digest(self.mirror, op.source_rev, env=self.git_env) + assert op.expected_base is not None + op.operation_id = operation_id_for(op.key, op.source_rev, op.expected_base, op.digest) + self._record( + op, + State.FETCHED, + { + "digest": op.digest, + "mirror": str(self.mirror), + "established_by": ( + "cat-file -e on the mirror and the tree id recomputed from the object" + ), + }, + ) + + def _gate(self, gate_id: str, passed: bool, detail: str) -> GateResult: + result = "pass" if passed else "fail" + return GateResult( + gate_id=gate_id, + result=result, + detail=detail, + result_hash=_sha256_json({"gate": gate_id, "result": result, "detail": detail}), + ) + + def _plan(self, op: _Op) -> bool: + assert op.expected_base is not None and op.digest is not None + gates: list[GateResult] = [] + + allowed = op.coordinate.direction in self.policy.allowed_directions + gates.append( + self._gate( + "policy.direction", + allowed, + f"direction={op.coordinate.direction} " + f"allowed={sorted(str(d) for d in self.policy.allowed_directions)}", + ) + ) + + op.observed_base = self.read_head(op.destination) + gates.append( + self._gate( + "destination.base-unmoved", + op.observed_base == op.expected_base, + f"expected_base={op.expected_base} observed_base={op.observed_base}", + ) + ) + + porcelain = self._porcelain(op.destination) + gates.append( + self._gate( + "destination.clean", + porcelain == "", + "worktree and index clean" + if porcelain == "" + else f"dirty: {len(porcelain.splitlines())} path(s)", + ) + ) + + # the destination's HEAD is fetched INTO the mirror (a read of the destination), + # so ahead/behind is computed in the sink's own store + observed_tag = hashlib.sha256(op.destination.destination_id.encode()).hexdigest()[:16] + observed_ref = f"refs/observed/{observed_tag}" + _git( + self.mirror, + "fetch", + "-q", + str(op.destination.path), + f"+HEAD:{observed_ref}", + env=self.git_env, + ) + counts = _git( + self.mirror, + "rev-list", + "--left-right", + "--count", + f"{op.source_rev}...{observed_ref}", + env=self.git_env, + ) + behind_s, ahead_s = counts.split() + behind, ahead = int(behind_s), int(ahead_s) + gates.append( + self._gate( + "destination.fast-forward", + ahead == 0, + f"ahead={ahead} behind={behind} (destination relative to source revision)", + ) + ) + + op.intended_after = op.source_rev + op.gate_results = tuple(gates) + op.plan_hash = _sha256_json( + { + "operation_id": op.operation_id, + "intended_after": op.intended_after, + "policy_hash": self.policy.policy_hash, + "gates": [g.gate_id for g in gates], + "destination_kind": str(op.destination.kind), + } + ) + self._record( + op, + State.PLANNED, + { + "observed_base": op.observed_base, + "intended_after": op.intended_after, + "plan_hash": op.plan_hash, + "policy_hash": self.policy.policy_hash, + "gate_results": [g.as_dict() for g in gates], + "established_by": ( + "every gate evaluated and recorded individually against the destination " + "as read now" + ), + }, + ) + failed = [g for g in gates if g.result == "fail"] + if failed: + first = failed[0] + self._record( + op, + State.REFUSED, + { + "refusal_reason": f"{first.gate_id}: {first.detail}", + "failed_gates": [g.gate_id for g in failed], + "observed_base": op.observed_base, + }, + ) + return False + return True + + def _apply(self, op: _Op) -> bool: + assert op.expected_base is not None and op.intended_after is not None + try: + head = self.read_head(op.destination) + except DestinationUnreadable as exc: + # nothing has been done to the destination yet, so this is a named refusal + self._record( + op, + State.REFUSED, + {"refusal_reason": f"destination.unreadable: {exc}", "observed_base": None}, + ) + return False + + if head == op.intended_after: + # the verb landed in an earlier life of this sink; do not run it again + self._record( + op, + State.APPLIED, + { + "after": head, + "observed_base": head, + "verb_ran_now": False, + "established_by": ( + "read-back found the intended revision already at the destination" + ), + }, + ) + return True + if head != op.expected_base: + self._record( + op, + State.REFUSED, + { + "refusal_reason": ( + f"expected_base moved: expected {op.expected_base}, observed {head}" + ), + "observed_base": head, + }, + ) + return False + + self._note( + op, "apply-verb-started", {"expected_base": head, "intended_after": op.intended_after} + ) + _git( + op.destination.path, + "fetch", + "-q", + str(self.mirror), + f"refs/remotes/source/{self.branch}", + env=self.git_env, + ) + _git(op.destination.path, "merge", "-q", "--ff-only", op.intended_after, env=self.git_env) + if self.after_apply_verb is not None: + self.after_apply_verb() + if self.kill_after == KILL_AFTER_APPLY_VERB: + raise SinkKilled("killed after the apply verb, before the read back") + + try: + after = self.read_head(op.destination) + except DestinationUnreadable as exc: + self._record( + op, + State.UNVERIFIABLE, + { + "detail": ( + f"the apply verb returned but the destination could not be read back: {exc}" + ), + "observed_base": None, + }, + ) + return False + + if after != op.intended_after: + self._record( + op, + State.REFUSED, + { + "refusal_reason": ( + f"destination.apply-mismatch: verb returned but destination reports {after}" + ), + "observed_base": after, + }, + ) + return False + + self._record( + op, + State.APPLIED, + { + "after": after, + "observed_base": head, + "verb_ran_now": True, + "established_by": ( + "rev-parse HEAD on the destination after the verb reports the intended revision" + ), + }, + ) + return True + + def _verify(self, op: _Op) -> bool: + assert op.intended_after is not None and op.digest is not None + head = self.read_head(op.destination) + tree = tree_digest(op.destination.path, head, env=self.git_env) + porcelain = self._porcelain(op.destination) + holds = head == op.intended_after and tree == op.digest and porcelain == "" + detail = f"head={head} tree={tree} clean={porcelain == ''}" + if not holds: + self._note( + op, "postcondition-failed", {"postcondition": _POSTCONDITION, "detail": detail} + ) + return False + self._record( + op, + State.VERIFIED, + { + "postcondition_checked": _POSTCONDITION, + "holds": True, + "detail": detail, + "established_by": "the named postcondition re-read from the destination", + }, + ) + return True + + def _acknowledge(self, op: _Op) -> None: + # the journal row is the acknowledgment; the cursor is derived from it and + # repaired from it on replay, so the row is written first + self._record( + op, + State.ACKNOWLEDGED, + {"cursor": op.source_rev, "established_by": "verified was recorded for this operation"}, + ) + self.journal.advance_cursor(op.key, op.source_rev, op.pending_id) + + # -- receipts + + def _receipt( + self, coordinate: Coordinate, source_rev: str, attempt: int, *, replayed: bool + ) -> Receipt: + key = coordinate.key() + # rows are read in journal order: a note and a later transition about the same + # fact resolve to whichever came last, never to whichever kind is handled last + rows = [r for r in self.journal.rows_for(key, source_rev) if int(r["attempt"]) == attempt] + transitions: list[Transition] = [] + + expected_base = "" + observed_base: str | None = None + after: str | None = None + digest: str | None = None + operation_id: str | None = None + plan_hash: str | None = None + gate_results: tuple[GateResult, ...] = () + postcondition_checked: str | None = None + postcondition_holds: bool | None = None + refusal_reason: str | None = None + detail = "" + + for row in rows: + if row.get("kind") == "note": + if row.get("note") == "postcondition-failed": + data = row.get("data") or {} + postcondition_checked = str(data.get("postcondition", _POSTCONDITION)) # type: ignore[union-attr] + postcondition_holds = False + detail = str(data.get("detail", "")) # type: ignore[union-attr] + continue + t = Transition( + state=State(str(row["state"])), + observation=dict(row["observation"]), # type: ignore[arg-type] + timestamp=str(row["timestamp"]), + operation_id=row.get("operation_id"), # type: ignore[arg-type] + ) + transitions.append(t) + obs = t.observation + if t.state is State.OBSERVED: + expected_base = str(obs["expected_base"]) + elif t.state is State.FETCHED: + digest = str(obs["digest"]) + operation_id = t.operation_id + elif t.state is State.PLANNED: + observed_base = str(obs["observed_base"]) + plan_hash = str(obs["plan_hash"]) + gate_results = tuple(GateResult(**g) for g in obs["gate_results"]) # type: ignore[arg-type] + elif t.state is State.APPLIED: + after = str(obs["after"]) + observed_base = str(obs["observed_base"]) + elif t.state is State.VERIFIED: + postcondition_checked = str(obs["postcondition_checked"]) + postcondition_holds = True + detail = str(obs.get("detail", "")) + elif t.state is State.REFUSED: + refusal_reason = str(obs["refusal_reason"]) + if obs.get("observed_base") is not None: + observed_base = str(obs["observed_base"]) + elif t.state is State.UNVERIFIABLE: + detail = str(obs["detail"]) + + if not transitions: + raise LookupError( + f"no transitions journaled for attempt {attempt} of {key} @ {source_rev}" + ) + state = transitions[-1].state + return Receipt( + pending_id=pending_id_for(key, source_rev, attempt), + attempt=attempt, + operation_id=operation_id, + coordinate=coordinate, + source_rev=source_rev, + expected_base=expected_base, + observed_base=observed_base, + after=after, + digest=digest, + plan_hash=plan_hash, + policy_hash=self.policy.policy_hash, + gate_results=gate_results, + state=state, + postcondition_checked=postcondition_checked, + postcondition_holds=postcondition_holds, + timestamp=transitions[-1].timestamp, + refusal_reason=refusal_reason, + detail=detail, + transitions=tuple(transitions), + replayed=replayed, + ) + + +__all__ = [ + "KILL_AFTER_APPLY_VERB", + "Coordinate", + "Destination", + "DestinationKind", + "DestinationUnreadable", + "Direction", + "GateResult", + "Journal", + "Observation", + "Operation", + "PlanOutcome", + "Policy", + "Propagator", + "Receipt", + "SinkKilled", + "State", + "Transition", + "operation_id_for", + "pending_id_for", + "tree_digest", +] diff --git a/gr2/tests/test_propagation_state_machine.py b/gr2/tests/test_propagation_state_machine.py new file mode 100644 index 0000000..f4b0100 --- /dev/null +++ b/gr2/tests/test_propagation_state_machine.py @@ -0,0 +1,660 @@ +"""Prototype 0: the propagation state machine, proven on synthetic git repositories. + +Everything here runs against throwaway repositories under ``tmp_path``. No real +workspace, remote, or authoring clone is touched. The synthetic topology is: + +* one bare "source" remote with a config-like file on ``main`` +* three destinations: a clean managed replica, a dirty authoring clone, and an + authoring clone that is ahead of (and behind) the source + +What the tests prove, each as its own witness: + +* one change walks observed -> fetched -> planned -> applied -> verified -> + acknowledged, and every receipt names exact source and destination revisions +* killing the sink between each pair of states and replaying applies the change + exactly once, with the cursor advancing only on acknowledged +* a moved expected base refuses (compare-and-swap) and the receipt carries the + observed base; nothing is merged or forced +* a dirty authoring clone and a diverged authoring clone are refused and left + byte-for-byte untouched +* a destination that cannot be read back after the apply verb is recorded as + ``unverifiable`` (never collapsed into refused or applied) and resolves on replay +* some destinations verifying while others refuse is ``partial``, both sides + enumerated +* the born-red witness for the outbox: a consumer that fails after reading an + event loses that event for good, because the cursor advances before the effect +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest +from gr2.prototypes.propagation_state_machine import ( + KILL_AFTER_APPLY_VERB, + Coordinate, + Destination, + DestinationKind, + Direction, + Journal, + Operation, + Policy, + Propagator, + SinkKilled, + State, + tree_digest, +) +from gr2.python_cli.events import EventType, emit, read_events + +_SHA = re.compile(r"^[0-9a-f]{40}$") + +# Commits in the synthetic repositories must not depend on, or touch, the +# machine's global git configuration (signing keys, hooks paths, identities). +_GIT_ENV = { + "GIT_AUTHOR_NAME": "prototype", + "GIT_AUTHOR_EMAIL": "prototype@example.invalid", + "GIT_COMMITTER_NAME": "prototype", + "GIT_COMMITTER_EMAIL": "prototype@example.invalid", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", +} + + +def git(repo: Path, *args: str) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + return proc.stdout.strip() + + +def head_moves(repo: Path) -> int: + """How many times HEAD has moved since the clone (reflog entries minus the clone).""" + lines = git(repo, "reflog", "show", "--format=%gs", "HEAD").splitlines() + return len(lines) - 1 + + +def snapshot(repo: Path) -> dict[str, str]: + """Everything an 'untouched' claim is about: refs, HEAD, index+worktree state, bytes.""" + return { + "refs": git(repo, "for-each-ref"), + "head": git(repo, "rev-parse", "HEAD"), + "porcelain": git(repo, "status", "--porcelain"), + "canon": (repo / "canon.md").read_text(), + "reflog": git(repo, "reflog", "show", "--format=%gs", "HEAD"), + } + + +@dataclass +class Synthetic: + remote: Path + author: Path + base: str + state_dir: Path + root: Path + + def push_change(self, text: str) -> str: + (self.author / "canon.md").write_text(text) + git(self.author, "add", "canon.md") + git(self.author, "commit", "-q", "-m", f"canon: {text.strip()[:40]}") + git(self.author, "push", "-q", "origin", "main") + return git(self.author, "rev-parse", "HEAD") + + def clone(self, name: str) -> Path: + path = self.root / name + subprocess.run( + ["git", "clone", "-q", str(self.remote), str(path)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + return path + + +@pytest.fixture +def synthetic(tmp_path: Path) -> Synthetic: + remote = tmp_path / "source.git" + subprocess.run( + ["git", "init", "-q", "--bare", "--initial-branch=main", str(remote)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + author = tmp_path / "author" + subprocess.run( + ["git", "clone", "-q", str(remote), str(author)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + git(author, "switch", "-q", "-c", "main") + (author / "canon.md").write_text("canon v1\n") + git(author, "add", "canon.md") + git(author, "commit", "-q", "-m", "canon v1") + git(author, "push", "-q", "-u", "origin", "main") + base = git(author, "rev-parse", "HEAD") + state_dir = tmp_path / "sink-state" + return Synthetic(remote=remote, author=author, base=base, state_dir=state_dir, root=tmp_path) + + +POLICY = Policy(policy_hash="policy-prototype-0", allowed_directions=frozenset({Direction.DOWN})) + + +def coordinate(destination_id: str, direction: Direction = Direction.DOWN) -> Coordinate: + return Coordinate( + source="source-0", + destination=destination_id, + layer="layer-0", + direction=direction, + operation=Operation.APPLY, + artifact_class="class-0", + ) + + +def replica(syn: Synthetic, name: str = "replica") -> Destination: + return Destination(destination_id=name, path=syn.clone(name), kind=DestinationKind.REPLICA) + + +def authoring(syn: Synthetic, name: str = "authoring") -> Destination: + return Destination(destination_id=name, path=syn.clone(name), kind=DestinationKind.AUTHORING) + + +def propagator(syn: Synthetic, **kwargs) -> Propagator: + return Propagator( + source_remote=syn.remote, branch="main", state_dir=syn.state_dir, policy=POLICY, **kwargs + ) + + +# --------------------------------------------------------------------------- happy path + + +def test_one_change_reaches_acknowledged_and_names_exact_revisions(synthetic: Synthetic) -> None: + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None + assert receipt.state is State.ACKNOWLEDGED + assert [t.state for t in receipt.transitions] == [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + State.APPLIED, + State.VERIFIED, + State.ACKNOWLEDGED, + ] + # exact revisions, never prefixes + assert receipt.source_rev == new and _SHA.match(receipt.source_rev) + assert receipt.expected_base == synthetic.base + assert receipt.observed_base == synthetic.base + assert receipt.after == new + assert receipt.operation_id is not None and len(receipt.operation_id) == 64 + assert receipt.digest == git(dest.path, "rev-parse", f"{new}^{{tree}}") + # the destination really moved, read back from the destination itself + assert git(dest.path, "rev-parse", "HEAD") == new + assert (dest.path / "canon.md").read_text() == "canon v2\n" + assert head_moves(dest.path) == 1 + # gate results are individual, each with a hash; the postcondition is named + assert {g.gate_id for g in receipt.gate_results} == { + "policy.direction", + "destination.base-unmoved", + "destination.clean", + "destination.fast-forward", + } + assert all(g.result == "pass" and len(g.result_hash) == 64 for g in receipt.gate_results) + assert receipt.postcondition_checked == "head-is-intended-after-and-tree-matches-digest" + assert receipt.postcondition_holds is True + assert receipt.refusal_reason is None + # the cursor advanced, and only because acknowledged was reached + assert Journal(synthetic.state_dir).cursor(coord.key()) == new + # the receipt is a plain, JSON-serialisable record + json.dumps(receipt.as_dict()) + + +def test_no_new_source_revision_is_not_an_operation(synthetic: Synthetic) -> None: + dest = replica(synthetic) + synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + sink = propagator(synthetic) + assert sink.run(coord, dest) is not None + journal_lines = Journal(synthetic.state_dir).path.read_text() + before = snapshot(dest.path) + + assert sink.run(coord, dest) is None + + assert snapshot(dest.path) == before + assert Journal(synthetic.state_dir).path.read_text() == journal_lines + + +def test_observe_control_rereads_at_a_known_revision_and_reports_nothing( + synthetic: Synthetic, +) -> None: + new = synthetic.push_change("canon v2\n") + sink = propagator(synthetic) + fresh = sink.observe_source_at(cursor=None) + assert fresh.source_rev == new and fresh.is_new is True + # the control: the same read, at a cursor that already names the revision + assert sink.observe_source_at(cursor=new).is_new is False + # and at a cursor known to predate it, the change IS reported + assert sink.observe_source_at(cursor=synthetic.base).is_new is True + + +def test_fetched_digest_control_known_different_object_mismatches(synthetic: Synthetic) -> None: + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + receipt = propagator(synthetic).run(coordinate(dest.destination_id), dest) + assert receipt is not None and receipt.digest is not None + mirror = synthetic.state_dir / "mirror.git" + assert tree_digest(mirror, new) == receipt.digest + # the discriminating control: a known-different object recomputes to a different digest + assert tree_digest(mirror, synthetic.base) != receipt.digest + + +# ------------------------------------------------------------------ kill + replay + + +@pytest.mark.parametrize( + "kill_after", + [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + KILL_AFTER_APPLY_VERB, + State.APPLIED, + State.VERIFIED, + ], +) +def test_kill_between_states_then_replay_applies_exactly_once( + synthetic: Synthetic, kill_after +) -> None: + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + + with pytest.raises(SinkKilled): + propagator(synthetic, kill_after=kill_after).run(coord, dest) + + journal = Journal(synthetic.state_dir) + attempts = journal.find(coord.key(), new) + assert len(attempts) == 1 + recorded = [t.state for t in attempts[0]] + expected_last = State.PLANNED if kill_after == KILL_AFTER_APPLY_VERB else kill_after + assert recorded[-1] is expected_last + # the cursor never advances before acknowledged, whatever state the sink died in + assert journal.cursor(coord.key()) is None + verb_ran_before_kill = kill_after in (KILL_AFTER_APPLY_VERB, State.APPLIED, State.VERIFIED) + assert git(dest.path, "rev-parse", "HEAD") == (new if verb_ran_before_kill else synthetic.base) + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None and receipt.state is State.ACKNOWLEDGED + assert receipt.replayed is False # it was resumed, not a terminal no-op + assert receipt.after == new and git(dest.path, "rev-parse", "HEAD") == new + assert journal.cursor(coord.key()) == new + # exactly one apply, measured two ways: the destination's own reflog and the journal + assert head_moves(dest.path) == 1 + assert journal.notes(coord.key(), new, "apply-verb-started") == 1 + # one attempt, states strictly in order, no state skipped + (transitions,) = journal.find(coord.key(), new) + states = [t.state for t in transitions] + assert states == [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + State.APPLIED, + State.VERIFIED, + State.ACKNOWLEDGED, + ] + + +def test_replay_of_an_acknowledged_operation_is_a_no_op_returning_the_original_outcome( + synthetic: Synthetic, +) -> None: + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + first = propagator(synthetic).run(coord, dest) + assert first is not None and first.state is State.ACKNOWLEDGED + # lose the cursor store (the kind of thing that happens); the journal still knows + cursor_file = Journal(synthetic.state_dir).cursor_path(coord.key()) + cursor_file.unlink() + before = snapshot(dest.path) + + again = propagator(synthetic).run(coord, dest) + + assert again is not None + assert again.replayed is True + assert again.state is State.ACKNOWLEDGED + assert again.operation_id == first.operation_id + assert again.after == first.after == new + assert snapshot(dest.path) == before # no verb ran against the destination + assert head_moves(dest.path) == 1 + + +def test_kill_after_acknowledged_row_before_cursor_replays_as_terminal_and_repairs_cursor( + synthetic: Synthetic, +) -> None: + # the acknowledged row is written before the cursor advances, so the journal is + # the source of truth and the cursor is derived from it; a death in between must + # not re-run anything and must leave the cursor repaired afterwards + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + with pytest.raises(SinkKilled): + propagator(synthetic, kill_after=State.ACKNOWLEDGED).run(coord, dest) + journal = Journal(synthetic.state_dir) + (transitions,) = journal.find(coord.key(), new) + assert transitions[-1].state is State.ACKNOWLEDGED + assert journal.cursor(coord.key()) is None + before = snapshot(dest.path) + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None and receipt.replayed is True + assert receipt.state is State.ACKNOWLEDGED + assert journal.cursor(coord.key()) == new + assert snapshot(dest.path) == before + assert head_moves(dest.path) == 1 + # and now the source has nothing new for this coordinate + assert propagator(synthetic).run(coord, dest) is None + + +# ---------------------------------------------------------------- refusals + + +def test_moved_expected_base_refuses_and_reports_observed_base(synthetic: Synthetic) -> None: + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + with pytest.raises(SinkKilled): + propagator(synthetic, kill_after=State.PLANNED).run(coord, dest) + # between plan and apply, the destination moves + (dest.path / "canon.md").write_text("local edit committed\n") + git(dest.path, "commit", "-q", "-am", "local") + moved = git(dest.path, "rev-parse", "HEAD") + assert moved != synthetic.base + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None + assert receipt.state is State.REFUSED + assert receipt.expected_base == synthetic.base + assert receipt.observed_base == moved + assert receipt.refusal_reason is not None and "expected_base" in receipt.refusal_reason + assert receipt.after is None + # not merged, not forced, not retried against the new base + assert git(dest.path, "rev-parse", "HEAD") == moved + assert (dest.path / "canon.md").read_text() == "local edit committed\n" + assert Journal(synthetic.state_dir).cursor(coord.key()) is None + assert Journal(synthetic.state_dir).notes(coord.key(), new, "apply-verb-started") == 0 + + +def test_dirty_authoring_clone_is_refused_and_untouched(synthetic: Synthetic) -> None: + dest = authoring(synthetic) + (dest.path / "canon.md").write_text("uncommitted authoring in progress\n") + (dest.path / "scratch.txt").write_text("untracked\n") + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + before = snapshot(dest.path) + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None and receipt.state is State.REFUSED + assert receipt.source_rev == new and receipt.expected_base == synthetic.base + failed = {g.gate_id: g for g in receipt.gate_results if g.result == "fail"} + assert "destination.clean" in failed + assert receipt.refusal_reason is not None and receipt.refusal_reason.startswith( + "destination.clean" + ) + # gates are recorded individually, including the ones that passed + assert {g.gate_id for g in receipt.gate_results} >= {"policy.direction", "destination.clean"} + # untouched means untouched: refs, HEAD, index/worktree, bytes, reflog + assert snapshot(dest.path) == before + assert (dest.path / "scratch.txt").read_text() == "untracked\n" + + +def test_diverged_authoring_clone_is_refused_with_counts_and_untouched( + synthetic: Synthetic, +) -> None: + dest = authoring(synthetic) + (dest.path / "canon.md").write_text("local canon branch\n") + git(dest.path, "commit", "-q", "-am", "local authoring commit") + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + before = snapshot(dest.path) + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None and receipt.state is State.REFUSED + failed = {g.gate_id: g for g in receipt.gate_results if g.result == "fail"} + assert "destination.fast-forward" in failed + assert "ahead=1" in failed["destination.fast-forward"].detail + assert "behind=1" in failed["destination.fast-forward"].detail + assert receipt.source_rev == new + assert receipt.observed_base == before["head"] + assert snapshot(dest.path) == before + + +def test_direction_outside_policy_is_refused_at_plan_and_destination_untouched( + synthetic: Synthetic, +) -> None: + dest = replica(synthetic) + synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id, direction=Direction.UP) + before = snapshot(dest.path) + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None and receipt.state is State.REFUSED + assert [t.state for t in receipt.transitions][-2:] == [State.PLANNED, State.REFUSED] + assert receipt.refusal_reason is not None and receipt.refusal_reason.startswith( + "policy.direction" + ) + assert snapshot(dest.path) == before + + +def test_refused_then_cleaned_is_a_new_attempt_not_a_replayed_refusal(synthetic: Synthetic) -> None: + dest = authoring(synthetic) + (dest.path / "canon.md").write_text("uncommitted\n") + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + first = propagator(synthetic).run(coord, dest) + assert first is not None and first.state is State.REFUSED + # the author cleans up; the same source revision at the same base is now applicable + git(dest.path, "checkout", "--", "canon.md") + + second = propagator(synthetic).run(coord, dest) + + assert second is not None and second.state is State.ACKNOWLEDGED + assert second.replayed is False + assert second.after == new + attempts = Journal(synthetic.state_dir).find(coord.key(), new) + assert len(attempts) == 2 + assert [t.state for t in attempts[0]][-1] is State.REFUSED + assert [t.state for t in attempts[1]][-1] is State.ACKNOWLEDGED + + +# ---------------------------------------------------------- verification + unverifiable + + +def test_verified_postcondition_mutation_fails_the_check_and_blocks_acknowledgement( + synthetic: Synthetic, +) -> None: + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + with pytest.raises(SinkKilled): + propagator(synthetic, kill_after=State.APPLIED).run(coord, dest) + assert git(dest.path, "rev-parse", "HEAD") == new + # mutate the postcondition: the worktree no longer matches the applied tree + (dest.path / "canon.md").write_text("mutated after apply\n") + + receipt = propagator(synthetic).run(coord, dest) + + assert receipt is not None + assert receipt.state is State.APPLIED # the highest state actually established + assert receipt.postcondition_checked == "head-is-intended-after-and-tree-matches-digest" + assert receipt.postcondition_holds is False + assert Journal(synthetic.state_dir).cursor(coord.key()) is None + assert [t.state for t in receipt.transitions][-1] is State.APPLIED + # the control: undo the mutation and the same check verifies + (dest.path / "canon.md").write_text("canon v2\n") + control = propagator(synthetic).run(coord, dest) + assert control is not None and control.state is State.ACKNOWLEDGED + assert control.postcondition_holds is True + + +def test_unreadable_destination_after_apply_is_unverifiable_and_resolves_on_replay( + synthetic: Synthetic, +) -> None: + dest = replica(synthetic) + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + hidden = dest.path.with_name("replica-hidden") + + def hide_destination() -> None: + dest.path.rename(hidden) + + receipt = propagator(synthetic, after_apply_verb=hide_destination).run(coord, dest) + + assert receipt is not None + assert receipt.state is State.UNVERIFIABLE + assert receipt.refusal_reason is None # not refused + assert receipt.after is None # not applied either; nobody read it back + assert "read back" in receipt.detail + assert Journal(synthetic.state_dir).cursor(coord.key()) is None + assert [t.state for t in receipt.transitions][-2:] == [State.PLANNED, State.UNVERIFIABLE] + + hidden.rename(dest.path) + resolved = propagator(synthetic).run(coord, dest) + + assert resolved is not None and resolved.state is State.ACKNOWLEDGED + assert resolved.after == new + assert head_moves(dest.path) == 1 + assert Journal(synthetic.state_dir).notes(coord.key(), new, "apply-verb-started") == 1 + assert [t.state for t in resolved.transitions] == [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + State.UNVERIFIABLE, + State.APPLIED, + State.VERIFIED, + State.ACKNOWLEDGED, + ] + + +# ------------------------------------------------------------------- partial + + +def test_mixed_destinations_are_partial_with_both_sides_enumerated(synthetic: Synthetic) -> None: + clean = replica(synthetic, "replica") + dirty = authoring(synthetic, "dirty-authoring") + (dirty.path / "canon.md").write_text("uncommitted\n") + ahead = authoring(synthetic, "ahead-authoring") + (ahead.path / "canon.md").write_text("local\n") + git(ahead.path, "commit", "-q", "-am", "local") + new = synthetic.push_change("canon v2\n") + targets = [(coordinate(d.destination_id), d) for d in (clean, dirty, ahead)] + + outcome = propagator(synthetic).run_all(targets) + + assert outcome.state is State.PARTIAL + assert outcome.reached == ("replica",) + assert {d for d, _reason in outcome.not_reached} == {"dirty-authoring", "ahead-authoring"} + reasons = dict(outcome.not_reached) + assert reasons["dirty-authoring"].startswith("destination.clean") + assert reasons["ahead-authoring"].startswith("destination.fast-forward") + # every receipt, reached or not, names exact revisions + for receipt in outcome.receipts: + assert _SHA.match(receipt.source_rev) and receipt.source_rev == new + assert _SHA.match(receipt.expected_base) + assert receipt.observed_base is not None and _SHA.match(receipt.observed_base) + assert git(clean.path, "rev-parse", "HEAD") == new + assert git(dirty.path, "rev-parse", "HEAD") == synthetic.base + assert (dirty.path / "canon.md").read_text() == "uncommitted\n" + + +def test_all_destinations_verifying_is_acknowledged_not_partial(synthetic: Synthetic) -> None: + a = replica(synthetic, "replica-a") + b = replica(synthetic, "replica-b") + synthetic.push_change("canon v2\n") + outcome = propagator(synthetic).run_all( + [(coordinate(a.destination_id), a), (coordinate(b.destination_id), b)] + ) + assert outcome.state is State.ACKNOWLEDGED + assert set(outcome.reached) == {"replica-a", "replica-b"} + assert outcome.not_reached == () + + +# ------------------------------------------------- operation identity + + +def test_operation_id_binds_coordinate_source_base_and_digest(synthetic: Synthetic) -> None: + a = replica(synthetic, "replica-a") + b = replica(synthetic, "replica-b") + new = synthetic.push_change("canon v2\n") + ra = propagator(synthetic).run(coordinate(a.destination_id), a) + rb = propagator(synthetic).run(coordinate(b.destination_id), b) + assert ra is not None and rb is not None + # same source revision, same base, same digest: the coordinate differs, so the id differs + assert ra.operation_id != rb.operation_id + expected = hashlib.sha256( + json.dumps( + { + "coordinate": coordinate(a.destination_id).key(), + "source_rev": new, + "expected_base": synthetic.base, + "digest": ra.digest, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert ra.operation_id == expected + + +# ---------------------------------------------------- born-red outbox witness + + +@pytest.mark.xfail( + strict=True, + reason=( + "outbox consumers lose events: read_events() advances the cursor before the " + "caller performs its effect, so a consumer that fails after reading never sees " + "the event again. This witness turns green only when acknowledgment moves " + "after the effect; strict=True forces the marker off at that moment." + ), +) +def test_outbox_event_survives_a_consumer_that_fails_after_reading(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + emit( + EventType.SYNC_COMPLETED, + workspace, + actor="sink", + owner_unit="unit-0", + payload={"detail": "one"}, + ) + + with pytest.raises(RuntimeError): + offered = read_events(workspace, "propagation-consumer") + assert [e["type"] for e in offered] == ["sync.completed"] + raise RuntimeError("effect failed after the read") + + # the contract a propagation consumer needs: an event whose effect did not happen + # is offered again + assert [e["type"] for e in read_events(workspace, "propagation-consumer")] == ["sync.completed"] From 854eeead05061a2b8fc8ec2b7d4d413e48038eaa Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 09:45:28 -0500 Subject: [PATCH 06/29] fix(gr2): make the coordinate key injective over opaque fields The first cut of the propagation prototype keyed journal rows and cursor files by joining the six coordinate fields with "|". The fields are opaque, so no delimiter can be assumed absent from them, and review proved the collision: (source="source|dest", destination="target") and (source="source", destination="dest|target") produced one key, so two distinct coordinates would have shared cursor and replay state. That contradicts the prototype's own opaque-identifier contract. The key is now the canonical JSON of the coordinate (sorted keys, compact separators), the same encoding operation_id_for already uses, and Coordinate.from_key decodes it. Injectivity is therefore STRUCTURAL, and the witness asserts it that way: every key round-trips to the coordinate that produced it, over the proven pair, over shifts across every field boundary including the enum-guarded one (an opaque field may itself contain the enum words), and over the bytes JSON uses for structure. The consequence is witnessed too: advancing one coordinate's cursor and writing its note leaves the other's cursor absent and its notes at zero. Each collision pair asserts its own control first, that the old scheme DID collide on it, so a green says the fix discriminates rather than that the pair was harmless. Mutation: restoring the pipe-join reds exactly the new witnesses (three pairs, eleven round-trips, the cursor-and-journal isolation) and nothing else, 22 of the original tests staying green; module restored and hash-verified against the pre-mutation bytes. 37 passed, 1 expected xfail, ruff clean. Co-Authored-By: Claude --- gr2/prototypes/propagation_state_machine.py | 33 +++++--- gr2/tests/test_propagation_state_machine.py | 85 +++++++++++++++++++++ 2 files changed, 109 insertions(+), 9 deletions(-) diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index 06c67c4..42ed1db 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -151,15 +151,30 @@ class Coordinate: artifact_class: str def key(self) -> str: - return "|".join( - ( - self.source, - self.destination, - self.layer, - str(self.direction), - str(self.operation), - self.artifact_class, - ) + """Canonical, injective key: the JSON of ``as_dict()`` with sorted keys. + + The fields are OPAQUE identifiers, so no delimiter can be assumed absent + from them. A ``"|".join`` made ``("source|dest", "target", ...)`` and + ``("source", "dest|target", ...)`` the same key, and the key names the + journal rows and the cursor file, so two coordinates would have shared + replay state (found in review of the first cut). JSON string encoding + escapes every byte that could be mistaken for structure, so the key + round-trips: ``json.loads(key) == as_dict()``. Distinct coordinates + therefore cannot collide, and the witness asserts the round-trip rather + than any particular pair. + """ + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + @classmethod + def from_key(cls, key: str) -> Coordinate: + data = json.loads(key) + return cls( + source=str(data["source"]), + destination=str(data["destination"]), + layer=str(data["layer"]), + direction=Direction(str(data["direction"])), + operation=Operation(str(data["operation"])), + artifact_class=str(data["artifact_class"]), ) def as_dict(self) -> dict[str, str]: diff --git a/gr2/tests/test_propagation_state_machine.py b/gr2/tests/test_propagation_state_machine.py index f4b0100..04059fa 100644 --- a/gr2/tests/test_propagation_state_machine.py +++ b/gr2/tests/test_propagation_state_machine.py @@ -627,6 +627,91 @@ def test_operation_id_binds_coordinate_source_base_and_digest(synthetic: Synthet assert ra.operation_id == expected +# ------------------------------------------------ coordinate key is injective + + +def _coord(**overrides: object) -> Coordinate: + base: dict[str, object] = { + "source": "source", + "destination": "target", + "layer": "layer", + "direction": Direction.DOWN, + "operation": Operation.APPLY, + "artifact_class": "class", + } + return Coordinate(**{**base, **overrides}) # type: ignore[arg-type] + + +def _joined_with_pipes(c: Coordinate) -> str: + """The scheme the first cut used, kept here ONLY as the control.""" + return "|".join( + (c.source, c.destination, c.layer, str(c.direction), str(c.operation), c.artifact_class) + ) + + +# Every pair below is two DISTINCT coordinates whose fields, joined with "|", +# produced ONE key on the first cut. The first pair is the reviewer's exact +# discriminator; the second shifts the byte across the source/destination/layer +# boundaries in one move; the third shows the enum fields are not a fence either, +# because an opaque field may itself contain the enum words. +_COLLIDING_ON_THE_OLD_SCHEME = [ + (_coord(source="source|dest"), _coord(destination="dest|target")), + (_coord(layer="layer|down"), _coord(source="source|target", destination="layer", layer="down")), + ( + _coord(artifact_class="class|down|apply|z"), + _coord(layer="layer|down|apply|class", artifact_class="z"), + ), +] + +# Bytes JSON itself uses for structure, in the opaque fields: these never collided +# under the old scheme, so they carry no control; they exist to show the new +# encoding escapes them and round-trips regardless. +_JSON_STRUCTURE_BYTES = [ + _coord(source='a","destination":"b'), + _coord(source="a\\"), + _coord(source="a\\\\"), + _coord(destination='{"source":"x"}'), + _coord(layer="|", artifact_class="|"), +] + + +@pytest.mark.parametrize("left,right", _COLLIDING_ON_THE_OLD_SCHEME) +def test_distinct_coordinates_never_share_a_key(left: Coordinate, right: Coordinate) -> None: + assert left != right, "the pair must be two different coordinates or it proves nothing" + # Control first: the pair MUST collide under the old scheme, or the witness is + # not discriminating and a green here would say nothing about the fix. + assert _joined_with_pipes(left) == _joined_with_pipes(right) + assert left.key() != right.key() + + +@pytest.mark.parametrize( + "coord", + [c for pair in _COLLIDING_ON_THE_OLD_SCHEME for c in pair] + _JSON_STRUCTURE_BYTES, +) +def test_coordinate_key_round_trips_so_injectivity_is_structural(coord: Coordinate) -> None: + # Injectivity is asserted as a ROUND TRIP, not as "this pair differs": if every + # key decodes back to the coordinate that produced it, no two coordinates can + # share one, whatever bytes the opaque fields carry. + assert Coordinate.from_key(coord.key()) == coord + assert json.loads(coord.key()) == coord.as_dict() + + +def test_distinct_coordinates_never_share_cursor_or_journal_rows(tmp_path: Path) -> None: + # The consequence the review named: the key names the cursor file and the journal + # rows, so a shared key is shared replay state. Advance one; the other is untouched. + left, right = _COLLIDING_ON_THE_OLD_SCHEME[0] + journal = Journal(tmp_path / "state") + assert journal.cursor_path(left.key()) != journal.cursor_path(right.key()) + journal.advance_cursor(left.key(), "a" * 40, pending_id="p") + assert journal.cursor(left.key()) == "a" * 40 + assert journal.cursor(right.key()) is None + journal.note( + pending_id="p", attempt=1, coordinate_key=left.key(), source_rev="a" * 40, note="only-left" + ) + assert journal.notes(left.key(), "a" * 40, "only-left") == 1 + assert journal.notes(right.key(), "a" * 40, "only-left") == 0 + + # ---------------------------------------------------- born-red outbox witness From 1b818c09ef5eb64a524d0dad900299c1ae2f47f0 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 10:24:43 -0500 Subject: [PATCH 07/29] fix(gr2): keep already-reached targets in an aggregate replay; constrain the outbox xfail to its cause Two defects found in second review, both in how the prototype REPORTS rather than in how it propagates. run_all dropped declared targets whose cursor was already at the source revision. run() returns None for "no new source revision" (not an operation, and that contract stands), and run_all took None to mean "not part of this invocation", so a replayed partial outcome reclassified itself: first run partial with reached=(clean,), second run -- nothing changed -- refused with reached=(). The aggregate must distinguish "already reached" from "never declared", and the only way to do that is to keep reporting the reached target. run_all now drives each target through a private path that, when the cursor is at the source revision, returns the terminal receipt from the journal (replayed=True) instead of None; a cursor at a revision the journal never acknowledged is a corrupted sink state and raises rather than guessing. The public run() is unchanged. Witnessed across three invocations: partial, then partial again with the reached target enumerated as a replayed terminal receipt and not applied twice (reflog count held at 1) while the refused target starts attempt 2, then acknowledged with both reached once the author cleans the refusal; plus an empty-targets control. The born-red outbox witness accepted any failure. xfail(strict=True) marks a test expected to fail for ANY reason, so a broken premise -- nothing emitted, nothing offered on the first read -- would have satisfied the marker forever while its reason text kept claiming cursor loss; replacing the first read's result with [] still reported the same expected xfail. The marker is now xfail(strict=True, raises=EventLost), where EventLost is raised only by the final survival check; every premise assertion is a plain assert, which is not the expected type and is reported as a real failure. The premise itself is also an ordinary test of its own, so an upstream outbox regression reds the suite instead of hiding inside the envelope. Evidence: 40 passed, 1 xfailed, ruff clean. Mutations, each restored from a saved copy and hash-verified: run_all back to the first cut's run() reds exactly the new replay witness; the witness's own first read returning [] reds the premise test AND reports the witness as FAILED rather than xfailed; an outbox that offers the event again reports XPASS(strict) as a failure, which is the marker-off signal the witness exists to give. Co-Authored-By: Claude --- gr2/prototypes/propagation_state_machine.py | 39 +++++++- gr2/tests/test_propagation_state_machine.py | 100 ++++++++++++++++++-- 2 files changed, 131 insertions(+), 8 deletions(-) diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index 42ed1db..fd7ee34 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -573,10 +573,23 @@ def _porcelain(self, destination: Destination) -> str: # -- driver def run(self, coordinate: Coordinate, destination: Destination) -> Receipt | None: + """Drive one coordinate. ``None`` means no new source revision: not an operation.""" + return self._run(coordinate, destination, report_current=False) + + def _run( + self, coordinate: Coordinate, destination: Destination, *, report_current: bool + ) -> Receipt | None: key = coordinate.key() observation = self.observe_source(coordinate) if not observation.is_new: - return None + if not report_current: + return None + # The cursor is AT the source revision, which only happens after an + # acknowledgement at that revision. For an aggregate the target was + # declared and already reached; report the terminal outcome rather + # than dropping the target, because a dropped target is + # indistinguishable from one that was never part of the invocation. + return self._terminal_receipt(coordinate, observation.source_rev) source_rev = observation.source_rev attempts = self.journal.find(key, source_rev) @@ -620,10 +633,32 @@ def run(self, coordinate: Coordinate, destination: Destination) -> Receipt | Non self._drive(op) return self._receipt(coordinate, source_rev, op.attempt, replayed=False) + def _terminal_receipt(self, coordinate: Coordinate, source_rev: str) -> Receipt: + """The original outcome for a coordinate whose cursor is already at ``source_rev``.""" + attempts = self.journal.find(coordinate.key(), source_rev) + if not attempts or attempts[-1][-1].state is not State.ACKNOWLEDGED: + # A cursor at a revision the journal never acknowledged is a corrupted + # sink state, and the prototype says so rather than guessing an outcome. + raise RuntimeError( + f"cursor for {coordinate.destination} is at {source_rev} but the journal " + "carries no acknowledged attempt at that revision" + ) + return self._receipt(coordinate, source_rev, len(attempts), replayed=True) + def run_all(self, targets: Iterable[tuple[Coordinate, Destination]]) -> PlanOutcome | None: + """Drive every declared target and classify the aggregate. + + A declared target whose cursor is already at the source revision was reached + by an earlier invocation; it is reported as reached with its terminal receipt + (``replayed=True``), never dropped. Dropping it would reclassify a replayed + ``partial`` as ``refused``, which is what the first cut did (found in review): + the aggregate must distinguish "already reached" from "not part of this + invocation", and the only way to do that is to keep reporting it. + ``None`` only when no targets were declared. + """ receipts: list[Receipt] = [] for coordinate, destination in targets: - receipt = self.run(coordinate, destination) + receipt = self._run(coordinate, destination, report_current=True) if receipt is not None: receipts.append(receipt) if not receipts: diff --git a/gr2/tests/test_propagation_state_machine.py b/gr2/tests/test_propagation_state_machine.py index 04059fa..8ec6869 100644 --- a/gr2/tests/test_propagation_state_machine.py +++ b/gr2/tests/test_propagation_state_machine.py @@ -588,6 +588,55 @@ def test_mixed_destinations_are_partial_with_both_sides_enumerated(synthetic: Sy assert (dirty.path / "canon.md").read_text() == "uncommitted\n" +def test_replaying_a_partial_outcome_keeps_the_reached_target_and_stays_partial( + synthetic: Synthetic, +) -> None: + # Found in review: the first cut's run_all dropped a declared target whose cursor + # was already at the source revision (run() returns None for "nothing new"), so a + # replayed partial reclassified itself as REFUSED with reached=() -- the + # aggregate could not tell "already reached" from "not part of this invocation". + clean = replica(synthetic, "replica") + dirty = authoring(synthetic, "dirty-authoring") + (dirty.path / "canon.md").write_text("uncommitted\n") + new = synthetic.push_change("canon v2\n") + targets = [(coordinate(d.destination_id), d) for d in (clean, dirty)] + + first = propagator(synthetic).run_all(targets) + assert first is not None and first.state is State.PARTIAL + assert first.reached == ("replica",) + assert [d for d, _ in first.not_reached] == ["dirty-authoring"] + assert head_moves(clean.path) == 1 + + # Replay with nothing changed: the SAME classification, the reached target is + # still enumerated (as a replayed terminal receipt), and it is not applied again. + second = propagator(synthetic).run_all(targets) + assert second is not None and second.state is State.PARTIAL, second + assert second.reached == ("replica",) + assert [d for d, _ in second.not_reached] == ["dirty-authoring"] + by_dest = {r.coordinate.destination: r for r in second.receipts} + assert by_dest["replica"].replayed is True + assert by_dest["replica"].state is State.ACKNOWLEDGED + assert by_dest["replica"].after == new + assert by_dest["dirty-authoring"].replayed is False # a refusal starts a new attempt + assert by_dest["dirty-authoring"].attempt == 2 + assert head_moves(clean.path) == 1, "the reached target must not be applied twice" + + # The author cleans up: the refused target is reached on a fresh attempt and the + # aggregate becomes ACKNOWLEDGED with BOTH targets enumerated as reached. + git(dirty.path, "checkout", "--", "canon.md") + third = propagator(synthetic).run_all(targets) + assert third is not None and third.state is State.ACKNOWLEDGED, third + assert set(third.reached) == {"replica", "dirty-authoring"} + assert third.not_reached == () + assert git(dirty.path, "rev-parse", "HEAD") == new + assert head_moves(clean.path) == 1 + + +def test_run_all_with_no_declared_targets_is_none(synthetic: Synthetic) -> None: + synthetic.push_change("canon v2\n") + assert propagator(synthetic).run_all([]) is None + + def test_all_destinations_verifying_is_acknowledged_not_partial(synthetic: Synthetic) -> None: a = replica(synthetic, "replica-a") b = replica(synthetic, "replica-b") @@ -715,13 +764,49 @@ def test_distinct_coordinates_never_share_cursor_or_journal_rows(tmp_path: Path) # ---------------------------------------------------- born-red outbox witness +class EventLost(AssertionError): + """Raised ONLY by the survival check below; the xfail is constrained to it. + + ``xfail(strict=True)`` alone accepts ANY failure in the marked test, so a + premise that broke -- no event emitted, nothing offered on the first read -- + would satisfy the marker forever while its reason text kept claiming cursor + loss (found in review: replacing the first read's result with ``[]`` still + reported the same expected xfail). With ``raises=EventLost`` a premise + failure is a plain AssertionError, which is NOT the expected type, so it is + reported as a real failure; only the final survival check can satisfy the + marker, and only by raising this class explicitly. + """ + + +def _offered_types(workspace: Path) -> list[str]: + return [e["type"] for e in read_events(workspace, "propagation-consumer")] + + +def test_outbox_offers_an_emitted_event_to_a_fresh_consumer(tmp_path: Path) -> None: + # The PREMISE of the born-red witness, as its own ordinary test: an emitted + # event is offered on the first read. If this goes red the outbox regressed + # upstream of the cursor question and the witness below is not about that. + workspace = tmp_path / "workspace" + workspace.mkdir() + emit( + EventType.SYNC_COMPLETED, + workspace, + actor="sink", + owner_unit="unit-0", + payload={"detail": "one"}, + ) + assert _offered_types(workspace) == ["sync.completed"] + + @pytest.mark.xfail( strict=True, + raises=EventLost, reason=( "outbox consumers lose events: read_events() advances the cursor before the " "caller performs its effect, so a consumer that fails after reading never sees " "the event again. This witness turns green only when acknowledgment moves " - "after the effect; strict=True forces the marker off at that moment." + "after the effect; strict=True forces the marker off at that moment, and " + "raises=EventLost keeps every premise failure outside the expected envelope." ), ) def test_outbox_event_survives_a_consumer_that_fails_after_reading(tmp_path: Path) -> None: @@ -735,11 +820,14 @@ def test_outbox_event_survives_a_consumer_that_fails_after_reading(tmp_path: Pat payload={"detail": "one"}, ) + # PREMISE (plain asserts: a failure here is a FAILURE, not the expected xfail) with pytest.raises(RuntimeError): - offered = read_events(workspace, "propagation-consumer") - assert [e["type"] for e in offered] == ["sync.completed"] + offered = _offered_types(workspace) + assert offered == ["sync.completed"], f"premise: nothing offered, got {offered!r}" raise RuntimeError("effect failed after the read") - # the contract a propagation consumer needs: an event whose effect did not happen - # is offered again - assert [e["type"] for e in read_events(workspace, "propagation-consumer")] == ["sync.completed"] + # SURVIVAL CHECK, the only statement allowed to satisfy the marker: an event + # whose effect did not happen is offered again + again = _offered_types(workspace) + if again != ["sync.completed"]: + raise EventLost(f"offered once, then lost: second read returned {again!r}") From 554ceafa5925233b196ffa3518582583578fc7b3 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 10:43:59 -0500 Subject: [PATCH 08/29] fix(gr2): witness the corrupted-cursor refusal on both driver paths Found in third review: the branch that refuses a cursor sitting at the source revision with no acknowledged journal attempt behind it was promised by the previous fix and by the PR body, and nothing witnessed it. Replacing the RuntimeError with a silent return None kept the whole file green -- and a silent None there is the exact outcome class the previous fix closed, a declared target dropped from the aggregate, now with a corrupted sink state underneath it instead of a healthy one. The check now runs on both driver paths, not only the aggregate one: run() and run_all() both consult the journal when the cursor is already at the source revision, so a cursor the journal never acknowledged raises from either entry point instead of run() reporting "nothing new" over it. The witness forges the one state the journal can never produce (cursor advanced with no rows), asserts both entry points raise with the missing- acknowledgement detail, asserts the destination is byte-for-byte untouched, and carries a healthy control: an acknowledged cursor is the ordinary current state, None from run() and an already-reached replayed receipt from run_all(). Evidence: 41 passed, 1 xfailed, ruff clean. Mutation, restored from a saved copy and hash-verified against the pre-mutation bytes: the raise replaced by return None reds exactly the new witness and nothing else. Co-Authored-By: Claude --- gr2/prototypes/propagation_state_machine.py | 21 ++++++++------ gr2/tests/test_propagation_state_machine.py | 32 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index fd7ee34..8c03426 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -573,7 +573,10 @@ def _porcelain(self, destination: Destination) -> str: # -- driver def run(self, coordinate: Coordinate, destination: Destination) -> Receipt | None: - """Drive one coordinate. ``None`` means no new source revision: not an operation.""" + """Drive one coordinate. ``None`` means no new source revision: not an operation. + + Raises if the cursor sits at a revision the journal never acknowledged. + """ return self._run(coordinate, destination, report_current=False) def _run( @@ -582,14 +585,16 @@ def _run( key = coordinate.key() observation = self.observe_source(coordinate) if not observation.is_new: - if not report_current: - return None # The cursor is AT the source revision, which only happens after an - # acknowledgement at that revision. For an aggregate the target was - # declared and already reached; report the terminal outcome rather - # than dropping the target, because a dropped target is - # indistinguishable from one that was never part of the invocation. - return self._terminal_receipt(coordinate, observation.source_rev) + # acknowledgement at that revision; _terminal_receipt RAISES if the + # journal carries no such acknowledgement, on both paths, because a + # cursor the journal cannot account for is a corrupted sink state and + # "nothing new" is exactly the answer that would hide it. For an + # aggregate the target was declared and already reached; report the + # terminal outcome rather than dropping the target, because a dropped + # target is indistinguishable from one never part of the invocation. + terminal = self._terminal_receipt(coordinate, observation.source_rev) + return terminal if report_current else None source_rev = observation.source_rev attempts = self.journal.find(key, source_rev) diff --git a/gr2/tests/test_propagation_state_machine.py b/gr2/tests/test_propagation_state_machine.py index 8ec6869..952bd94 100644 --- a/gr2/tests/test_propagation_state_machine.py +++ b/gr2/tests/test_propagation_state_machine.py @@ -632,6 +632,38 @@ def test_replaying_a_partial_outcome_keeps_the_reached_target_and_stays_partial( assert head_moves(clean.path) == 1 +def test_a_cursor_the_journal_never_acknowledged_is_a_corrupted_sink_state_and_raises( + synthetic: Synthetic, +) -> None: + # Found in review: the "corrupted sink state" branch was unwitnessed, so a silent + # `return None` in its place kept the whole file green while dropping a declared + # target from the aggregate -- the same outcome class the previous fix closed. + dest = replica(synthetic, "replica") + new = synthetic.push_change("canon v2\n") + coord = coordinate(dest.destination_id) + # Forge the one state the journal can never produce: a cursor AT the source + # revision with no acknowledged attempt behind it. + Journal(synthetic.state_dir).advance_cursor(coord.key(), new, pending_id="forged") + before = snapshot(dest.path) + + with pytest.raises(RuntimeError, match="no acknowledged attempt"): + propagator(synthetic).run_all([(coord, dest)]) + with pytest.raises(RuntimeError, match="no acknowledged attempt"): + propagator(synthetic).run(coord, dest) + assert snapshot(dest.path) == before, "a refusal to guess must not touch the destination" + + # Control: a cursor the journal DID acknowledge is the ordinary current state -- + # run() reports nothing new and run_all reports the target as already reached. + healthy = replica(synthetic, "healthy") + hcoord = coordinate(healthy.destination_id) + first = propagator(synthetic).run(hcoord, healthy) + assert first is not None and first.state is State.ACKNOWLEDGED + assert propagator(synthetic).run(hcoord, healthy) is None + again = propagator(synthetic).run_all([(hcoord, healthy)]) + assert again is not None and again.state is State.ACKNOWLEDGED + assert again.reached == ("healthy",) and again.receipts[0].replayed is True + + def test_run_all_with_no_declared_targets_is_none(synthetic: Synthetic) -> None: synthetic.push_change("canon v2\n") assert propagator(synthetic).run_all([]) is None From 078dfabb01378e423ec67aabdc0cda7896280f61 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 11:50:47 -0500 Subject: [PATCH 09/29] feat(gr2): Prototype 1 propagation daemon on one declared managed replica Run the Prototype 0 state machine on a loop against a single destination that a declaration names as a managed replica of one branch of one source. One tick observes the source with ls-remote; when the cursor already names the source revision there is no operation and nothing is written; otherwise the machine runs once and its receipt, whatever its state, is written as its own JSON file with per-state latency derived from the receipt's own transition timestamps, and announced as one propagation.receipt event on the gr2 outbox. A refusal is a receipt too, and a refused replica is left untouched. The declaration can only name a replica: an authoring kind, an unknown git environment, a non-positive interval, or a missing field is refused before any git call. ensure_replica clones the declared branch single-branch when the path is absent and refuses a checkout whose origin or branch differ, or a non-git directory, before the machine ever reads it. Adds EventType.PROPAGATION_RECEIPT (documented in HOOK-EVENT-CONTRACT.md 3.2 and 7.2, with receipt_path declared as an explicit exception to the relative-path rule) and bumps the exhaustive EventType count test to 36. Tests: 35 new witnesses on synthetic repositories (declaration refusals, replica ensure and refuse, first tick, current tick, pushed change, refused then applied, receipt round trip with recomputable latency, loop and CLI). run_loop resolves stdout at call time so a redirecting caller gets the lines. Co-Authored-By: Claude --- gr2/docs/HOOK-EVENT-CONTRACT.md | 17 + gr2/prototypes/propagation_daemon.py | 467 ++++++++++++++++++ gr2/python_cli/events.py | 4 + gr2/tests/test_events.py | 5 +- gr2/tests/test_propagation_daemon.py | 685 +++++++++++++++++++++++++++ 5 files changed, 1176 insertions(+), 2 deletions(-) create mode 100644 gr2/prototypes/propagation_daemon.py create mode 100644 gr2/tests/test_propagation_daemon.py diff --git a/gr2/docs/HOOK-EVENT-CONTRACT.md b/gr2/docs/HOOK-EVENT-CONTRACT.md index e961599..6b28851 100644 --- a/gr2/docs/HOOK-EVENT-CONTRACT.md +++ b/gr2/docs/HOOK-EVENT-CONTRACT.md @@ -197,6 +197,20 @@ and `lease.force_broken` (which fires when a live lease is broken with | `workspace.materialized` | `gr2 workspace materialize` or `gr2 apply` | `{repos: [{repo, first_materialize: bool}]}` | | `workspace.file_projected` | File link/copy applied | `{repo, kind, src, dest}` | +#### Propagation + +| Type | Trigger | Payload | +|------|---------|---------| +| `propagation.receipt` | The propagation daemon (`gr2/prototypes/propagation_daemon.py`) completed one operation against its declared managed replica, in any terminal state (acknowledged, refused, partial, unverifiable) | `{summary, state, pending_id, operation_id, source_rev, expected_base, after, replayed, receipt_path}` | + +`propagation.receipt` is emitted once per operation; a tick that finds the cursor +already at the source revision is not an operation and emits nothing. `summary` is +the one-line notification a channel consumer relays verbatim. `receipt_path` is the +full path of the receipt file the daemon wrote, an explicit exception to the +relative-path rule in 3.3: the receipt store lives under the daemon's declared +`state_dir`, which is not a workspace file and may sit outside `workspace_root`. +`after` is `null` when the operation did not reach `applied`. + ### 3.3 Payload Conventions - All paths in payloads are relative to `workspace_root`, never absolute. @@ -471,6 +485,9 @@ class EventType(str, Enum): # Workspace operations WORKSPACE_MATERIALIZED = "workspace.materialized" WORKSPACE_FILE_PROJECTED = "workspace.file_projected" + + # Propagation (one event per receipt from the propagation daemon) + PROPAGATION_RECEIPT = "propagation.receipt" ``` ### 7.3 Implementation Location diff --git a/gr2/prototypes/propagation_daemon.py b/gr2/prototypes/propagation_daemon.py new file mode 100644 index 0000000..c169c1f --- /dev/null +++ b/gr2/prototypes/propagation_daemon.py @@ -0,0 +1,467 @@ +"""Prototype 1 of the propagation daemon: ONE declared managed replica, a real source. + +Prototype 0 (``propagation_state_machine``) proved the state contract on synthetic +repositories. This module runs that same machine on a loop against a single destination +that a declaration names as a managed replica, so the daemon can watch a real canonical +upstream, fetch, plan, apply, verify, acknowledge, and leave a neutral receipt behind +for every operation it ran. Nothing here grants the daemon authority over an authoring +clone: the declaration can only name a replica, and the machine's own gates (direction, +base unmoved, clean, fast-forward) refuse everything the declaration cannot vouch for. + +What one tick does, in order: + +1. observe the source with ``git ls-remote``; if the cursor already names the source + revision there is no operation, no receipt, and nothing is written; +2. otherwise drive the machine once and take its receipt, whatever its state -- + a refusal is a receipt too, because "I did not apply and here is why" is the + only honest thing a replica manager can say about a destination it refused; +3. write the receipt as its own JSON file, with the per-state latency derived from + the receipt's own transition timestamps rather than from a stopwatch around + the call, so the numbers describe what the journal describes; +4. emit one ``propagation.receipt`` event on the gr2 outbox carrying the receipt's + one-line summary. The design names a one-line channel notification; this module + writes that line to the outbox and to stdout and knows no channel, so the + prototype depends on nothing outside ``gr2``. + +Success for Prototype 1 is measured latency plus exact receipts, not the absence of an +exception. Every receipt names the exact source and destination revisions, and every +latency figure is recomputable from the receipt that carries it. + +The git environment is a declared choice. Tests run isolated from the host's +configuration (the Prototype 0 default); a real private remote needs the host's +credential helper, so a declaration may say ``"git_env": "inherit"``. The daemon never +creates a commit (fast-forward only), so inheriting the host configuration does not +invoke signing. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass, field, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import TextIO + +from gr2.prototypes.propagation_state_machine import ( + Coordinate, + Destination, + DestinationKind, + Direction, + Operation, + Policy, + Propagator, + Receipt, + State, +) +from gr2.python_cli.events import EventType, emit + +_ISOLATED_GIT_ENV = { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", +} + +ACTOR = "propagation-daemon" + + +class DeclarationError(ValueError): + """The declaration does not describe exactly one managed replica.""" + + +class DeclarationMismatch(RuntimeError): + """The destination path exists and is not the declared replica.""" + + +@dataclass(frozen=True) +class Declaration: + """Exactly one managed replica of one branch of one source. + + Every field is opaque to the machine except ``branch`` (what ls-remote asks for) and + ``destination_path`` (where the replica lives). ``kind`` is carried so a reader of the + declaration can see the only value it may hold; ``load_declaration`` refuses any other. + """ + + source_url: str + branch: str + destination_id: str + destination_path: Path + state_dir: Path + outbox_root: Path + coordinate: Coordinate + interval_seconds: float + policy_hash: str + git_env_mode: str = "isolated" + kind: DestinationKind = DestinationKind.REPLICA + + @property + def receipts_dir(self) -> Path: + return self.state_dir / "receipts" + + @property + def git_env(self) -> dict[str, str]: + return {} if self.git_env_mode == "inherit" else dict(_ISOLATED_GIT_ENV) + + def destination(self) -> Destination: + return Destination( + destination_id=self.destination_id, path=self.destination_path, kind=self.kind + ) + + def policy(self) -> Policy: + # downward only: the one direction a managed replica can be the destination of + return Policy(policy_hash=self.policy_hash, allowed_directions=frozenset({Direction.DOWN})) + + +_REQUIRED = ( + "source_url", + "branch", + "destination_id", + "destination_path", + "state_dir", + "outbox_root", + "coordinate", +) + + +def declaration_from_dict(data: dict[str, object]) -> Declaration: + missing = [key for key in _REQUIRED if key not in data] + if missing: + raise DeclarationError(f"declaration is missing {missing}") + kind = str(data.get("kind", DestinationKind.REPLICA)) + if kind != str(DestinationKind.REPLICA): + # the one thing this daemon must never be told to do: an authoring clone is not + # a destination it may drive, and that is decided here, before any git call + raise DeclarationError(f"this daemon drives managed replicas only; declared kind={kind!r}") + git_env_mode = str(data.get("git_env", "isolated")) + if git_env_mode not in {"isolated", "inherit"}: + raise DeclarationError(f"git_env must be 'isolated' or 'inherit', got {git_env_mode!r}") + coord = data["coordinate"] + if not isinstance(coord, dict): + raise DeclarationError("coordinate must be an object") + for key in ("source", "layer", "artifact_class"): + if key not in coord: + raise DeclarationError(f"coordinate is missing {key!r}") + interval = float(data.get("interval_seconds", 30.0)) + if interval <= 0: + raise DeclarationError("interval_seconds must be positive") + destination_id = str(data["destination_id"]) + coordinate = Coordinate( + source=str(coord["source"]), + destination=destination_id, + layer=str(coord["layer"]), + direction=Direction.DOWN, + operation=Operation.APPLY, + artifact_class=str(coord["artifact_class"]), + ) + return Declaration( + source_url=str(data["source_url"]), + branch=str(data["branch"]), + destination_id=destination_id, + destination_path=Path(str(data["destination_path"])).expanduser(), + state_dir=Path(str(data["state_dir"])).expanduser(), + outbox_root=Path(str(data["outbox_root"])).expanduser(), + coordinate=coordinate, + interval_seconds=interval, + policy_hash=str(data.get("policy_hash", "prototype-1-downward-only")), + git_env_mode=git_env_mode, + ) + + +def load_declaration(path: Path) -> Declaration: + with path.open() as handle: + data = json.load(handle) + if not isinstance(data, dict): + raise DeclarationError("declaration must be a JSON object") + return declaration_from_dict(data) + + +# --------------------------------------------------------------------------- replica + + +def _git(repo: Path, *args: str, env: dict[str, str]) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + env={**os.environ, **env}, + ) + return proc.stdout.strip() + + +def ensure_replica(declaration: Declaration) -> Path: + """Make the declared replica exist, or refuse to treat what is there as the replica. + + Absent: clone the declared branch of the declared source, single-branch, so the only + thing at that path is what the declaration says. Present: it must be a git checkout + whose ``origin`` is the declared source and whose current branch is the declared + branch; anything else is refused here, before the machine ever reads it, because the + alternative is a daemon fast-forwarding a clone that happens to sit at the path. + """ + path = declaration.destination_path + env = declaration.git_env + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + "git", + "clone", + "-q", + "--branch", + declaration.branch, + "--single-branch", + declaration.source_url, + str(path), + ], + check=True, + capture_output=True, + text=True, + env={**os.environ, **env}, + ) + return path + try: + origin = _git(path, "remote", "get-url", "origin", env=env) + branch = _git(path, "rev-parse", "--abbrev-ref", "HEAD", env=env) + except (subprocess.CalledProcessError, OSError) as exc: + raise DeclarationMismatch(f"{path} exists and is not a git checkout: {exc}") from exc + if origin != declaration.source_url: + raise DeclarationMismatch( + f"{path} has origin {origin!r}, declaration names {declaration.source_url!r}" + ) + if branch != declaration.branch: + raise DeclarationMismatch( + f"{path} is on {branch!r}, declaration names {declaration.branch!r}" + ) + return path + + +# --------------------------------------------------------------------------- ticks + + +@dataclass(frozen=True) +class TickResult: + observed_at: str + source_rev: str + cursor_before: str | None + receipt: Receipt | None + receipt_path: Path | None + latency_seconds: dict[str, float] | None + summary: str + + @property + def was_operation(self) -> bool: + return self.receipt is not None + + +def _parse_ts(value: str) -> datetime: + return datetime.fromisoformat(value) + + +def latency_from_receipt(receipt: Receipt) -> dict[str, float]: + """Per-state latency derived from the receipt's own transition timestamps. + + Keys are ``->`` for each consecutive pair and ``total`` from the first to + the last transition. A replayed receipt carries the ORIGINAL transitions, so its + latency describes the run that did the work, not the replay that found it. + """ + transitions = receipt.transitions + out: dict[str, float] = {} + for previous, current in zip(transitions, transitions[1:], strict=False): + delta = (_parse_ts(current.timestamp) - _parse_ts(previous.timestamp)).total_seconds() + out[f"{previous.state}->{current.state}"] = round(delta, 6) + if transitions: + total = ( + _parse_ts(transitions[-1].timestamp) - _parse_ts(transitions[0].timestamp) + ).total_seconds() + out["total"] = round(total, 6) + return out + + +def _short(rev: str | None) -> str: + return (rev or "-")[:12] + + +def summarize(receipt: Receipt, latency: dict[str, float]) -> str: + """The one line a channel reader needs: what, where, which revisions, how long.""" + landed = _short(receipt.after) if receipt.after else "unchanged" + head = ( + f"propagation {receipt.state}: {receipt.coordinate.destination} " + f"at {_short(receipt.expected_base)} -> {landed}, intended {_short(receipt.source_rev)} " + f"(source {receipt.coordinate.source}, attempt {receipt.attempt}" + ) + if receipt.replayed: + head += ", replayed" + head += f", total {latency.get('total', 0.0):.3f}s)" + if receipt.state is State.REFUSED and receipt.refusal_reason: + head += f" refused: {receipt.refusal_reason}" + return head + + +def write_receipt( + declaration: Declaration, receipt: Receipt, latency: dict[str, float], observed_at: str +) -> Path: + declaration.receipts_dir.mkdir(parents=True, exist_ok=True) + stamp = observed_at.replace(":", "").replace("+00:00", "Z") + name = f"{stamp}-{receipt.pending_id[:12]}-{receipt.state}.json" + path = declaration.receipts_dir / name + payload = { + "daemon": ACTOR, + "declaration": { + "source_url": declaration.source_url, + "branch": declaration.branch, + "destination_id": declaration.destination_id, + "destination_path": str(declaration.destination_path), + }, + "observed_at": observed_at, + "latency_seconds": latency, + "receipt": receipt.as_dict(), + } + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(tmp, path) + return path + + +def notify(declaration: Declaration, receipt: Receipt, summary: str, receipt_path: Path) -> None: + emit( + EventType.PROPAGATION_RECEIPT, + declaration.outbox_root, + ACTOR, + declaration.destination_id, + { + "summary": summary, + "state": str(receipt.state), + "pending_id": receipt.pending_id, + "operation_id": receipt.operation_id, + "source_rev": receipt.source_rev, + "expected_base": receipt.expected_base, + "after": receipt.after, + "replayed": receipt.replayed, + "receipt_path": str(receipt_path), + }, + ) + + +def make_propagator(declaration: Declaration) -> Propagator: + # The machine only ever stringifies ``source_remote`` (ls-remote, fetch), so a URL + # passes through as-is. It must NOT be wrapped in Path: Path collapses the "//" of a + # URL scheme and the source would silently become a relative directory. + return Propagator( + source_remote=declaration.source_url, # type: ignore[arg-type] + branch=declaration.branch, + state_dir=declaration.state_dir, + policy=declaration.policy(), + git_env=declaration.git_env, + ) + + +def tick(declaration: Declaration, propagator: Propagator) -> TickResult: + observed_at = datetime.now(UTC).isoformat() + coordinate = declaration.coordinate + observation = propagator.observe_source(coordinate) + if not observation.is_new: + return TickResult( + observed_at=observed_at, + source_rev=observation.source_rev, + cursor_before=observation.cursor, + receipt=None, + receipt_path=None, + latency_seconds=None, + summary=( + f"propagation current: {declaration.destination_id} cursor already at " + f"{_short(observation.source_rev)}; not an operation" + ), + ) + receipt = propagator.run(coordinate, declaration.destination()) + if receipt is None: + # the source moved between observe and run, back to the cursor; nothing to do + return TickResult( + observed_at=observed_at, + source_rev=observation.source_rev, + cursor_before=observation.cursor, + receipt=None, + receipt_path=None, + latency_seconds=None, + summary=( + f"propagation current: {declaration.destination_id} source returned to the " + f"cursor between observe and run; not an operation" + ), + ) + latency = latency_from_receipt(receipt) + summary = summarize(receipt, latency) + receipt_path = write_receipt(declaration, receipt, latency, observed_at) + notify(declaration, receipt, summary, receipt_path) + return TickResult( + observed_at=observed_at, + source_rev=observation.source_rev, + cursor_before=observation.cursor, + receipt=receipt, + receipt_path=receipt_path, + latency_seconds=latency, + summary=summary, + ) + + +@dataclass +class LoopStats: + ticks: int = 0 + operations: int = 0 + by_state: dict[str, int] = field(default_factory=dict) + last: TickResult | None = None + + +def run_loop( + declaration: Declaration, + *, + once: bool = False, + stop: Callable[[], bool] | None = None, + sleep: Callable[[float], None] = time.sleep, + out: TextIO | None = None, +) -> LoopStats: + """Tick until ``stop()`` says so (or once). Every tick prints exactly one line. + + ``out`` defaults to the stdout in force at CALL time, not at import time, so a caller + that redirects stdout (a test, a wrapper, a supervisor) gets the lines. + """ + stream = out if out is not None else sys.stdout + ensure_replica(declaration) + propagator = make_propagator(declaration) + stats = LoopStats() + while True: + result = tick(declaration, propagator) + stats.ticks += 1 + stats.last = result + if result.receipt is not None: + stats.operations += 1 + key = str(result.receipt.state) + stats.by_state[key] = stats.by_state.get(key, 0) + 1 + print(f"{result.observed_at} {result.summary}", file=stream, flush=True) + if once or (stop is not None and stop()): + return stats + sleep(declaration.interval_seconds) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--declaration", required=True, type=Path) + parser.add_argument("--once", action="store_true", help="one tick, then exit") + parser.add_argument( + "--interval", type=float, default=None, help="override the declared interval" + ) + args = parser.parse_args(argv) + declaration = load_declaration(args.declaration) + if args.interval is not None: + declaration = replace(declaration, interval_seconds=float(args.interval)) + try: + run_loop(declaration, once=args.once) + except KeyboardInterrupt: + print("propagation daemon: stopped", flush=True) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() in tests + sys.exit(main()) diff --git a/gr2/python_cli/events.py b/gr2/python_cli/events.py index d0d7a1d..0793108 100644 --- a/gr2/python_cli/events.py +++ b/gr2/python_cli/events.py @@ -84,6 +84,10 @@ class EventType(str, Enum): WORKSPACE_MATERIALIZED = "workspace.materialized" WORKSPACE_FILE_PROJECTED = "workspace.file_projected" + # one event per propagation receipt: the daemon's notification line, carried on the + # outbox so a consumer can relay it without the daemon knowing any channel + PROPAGATION_RECEIPT = "propagation.receipt" + def _outbox_path(workspace_root: Path) -> Path: return workspace_root / ".grip" / "events" / "outbox.jsonl" diff --git a/gr2/tests/test_events.py b/gr2/tests/test_events.py index 95c7019..eb78ede 100644 --- a/gr2/tests/test_events.py +++ b/gr2/tests/test_events.py @@ -81,8 +81,9 @@ def test_workspace_operation_types(self): def test_total_count(self): from gr2.python_cli.events import EventType - # 5 lane + 4 lease + 4 hook + 7 PR + 8 sync + 3 exec + 2 recovery + 2 workspace = 35 - assert len(EventType) == 35 + # 5 lane + 4 lease + 4 hook + 7 PR + 8 sync + 3 exec + 2 recovery + 2 workspace + # + 1 propagation = 36 + assert len(EventType) == 36 # --------------------------------------------------------------------------- diff --git a/gr2/tests/test_propagation_daemon.py b/gr2/tests/test_propagation_daemon.py new file mode 100644 index 0000000..c9eca76 --- /dev/null +++ b/gr2/tests/test_propagation_daemon.py @@ -0,0 +1,685 @@ +"""Prototype 1: the propagation daemon, proven on a synthetic source and a declared replica. + +Everything here runs under ``tmp_path``: one bare "source" remote with a config-like file +on ``main``, an authoring clone that pushes changes to it, and a replica path that the +declaration names. No real workspace, remote, or authoring clone is touched. + +What the tests prove, each as its own witness: + +* a declaration names exactly one managed replica; an authoring kind, an unknown git + environment, a non-positive interval, or a missing field is refused before any git call +* ``ensure_replica`` clones the declared branch when the path is absent, accepts the path + it declared, and refuses a checkout whose origin or branch differ, or a non-git directory +* the first tick on a fresh replica acknowledges without running the verb (the replica + was born at the source revision); a tick with no new revision is not an operation and + writes nothing; a pushed change is applied on the next tick with exact revisions +* every receipt that is an operation is written as its own JSON file and announced as one + ``propagation.receipt`` event on the outbox; a refusal is a receipt too, and the refused + destination is left untouched +* the latency figures in a written receipt are recomputable from that receipt's own + transition timestamps +* ``run_loop`` ticks once under ``once``, stops when told, sleeps the declared interval, + and prints exactly one line per tick +""" + +from __future__ import annotations + +import io +import json +import os +import subprocess +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +import pytest +from gr2.prototypes.propagation_daemon import ( + ACTOR, + Declaration, + DeclarationError, + DeclarationMismatch, + declaration_from_dict, + ensure_replica, + latency_from_receipt, + load_declaration, + main, + make_propagator, + run_loop, + summarize, + tick, +) +from gr2.prototypes.propagation_state_machine import ( + DestinationKind, + Direction, + Operation, + State, +) +from gr2.python_cli.events import EventType + +# Commits in the synthetic repositories must not depend on, or touch, the machine's +# global git configuration (signing keys, hooks paths, identities). +_GIT_ENV = { + "GIT_AUTHOR_NAME": "prototype", + "GIT_AUTHOR_EMAIL": "prototype@example.invalid", + "GIT_COMMITTER_NAME": "prototype", + "GIT_COMMITTER_EMAIL": "prototype@example.invalid", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", +} + + +def git(repo: Path, *args: str) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + return proc.stdout.strip() + + +def _run(*args: str) -> None: + subprocess.run( + list(args), check=True, capture_output=True, text=True, env={**os.environ, **_GIT_ENV} + ) + + +def snapshot(repo: Path) -> dict[str, str]: + """Everything an 'untouched' claim is about: refs, HEAD, index+worktree state, bytes.""" + return { + "refs": git(repo, "for-each-ref"), + "head": git(repo, "rev-parse", "HEAD"), + "porcelain": git(repo, "status", "--porcelain"), + "canon": (repo / "canon.md").read_text(), + "reflog": git(repo, "reflog", "show", "--format=%gs", "HEAD"), + } + + +@dataclass +class Synthetic: + remote: Path + author: Path + base: str + root: Path + + def push_change(self, text: str) -> str: + (self.author / "canon.md").write_text(text) + git(self.author, "add", "canon.md") + git(self.author, "commit", "-q", "-m", f"canon: {text.strip()[:40]}") + git(self.author, "push", "-q", "origin", "main") + return git(self.author, "rev-parse", "HEAD") + + +def _bare_with_one_commit(root: Path, name: str, text: str) -> tuple[Path, Path, str]: + remote = root / f"{name}.git" + _run("git", "init", "-q", "--bare", "--initial-branch=main", str(remote)) + author = root / f"{name}-author" + _run("git", "clone", "-q", str(remote), str(author)) + git(author, "switch", "-q", "-c", "main") + (author / "canon.md").write_text(text) + git(author, "add", "canon.md") + git(author, "commit", "-q", "-m", text.strip()) + git(author, "push", "-q", "-u", "origin", "main") + return remote, author, git(author, "rev-parse", "HEAD") + + +@pytest.fixture +def synthetic(tmp_path: Path) -> Synthetic: + remote, author, base = _bare_with_one_commit(tmp_path, "source", "canon v1\n") + return Synthetic(remote=remote, author=author, base=base, root=tmp_path) + + +def declaration_dict(syn: Synthetic, **overrides: object) -> dict[str, object]: + data: dict[str, object] = { + "source_url": str(syn.remote), + "branch": "main", + "destination_id": "config-main-replica", + "destination_path": str(syn.root / "replicas" / "config-main"), + "state_dir": str(syn.root / "propagation" / "config-main"), + "outbox_root": str(syn.root / "workspace"), + "coordinate": { + "source": "synthetic/source", + "layer": "config", + "artifact_class": "config-tree", + }, + "interval_seconds": 7.5, + } + data.update(overrides) + return data + + +@pytest.fixture +def declaration(synthetic: Synthetic) -> Declaration: + return declaration_from_dict(declaration_dict(synthetic)) + + +def outbox_events(root: Path) -> list[dict[str, object]]: + outbox = root / ".grip" / "events" / "outbox.jsonl" + if not outbox.exists(): + return [] + return [json.loads(line) for line in outbox.read_text().splitlines() if line.strip()] + + +def receipt_files(decl: Declaration) -> list[Path]: + if not decl.receipts_dir.exists(): + return [] + return sorted(p for p in decl.receipts_dir.iterdir() if p.suffix == ".json") + + +def applied_observation(receipt) -> dict[str, object]: + applied = [t for t in receipt.transitions if t.state is State.APPLIED] + assert len(applied) == 1, [t.state for t in receipt.transitions] + return applied[0].observation + + +# ----------------------------------------------------------------------- declaration + + +def test_declaration_builds_a_downward_apply_coordinate_for_one_managed_replica( + synthetic: Synthetic, +) -> None: + decl = declaration_from_dict(declaration_dict(synthetic)) + assert decl.source_url == str(synthetic.remote) + assert decl.branch == "main" + assert decl.kind is DestinationKind.REPLICA + assert decl.destination().kind is DestinationKind.REPLICA + assert decl.destination().destination_id == "config-main-replica" + assert decl.coordinate.destination == "config-main-replica" + assert decl.coordinate.direction is Direction.DOWN + assert decl.coordinate.operation is Operation.APPLY + assert decl.coordinate.source == "synthetic/source" + assert decl.coordinate.layer == "config" + assert decl.coordinate.artifact_class == "config-tree" + assert decl.interval_seconds == 7.5 + assert decl.policy().allowed_directions == frozenset({Direction.DOWN}) + assert decl.receipts_dir == decl.state_dir / "receipts" + # the default git environment is isolated from the host; "inherit" is the opt-in + assert decl.git_env_mode == "isolated" + assert decl.git_env["GIT_CONFIG_GLOBAL"] == os.devnull + assert decl.git_env["GIT_CONFIG_NOSYSTEM"] == "1" + inherit = declaration_from_dict(declaration_dict(synthetic, git_env="inherit")) + assert inherit.git_env == {} + + +def test_declaration_defaults_interval_and_policy_hash_when_absent(synthetic: Synthetic) -> None: + data = declaration_dict(synthetic) + del data["interval_seconds"] + decl = declaration_from_dict(data) + assert decl.interval_seconds == 30.0 + assert decl.policy_hash == "prototype-1-downward-only" + + +@pytest.mark.parametrize( + "missing", + ["source_url", "branch", "destination_id", "destination_path", "state_dir", "outbox_root"], +) +def test_declaration_refuses_a_missing_field_by_name(synthetic: Synthetic, missing: str) -> None: + data = declaration_dict(synthetic) + del data[missing] + with pytest.raises(DeclarationError, match=missing): + declaration_from_dict(data) + + +def test_declaration_refuses_a_missing_or_incomplete_coordinate(synthetic: Synthetic) -> None: + data = declaration_dict(synthetic) + del data["coordinate"] + with pytest.raises(DeclarationError, match="coordinate"): + declaration_from_dict(data) + with pytest.raises(DeclarationError, match="coordinate must be an object"): + declaration_from_dict(declaration_dict(synthetic, coordinate="synthetic/source")) + partial = {"source": "synthetic/source", "layer": "config"} + with pytest.raises(DeclarationError, match="artifact_class"): + declaration_from_dict(declaration_dict(synthetic, coordinate=partial)) + + +@pytest.mark.parametrize("kind", ["authoring", "mirror", ""]) +def test_declaration_refuses_any_kind_but_replica_before_any_git_call( + synthetic: Synthetic, kind: str, monkeypatch: pytest.MonkeyPatch +) -> None: + # if the refusal happened after a git call, this would raise something else first + monkeypatch.setattr( + subprocess, "run", lambda *a, **k: pytest.fail("git was invoked for a refused kind") + ) + with pytest.raises(DeclarationError, match="managed replicas only"): + declaration_from_dict(declaration_dict(synthetic, kind=kind)) + + +def test_declaration_accepts_the_replica_kind_spelled_out(synthetic: Synthetic) -> None: + decl = declaration_from_dict(declaration_dict(synthetic, kind="replica")) + assert decl.kind is DestinationKind.REPLICA + + +def test_declaration_refuses_an_unknown_git_environment(synthetic: Synthetic) -> None: + with pytest.raises(DeclarationError, match="git_env must be"): + declaration_from_dict(declaration_dict(synthetic, git_env="host")) + + +@pytest.mark.parametrize("interval", [0, -1, -0.5]) +def test_declaration_refuses_a_non_positive_interval(synthetic: Synthetic, interval: float) -> None: + with pytest.raises(DeclarationError, match="interval_seconds must be positive"): + declaration_from_dict(declaration_dict(synthetic, interval_seconds=interval)) + + +def test_load_declaration_reads_a_json_object_and_refuses_anything_else( + synthetic: Synthetic, tmp_path: Path +) -> None: + path = tmp_path / "declaration.json" + path.write_text(json.dumps(declaration_dict(synthetic))) + decl = load_declaration(path) + assert decl.destination_id == "config-main-replica" + assert decl.destination_path == synthetic.root / "replicas" / "config-main" + path.write_text(json.dumps([declaration_dict(synthetic)])) + with pytest.raises(DeclarationError, match="JSON object"): + load_declaration(path) + + +def test_make_propagator_passes_the_source_url_through_as_the_exact_string( + synthetic: Synthetic, +) -> None: + # a Path would collapse the "//" of a URL scheme into "/" and the source would + # silently become a relative directory; the machine only ever stringifies it + url = "https://example.invalid/org/config.git" + decl = declaration_from_dict(declaration_dict(synthetic, source_url=url)) + propagator = make_propagator(decl) + assert str(propagator.source_remote) == url + assert "//" in str(propagator.source_remote) + assert propagator.branch == "main" + assert propagator.state_dir == decl.state_dir + assert propagator.policy == decl.policy() + assert propagator.git_env == decl.git_env + + +# ----------------------------------------------------------------------- ensure_replica + + +def test_ensure_replica_clones_the_declared_branch_single_branch_when_absent( + synthetic: Synthetic, declaration: Declaration +) -> None: + # a second branch on the source makes --single-branch load-bearing: without it the + # clone would carry refs/remotes/origin/scratch too, and the assertion below would fail + git(synthetic.author, "push", "-q", "origin", "main:refs/heads/scratch") + assert not declaration.destination_path.exists() + path = ensure_replica(declaration) + assert path == declaration.destination_path + assert git(path, "remote", "get-url", "origin") == str(synthetic.remote) + assert git(path, "rev-parse", "--abbrev-ref", "HEAD") == "main" + assert git(path, "rev-parse", "HEAD") == synthetic.base + assert (path / "canon.md").read_text() == "canon v1\n" + tracking = git(path, "for-each-ref", "--format=%(refname)", "refs/remotes/").splitlines() + assert tracking == ["refs/remotes/origin/main"] + + +def test_ensure_replica_accepts_the_path_it_declared_and_leaves_it_untouched( + declaration: Declaration, +) -> None: + ensure_replica(declaration) + before = snapshot(declaration.destination_path) + assert ensure_replica(declaration) == declaration.destination_path + assert snapshot(declaration.destination_path) == before + + +def test_ensure_replica_refuses_a_checkout_whose_origin_is_not_the_declared_source( + synthetic: Synthetic, declaration: Declaration +) -> None: + other, _, _ = _bare_with_one_commit(synthetic.root, "other", "other v1\n") + declaration.destination_path.parent.mkdir(parents=True) + _run("git", "clone", "-q", str(other), str(declaration.destination_path)) + before = snapshot(declaration.destination_path) + with pytest.raises(DeclarationMismatch, match="has origin"): + ensure_replica(declaration) + assert snapshot(declaration.destination_path) == before + + +def test_ensure_replica_refuses_a_checkout_on_another_branch( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + git(declaration.destination_path, "switch", "-q", "-c", "scratch") + before = snapshot(declaration.destination_path) + with pytest.raises(DeclarationMismatch, match="is on 'scratch'"): + ensure_replica(declaration) + assert snapshot(declaration.destination_path) == before + + +def test_ensure_replica_refuses_a_path_that_is_not_a_git_checkout( + declaration: Declaration, +) -> None: + declaration.destination_path.mkdir(parents=True) + (declaration.destination_path / "canon.md").write_text("not a clone\n") + with pytest.raises(DeclarationMismatch, match="not a git checkout"): + ensure_replica(declaration) + assert (declaration.destination_path / "canon.md").read_text() == "not a clone\n" + assert not (declaration.destination_path / ".git").exists() + + +# ----------------------------------------------------------------------- ticks + + +def test_first_tick_on_a_fresh_replica_acknowledges_without_running_the_verb( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + propagator = make_propagator(declaration) + result = tick(declaration, propagator) + assert result.was_operation + assert result.cursor_before is None + assert result.source_rev == synthetic.base + receipt = result.receipt + assert receipt is not None + assert receipt.state is State.ACKNOWLEDGED + assert receipt.attempt == 1 + assert receipt.replayed is False + assert receipt.source_rev == synthetic.base + assert receipt.expected_base == synthetic.base + assert receipt.after == synthetic.base + # the replica was born at the source revision: the read-back established it, no verb + assert applied_observation(receipt)["verb_ran_now"] is False + assert git(declaration.destination_path, "rev-parse", "HEAD") == synthetic.base + # one receipt file, one outbox event, one summary carried by both + files = receipt_files(declaration) + assert files == [result.receipt_path] + assert files[0].name.endswith(f"-{receipt.pending_id[:12]}-acknowledged.json") + events = outbox_events(declaration.outbox_root) + assert [e["type"] for e in events] == [EventType.PROPAGATION_RECEIPT.value] + event = events[0] + assert event["actor"] == ACTOR + assert event["owner_unit"] == declaration.destination_id + assert event["summary"] == result.summary + assert event["state"] == "acknowledged" + assert event["pending_id"] == receipt.pending_id + assert event["operation_id"] == receipt.operation_id + assert event["source_rev"] == synthetic.base + assert event["expected_base"] == synthetic.base + assert event["after"] == synthetic.base + assert event["replayed"] is False + assert event["receipt_path"] == str(result.receipt_path) + assert result.summary.startswith("propagation acknowledged: config-main-replica at ") + assert "attempt 1" in result.summary + + +def test_a_tick_with_no_new_revision_is_not_an_operation_and_writes_nothing( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + propagator = make_propagator(declaration) + first = tick(declaration, propagator) + assert first.was_operation + files_before = receipt_files(declaration) + events_before = outbox_events(declaration.outbox_root) + before = snapshot(declaration.destination_path) + second = tick(declaration, propagator) + assert not second.was_operation + assert second.receipt is None + assert second.receipt_path is None + assert second.latency_seconds is None + assert second.cursor_before == synthetic.base + assert second.source_rev == synthetic.base + assert "not an operation" in second.summary + assert "cursor already at" in second.summary + assert receipt_files(declaration) == files_before + assert outbox_events(declaration.outbox_root) == events_before + assert snapshot(declaration.destination_path) == before + + +def test_a_pushed_change_is_applied_on_the_next_tick_with_exact_revisions( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + propagator = make_propagator(declaration) + tick(declaration, propagator) + new = synthetic.push_change("canon v2\n") + assert new != synthetic.base + result = tick(declaration, propagator) + assert result.was_operation + assert result.cursor_before == synthetic.base + assert result.source_rev == new + receipt = result.receipt + assert receipt is not None + assert receipt.state is State.ACKNOWLEDGED + assert receipt.attempt == 1 + assert receipt.replayed is False + assert receipt.source_rev == new + assert receipt.expected_base == synthetic.base + assert receipt.after == new + assert applied_observation(receipt)["verb_ran_now"] is True + assert git(declaration.destination_path, "rev-parse", "HEAD") == new + assert (declaration.destination_path / "canon.md").read_text() == "canon v2\n" + assert git(declaration.destination_path, "status", "--porcelain") == "" + # exact revisions in the one line a channel reader sees + assert f"at {synthetic.base[:12]} -> {new[:12]}, intended {new[:12]}" in result.summary + assert "(source synthetic/source, attempt 1, total " in result.summary + # a second receipt file and a second outbox event, each naming the new revision + files = receipt_files(declaration) + assert len(files) == 2 + assert result.receipt_path in files + events = outbox_events(declaration.outbox_root) + assert len(events) == 2 + assert events[1]["source_rev"] == new + assert events[1]["expected_base"] == synthetic.base + assert events[1]["after"] == new + assert events[1]["state"] == "acknowledged" + # and the tick after that is current again: nothing written, nothing emitted + third = tick(declaration, propagator) + assert not third.was_operation + assert third.cursor_before == new + assert receipt_files(declaration) == files + assert outbox_events(declaration.outbox_root) == events + + +def test_written_receipt_round_trips_and_its_latency_is_recomputable_from_itself( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + propagator = make_propagator(declaration) + tick(declaration, propagator) + new = synthetic.push_change("canon v2\n") + result = tick(declaration, propagator) + assert result.receipt is not None and result.receipt_path is not None + payload = json.loads(result.receipt_path.read_text()) + assert set(payload) == {"daemon", "declaration", "observed_at", "latency_seconds", "receipt"} + assert payload["daemon"] == ACTOR + assert payload["declaration"] == { + "source_url": str(synthetic.remote), + "branch": "main", + "destination_id": "config-main-replica", + "destination_path": str(declaration.destination_path), + } + assert payload["observed_at"] == result.observed_at + assert payload["receipt"] == result.receipt.as_dict() + assert payload["receipt"]["state"] == "acknowledged" + assert payload["receipt"]["source_rev"] == new + assert payload["receipt"]["after"] == new + # the latency the daemon reports is the latency the module derives from the receipt + assert payload["latency_seconds"] == result.latency_seconds + assert result.latency_seconds == latency_from_receipt(result.receipt) + # and it is recomputable by a reader who has ONLY the written file + transitions = payload["receipt"]["transitions"] + states = [t["state"] for t in transitions] + assert states == ["observed", "fetched", "planned", "applied", "verified", "acknowledged"] + stamps = [datetime.fromisoformat(t["timestamp"]) for t in transitions] + recomputed = { + f"{a['state']}->{b['state']}": round((tb - ta).total_seconds(), 6) + for (a, ta), (b, tb) in zip( + zip(transitions, stamps, strict=True), + zip(transitions[1:], stamps[1:], strict=True), + strict=False, + ) + } + recomputed["total"] = round((stamps[-1] - stamps[0]).total_seconds(), 6) + assert payload["latency_seconds"] == recomputed + assert set(recomputed) == { + "observed->fetched", + "fetched->planned", + "planned->applied", + "applied->verified", + "verified->acknowledged", + "total", + } + assert all(value >= 0 for value in recomputed.values()) + assert recomputed["total"] == pytest.approx( + sum(v for k, v in recomputed.items() if k != "total"), abs=1e-5 + ) + + +def test_a_dirty_replica_is_refused_with_a_receipt_left_untouched_then_applies_once_clean( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + propagator = make_propagator(declaration) + tick(declaration, propagator) + (declaration.destination_path / "canon.md").write_text("local edit on the replica\n") + new = synthetic.push_change("canon v2\n") + before = snapshot(declaration.destination_path) + refused = tick(declaration, propagator) + assert refused.was_operation + receipt = refused.receipt + assert receipt is not None + assert receipt.state is State.REFUSED + assert receipt.attempt == 1 + assert receipt.after is None + assert receipt.refusal_reason is not None + assert receipt.refusal_reason.startswith("destination.clean:") + assert "refused: destination.clean:" in refused.summary + assert "-> unchanged" in refused.summary + # refused means untouched: refs, HEAD, worktree bytes, reflog all as before + assert snapshot(declaration.destination_path) == before + # and a refusal is a receipt too: written and announced + files = receipt_files(declaration) + assert refused.receipt_path in files + assert refused.receipt_path.name.endswith("-refused.json") + events = outbox_events(declaration.outbox_root) + assert events[-1]["state"] == "refused" + assert events[-1]["after"] is None + assert events[-1]["source_rev"] == new + # clean up, and the next tick is a NEW attempt at the same revision, not a replay + git(declaration.destination_path, "checkout", "--", "canon.md") + applied = tick(declaration, propagator) + assert applied.receipt is not None + assert applied.receipt.state is State.ACKNOWLEDGED + assert applied.receipt.attempt == 2 + assert applied.receipt.replayed is False + assert applied.receipt.after == new + assert git(declaration.destination_path, "rev-parse", "HEAD") == new + assert "attempt 2" in applied.summary + assert len(receipt_files(declaration)) == len(files) + 1 + assert outbox_events(declaration.outbox_root)[-1]["state"] == "acknowledged" + + +def test_tick_writes_and_announces_nothing_when_the_machine_returns_none( + declaration: Declaration, monkeypatch: pytest.MonkeyPatch +) -> None: + # the only way to reach this branch is a race (the source moved back to the cursor + # between observe and run), so the machine's answer is forced here; the claim under + # test is the daemon's: no receipt, no file, no event + ensure_replica(declaration) + propagator = make_propagator(declaration) + monkeypatch.setattr(propagator, "run", lambda coordinate, destination: None) + result = tick(declaration, propagator) + assert not result.was_operation + assert result.receipt_path is None + assert "source returned to the cursor" in result.summary + assert receipt_files(declaration) == [] + assert outbox_events(declaration.outbox_root) == [] + + +def test_summarize_names_state_destination_revisions_attempt_and_total( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + propagator = make_propagator(declaration) + tick(declaration, propagator) + new = synthetic.push_change("canon v2\n") + receipt = tick(declaration, propagator).receipt + assert receipt is not None + line = summarize(receipt, {"total": 1.23456}) + assert line == ( + f"propagation acknowledged: config-main-replica at {synthetic.base[:12]} -> {new[:12]}, " + f"intended {new[:12]} (source synthetic/source, attempt 1, total 1.235s)" + ) + # a replayed receipt says so, and a missing total reads as 0.000s rather than failing + from dataclasses import replace + + replayed = replace(receipt, replayed=True) + assert summarize(replayed, {}).endswith( + "(source synthetic/source, attempt 1, replayed, total 0.000s)" + ) + + +# ----------------------------------------------------------------------- loop + CLI + + +def test_run_loop_once_ensures_the_replica_ticks_once_and_prints_exactly_one_line( + synthetic: Synthetic, declaration: Declaration +) -> None: + out = io.StringIO() + assert not declaration.destination_path.exists() + stats = run_loop( + declaration, once=True, sleep=lambda s: pytest.fail("slept under once"), out=out + ) + assert declaration.destination_path.exists() + assert stats.ticks == 1 + assert stats.operations == 1 + assert stats.by_state == {"acknowledged": 1} + assert stats.last is not None and stats.last.was_operation + lines = out.getvalue().splitlines() + assert len(lines) == 1 + assert lines[0] == f"{stats.last.observed_at} {stats.last.summary}" + + +def test_run_loop_stops_when_told_and_sleeps_the_declared_interval_between_ticks( + synthetic: Synthetic, declaration: Declaration +) -> None: + out = io.StringIO() + slept: list[float] = [] + ticks_seen = {"n": 0} + + def stop() -> bool: + ticks_seen["n"] += 1 + return ticks_seen["n"] >= 3 + + stats = run_loop(declaration, stop=stop, sleep=slept.append, out=out) + assert stats.ticks == 3 + assert stats.operations == 1 # the first tick; the next two found the cursor current + assert stats.by_state == {"acknowledged": 1} + assert slept == [7.5, 7.5] + lines = out.getvalue().splitlines() + assert len(lines) == 3 + assert "propagation acknowledged" in lines[0] + assert all("not an operation" in line for line in lines[1:]) + + +def test_main_once_runs_a_single_tick_from_a_declaration_file( + synthetic: Synthetic, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + path = tmp_path / "declaration.json" + path.write_text(json.dumps(declaration_dict(synthetic))) + decl = load_declaration(path) + assert main(["--declaration", str(path), "--once"]) == 0 + captured = capsys.readouterr() + lines = captured.out.splitlines() + assert len(lines) == 1 + assert "propagation acknowledged: config-main-replica" in lines[0] + assert len(receipt_files(decl)) == 1 + assert [e["type"] for e in outbox_events(decl.outbox_root)] == ["propagation.receipt"] + + +def test_main_interval_override_replaces_the_declared_interval( + synthetic: Synthetic, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "declaration.json" + path.write_text(json.dumps(declaration_dict(synthetic))) + seen: dict[str, object] = {} + + def fake_run_loop(declaration: Declaration, *, once: bool = False, **_: object): + seen["interval"] = declaration.interval_seconds + seen["once"] = once + return None + + import gr2.prototypes.propagation_daemon as daemon + + monkeypatch.setattr(daemon, "run_loop", fake_run_loop) + assert main(["--declaration", str(path), "--once", "--interval", "2.5"]) == 0 + assert seen == {"interval": 2.5, "once": True} + assert main(["--declaration", str(path)]) == 0 + assert seen == {"interval": 7.5, "once": False} From 47352e2b0cf5273b75e9bb7ce472be1f293c7dbe Mon Sep 17 00:00:00 2001 From: Atlas Date: Wed, 19 Aug 2026 11:45:31 -0500 Subject: [PATCH 10/29] fix(link): verify gripspace pins before applying --- src/cli/commands/link.rs | 98 ++++- src/cli/dispatch.rs | 40 +- src/core/gripspace.rs | 104 ++++- tests/link_apply_detached_freshness.rs | 514 +++++++++++++++++++++++++ 4 files changed, 728 insertions(+), 28 deletions(-) create mode 100644 tests/link_apply_detached_freshness.rs diff --git a/src/cli/commands/link.rs b/src/cli/commands/link.rs index 9927c26..90957fc 100644 --- a/src/cli/commands/link.rs +++ b/src/cli/commands/link.rs @@ -12,6 +12,7 @@ use crate::files::{process_composefiles, resolve_file_source}; use crate::git::{fetch_remote, path_exists}; use std::collections::BTreeSet; use std::path::{Path, PathBuf}; +use std::process::Command; /// Check if a source path contains glob characters (`*`, `?`, `[`). fn is_glob_pattern(src: &str) -> bool { @@ -176,13 +177,62 @@ fn referenced_gripspaces(manifest: &Manifest) -> BTreeSet { names } -/// Refuse a manual apply when a branch-backed gripspace is behind upstream. +/// Resolve a named tag from the remote itself, without trusting the local tag. +/// +/// A normal fetch deliberately refuses to move an existing local tag. Using +/// `revparse_single(rev)` after that fetch therefore compares HEAD with the +/// same stale local ref and can certify old content as current. `ls-remote` +/// observes the upstream ref directly and handles both lightweight and +/// annotated tags; the peeled commit wins when both rows are present. +fn remote_tag_commit(repo: &git2::Repository, rev: &str) -> anyhow::Result> { + let workdir = repo + .workdir() + .ok_or_else(|| anyhow::anyhow!("Gripspace source has no working directory"))?; + let tag_ref = format!("refs/tags/{rev}"); + let peeled_ref = format!("{tag_ref}^{{}}"); + let output = Command::new("git") + .args(["ls-remote", "--tags", "origin", &tag_ref, &peeled_ref]) + .current_dir(workdir) + .output() + .map_err(|error| anyhow::anyhow!("Cannot inspect origin tag '{rev}': {error}"))?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "Cannot inspect origin tag '{}': {}", + rev, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + + let mut direct = None; + let mut peeled = None; + for line in String::from_utf8_lossy(&output.stdout).lines() { + let Some((oid, reference)) = line.split_once(char::is_whitespace) else { + continue; + }; + let parsed = git2::Oid::from_str(oid.trim()).map_err(|error| { + anyhow::anyhow!("Origin returned an invalid object id for tag '{rev}': {error}") + })?; + match reference.trim() { + value if value == peeled_ref => peeled = Some(parsed), + value if value == tag_ref => direct = Some(parsed), + _ => {} + } + } + Ok(peeled.or(direct)) +} + +fn is_full_commit_id(rev: &str) -> bool { + rev.len() == 40 && rev.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +/// Refuse a manual apply when a gripspace is not at its configured revision. /// /// A successful local composition proves that the files are internally /// usable. It does not prove that the source clone is current. Fetch first, -/// then compare graph position. A detached HEAD is accepted only when the -/// materializer recorded an explicit tag or commit revision and HEAD still -/// resolves to that pin. Detachment by itself is not evidence of a pin. +/// then compare both branch identity and graph position. An attached HEAD must +/// still be on the configured branch. A detached HEAD is accepted only when +/// the materializer recorded an explicit tag or commit revision and HEAD still +/// resolves to that pin. Neither attachment nor detachment proves provenance. fn ensure_gripspace_sources_current( workspace_root: &Path, manifest: &Manifest, @@ -228,15 +278,23 @@ fn ensure_gripspace_sources_current( .into()); } - let pinned_oid = repo - .revparse_single(&rev) - .and_then(|object| object.peel_to_commit()) - .map(|commit| commit.id()) - .map_err(|error| { - CliOutcomeError::refusal(format!( - "Cannot verify gripspace source '{name}' at configured revision '{rev}': {error}" - )) - })?; + let pinned_oid = if let Some(tag_oid) = remote_tag_commit(&repo, &rev)? { + tag_oid + } else if is_full_commit_id(&rev) { + repo.revparse_single(&rev) + .and_then(|object| object.peel_to_commit()) + .map(|commit| commit.id()) + .map_err(|error| { + CliOutcomeError::refusal(format!( + "Cannot verify gripspace source '{name}' at configured commit '{rev}': {error}" + )) + })? + } else { + return Err(CliOutcomeError::refusal(format!( + "Cannot verify gripspace source '{name}' at configured revision '{rev}' against origin. Run `gr sync` before `gr link --apply`." + )) + .into()); + }; if pinned_oid != local_oid { return Err(CliOutcomeError::refusal(format!( "Gripspace source '{name}' does not match configured revision '{rev}'. Run `gr sync` before `gr link --apply`." @@ -251,6 +309,20 @@ fn ensure_gripspace_sources_current( .ok_or_else(|| anyhow::anyhow!("Cannot identify branch for gripspace source '{name}'"))? .to_string(); + let requested_rev = requested_gripspace_revision(&path).map_err(|error| { + CliOutcomeError::refusal(format!( + "Cannot verify gripspace source '{name}' on branch '{branch}': {error}" + )) + })?; + if let Some(rev) = requested_rev { + if rev != branch { + return Err(CliOutcomeError::refusal(format!( + "Gripspace source '{name}' is on branch '{branch}', configured revision '{rev}'. Run `gr sync` before `gr link --apply`." + )) + .into()); + } + } + let remote_ref = format!("refs/remotes/origin/{branch}"); let remote_oid = repo .find_reference(&remote_ref) diff --git a/src/cli/dispatch.rs b/src/cli/dispatch.rs index 3cc3b28..e1e851b 100644 --- a/src/cli/dispatch.rs +++ b/src/cli/dispatch.rs @@ -596,7 +596,11 @@ pub async fn dispatch_command( .await?; } Some(Commands::Link { status, apply }) => { - let ctx = load_workspace_context(quiet, verbose, json)?; + // Manual apply is a verification path, not a materialization path. + // Resolve included manifests from the clones exactly as they sit so + // a detached source cannot be silently reattached before link's + // freshness guard observes it. Status retains the historical loader. + let ctx = load_workspace_context_with_materialization(quiet, verbose, json, !apply)?; crate::cli::commands::link::run_link( &ctx.workspace_root, &ctx.manifest, @@ -1038,7 +1042,9 @@ pub async fn dispatch_command( /// `.griptree` many levels up eclipse a `.gitgrip` or checkout one level /// down, because the griptree pass never stopped climbing to give the nearer /// marker a chance). -fn load_gripspace() -> anyhow::Result<(std::path::PathBuf, crate::core::manifest::Manifest)> { +fn load_gripspace( + materialize_gripspaces: bool, +) -> anyhow::Result<(std::path::PathBuf, crate::core::manifest::Manifest)> { let current = std::env::current_dir()?; let mut search_path = current; loop { @@ -1047,7 +1053,7 @@ fn load_gripspace() -> anyhow::Result<(std::path::PathBuf, crate::core::manifest if let Ok(pointer) = crate::core::griptree::GriptreePointer::load(&griptree_pointer_path) { - return load_from_griptree(&search_path, &pointer); + return load_from_griptree(&search_path, &pointer, materialize_gripspaces); } } @@ -1079,7 +1085,7 @@ fn load_gripspace() -> anyhow::Result<(std::path::PathBuf, crate::core::manifest { let content = std::fs::read_to_string(&manifest_path)?; let mut manifest = crate::core::manifest::Manifest::parse(&content)?; - resolve_gripspace_includes(&mut manifest, &search_path); + resolve_gripspace_includes(&mut manifest, &search_path, materialize_gripspaces)?; return Ok((search_path, manifest)); } } @@ -1089,7 +1095,7 @@ fn load_gripspace() -> anyhow::Result<(std::path::PathBuf, crate::core::manifest { let content = std::fs::read_to_string(repo_yaml)?; let mut manifest = crate::core::manifest::Manifest::parse(&content)?; - resolve_gripspace_includes(&mut manifest, &search_path); + resolve_gripspace_includes(&mut manifest, &search_path, materialize_gripspaces)?; return Ok((search_path, manifest)); } @@ -1117,7 +1123,16 @@ fn load_workspace_context( verbose: bool, json: bool, ) -> anyhow::Result { - let (workspace_root, manifest) = load_gripspace()?; + load_workspace_context_with_materialization(quiet, verbose, json, true) +} + +fn load_workspace_context_with_materialization( + quiet: bool, + verbose: bool, + json: bool, + materialize_gripspaces: bool, +) -> anyhow::Result { + let (workspace_root, manifest) = load_gripspace(materialize_gripspaces)?; Ok(WorkspaceContext::new( workspace_root, manifest, @@ -1131,6 +1146,7 @@ fn load_workspace_context( fn load_from_griptree( griptree_path: &std::path::Path, pointer: &crate::core::griptree::GriptreePointer, + materialize_gripspaces: bool, ) -> anyhow::Result<(std::path::PathBuf, crate::core::manifest::Manifest)> { let griptree_manifest_path = crate::core::manifest_paths::resolve_gripspace_manifest_path(griptree_path); @@ -1152,7 +1168,7 @@ fn load_from_griptree( }; let mut manifest = crate::core::manifest::Manifest::parse(&content)?; - resolve_gripspace_includes(&mut manifest, griptree_path); + resolve_gripspace_includes(&mut manifest, griptree_path, materialize_gripspaces)?; Ok((griptree_path.to_path_buf(), manifest)) } @@ -1160,9 +1176,15 @@ fn load_from_griptree( fn resolve_gripspace_includes( manifest: &mut crate::core::manifest::Manifest, workspace_root: &std::path::Path, -) { + materialize_gripspaces: bool, +) -> anyhow::Result<()> { let spaces_dir = crate::core::manifest_paths::spaces_dir(workspace_root); if spaces_dir.exists() { - let _ = crate::core::gripspace::resolve_all_gripspaces(manifest, &spaces_dir); + if materialize_gripspaces { + let _ = crate::core::gripspace::resolve_all_gripspaces(manifest, &spaces_dir); + } else { + crate::core::gripspace::resolve_existing_gripspaces(manifest, &spaces_dir)?; + } } + Ok(()) } diff --git a/src/core/gripspace.rs b/src/core/gripspace.rs index 1638e4a..786ab8d 100644 --- a/src/core/gripspace.rs +++ b/src/core/gripspace.rs @@ -386,6 +386,55 @@ pub fn ensure_gripspace( Ok(gripspace_path) } +/// Locate an already-materialized gripspace without changing its checkout. +/// +/// Manual link application must inspect the source state the operator actually +/// has. Calling [`ensure_gripspace`] while loading that command can run +/// `checkout -B` and erase a detached-HEAD premise before the freshness guard +/// sees it. Prefer the recorded URL + revision identity. If provenance is +/// missing, a unique same-remote clone is still returned so the guard can +/// refuse it explicitly rather than resolution hiding the source altogether. +fn existing_gripspace( + spaces_dir: &Path, + config: &GripspaceConfig, +) -> Result { + let entries = std::fs::read_dir(spaces_dir).map_err(|error| { + ManifestError::GripspaceError(format!( + "Cannot inspect materialized gripspaces at '{}': {error}. Run `gr sync` before applying links.", + spaces_dir.display() + )) + })?; + + let mut same_remote = Vec::new(); + let mut exact = Vec::new(); + for entry in entries { + let path = entry + .map_err(|error| ManifestError::GripspaceError(error.to_string()))? + .path(); + if !path.is_dir() || !is_same_remote(&path, &config.url) { + continue; + } + let recorded_matches = match (requested_gripspace_revision(&path), config.rev.as_deref()) { + (Ok(Some(recorded)), Some(requested)) => recorded == requested, + (Ok(None), None) => true, + _ => false, + }; + if recorded_matches { + exact.push(path.clone()); + } + same_remote.push(path); + } + + match (exact.as_slice(), same_remote.as_slice()) { + ([path], _) => Ok(path.clone()), + ([], [path]) => Ok(path.clone()), + _ => Err(ManifestError::GripspaceError(format!( + "Cannot identify one existing gripspace for '{}'. Run `gr sync` before applying links.", + config.url + ))), + } +} + /// Update a gripspace by fetching and pulling latest. pub fn update_gripspace( gripspace_path: &Path, @@ -400,7 +449,11 @@ pub fn update_gripspace( // Fetch from origin let output = Command::new("git") - .args(["fetch", "origin"]) + // Managed gripspace clones treat the manifest's named tag as upstream + // authority. A plain fetch leaves a moved local tag untouched, so the + // subsequent checkout resolves the stale tag and `gr sync` cannot + // satisfy the recovery path named by the link freshness refusal. + .args(["fetch", "--force", "--tags", "origin"]) .current_dir(gripspace_path) .output() .map_err(|e| ManifestError::GripspaceError(format!("Failed to fetch gripspace: {}", e)))?; @@ -626,6 +679,26 @@ fn deep_merge_repo_config( pub fn resolve_all_gripspaces( manifest: &mut Manifest, spaces_dir: &Path, +) -> Result<(), ManifestError> { + resolve_all_gripspaces_with_materialization(manifest, spaces_dir, true) +} + +/// Resolve included manifests from existing clones without changing HEAD. +/// +/// This is the manual-link preflight form. It deliberately refuses a missing +/// or ambiguous clone instead of cloning, fetching, checking out, or recording +/// revision metadata while merely deciding whether an apply is safe. +pub fn resolve_existing_gripspaces( + manifest: &mut Manifest, + spaces_dir: &Path, +) -> Result<(), ManifestError> { + resolve_all_gripspaces_with_materialization(manifest, spaces_dir, false) +} + +fn resolve_all_gripspaces_with_materialization( + manifest: &mut Manifest, + spaces_dir: &Path, + materialize: bool, ) -> Result<(), ManifestError> { let gripspaces = match manifest.gripspaces.take() { Some(gs) if !gs.is_empty() => gs, @@ -652,6 +725,7 @@ pub fn resolve_all_gripspaces( &mut active_stack, &mut resolved, 0, + materialize, &mut merged_repos, &mut merged_scripts, &mut merged_env, @@ -866,6 +940,7 @@ fn resolve_gripspace_recursive( active_stack: &mut HashSet, resolved: &mut HashSet, depth: usize, + materialize: bool, merged_repos: &mut HashMap, merged_scripts: &mut HashMap, merged_env: &mut HashMap, @@ -901,10 +976,24 @@ fn resolve_gripspace_recursive( ))); } - let gripspace_path = ensure_gripspace(spaces_dir, config)?; - // Resolve the actual directory name (may differ from `name` due to reserved - // name suffixing or multi-rev disambiguation) - let dir_name = resolve_space_name(&config.url, config.rev.as_deref(), spaces_dir)?; + let gripspace_path = if materialize { + ensure_gripspace(spaces_dir, config)? + } else { + existing_gripspace(spaces_dir, config)? + }; + // Use the directory we actually opened. In read-only mode a detached source + // may no longer satisfy revision-derived allocation heuristics, and asking + // those heuristics again would select a different path and erase the premise. + let dir_name = gripspace_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + ManifestError::GripspaceError(format!( + "Cannot identify gripspace directory for '{}'", + gripspace_path.display() + )) + })? + .to_string(); // Load the gripspace's manifest let Some(manifest_path) = manifest_paths::resolve_manifest_file_in_dir(&gripspace_path) else { @@ -933,7 +1022,9 @@ fn resolve_gripspace_recursive( if let Some(ref nested_gripspaces) = gs_manifest.gripspaces { for nested_config in nested_gripspaces { // Clone the nested gripspace if it doesn't exist yet - ensure_gripspace(spaces_dir, nested_config)?; + if materialize { + ensure_gripspace(spaces_dir, nested_config)?; + } resolve_gripspace_recursive( nested_config, @@ -941,6 +1032,7 @@ fn resolve_gripspace_recursive( active_stack, resolved, depth + 1, + materialize, merged_repos, merged_scripts, merged_env, diff --git a/tests/link_apply_detached_freshness.rs b/tests/link_apply_detached_freshness.rs new file mode 100644 index 0000000..e244f19 --- /dev/null +++ b/tests/link_apply_detached_freshness.rs @@ -0,0 +1,514 @@ +//! End-to-end witnesses for detached gripspace freshness through the `gr` binary. +//! +//! These start above manifest resolution because that caller used to erase the +//! detached premise before the freshness guard could inspect it. + +use assert_cmd::Command; +use std::path::{Path, PathBuf}; +use std::process::Command as StdCommand; +use tempfile::TempDir; + +fn git(dir: &Path, args: &[&str]) -> String { + let out = StdCommand::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {:?} in {}: {}", + args, + dir.display(), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn repo(root: &Path, name: &str, files: &[(&str, &str)]) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).unwrap(); + git(&dir, &["init", "-q", "-b", "main"]); + git(&dir, &["config", "user.email", "t@e.com"]); + git(&dir, &["config", "user.name", "t"]); + for (p, b) in files { + let full = dir.join(p); + if let Some(par) = full.parent() { + std::fs::create_dir_all(par).unwrap(); + } + std::fs::write(full, b).unwrap(); + } + git(&dir, &["add", "-A"]); + git(&dir, &["commit", "-qm", "initial"]); + dir +} + +struct Fx { + _t: TempDir, + source: PathBuf, + workspace: PathBuf, + space: PathBuf, +} + +/// Build a workspace whose gripspace source is materialized at `rev`. +fn setup(rev: &str) -> Fx { + let t = TempDir::new().unwrap(); + let root = t.path().to_path_buf(); + let source = repo( + &root, + "source-space", + &[ + ("SECTION.md", "v1\n"), + ("gripspace.yml", "version: 2\nrepos: {}\n"), + ], + ); + let dummy = repo(&root, "dummy-repo", &[("README.md", "d\n")]); + let manifest = repo( + &root, + "workspace-manifest", + &[( + "gripspace.yml", + &format!( + r#"version: 2 +gripspaces: + - url: "{}" + rev: {} +manifest: + url: "{}" + revision: main + composefile: + - dest: OUT.md + parts: + - gripspace: source-space + src: SECTION.md +repos: + dummy-repo: + url: "{}" + path: ./dummy-repo + revision: main +"#, + source.display(), + rev, + root.join("workspace-manifest").display(), + dummy.display() + ), + )], + ); + + let workspace = root.join("workspace"); + let init = Command::cargo_bin("gr") + .unwrap() + .args([ + "init", + manifest.to_str().unwrap(), + "--path", + workspace.to_str().unwrap(), + "--no-interactive", + ]) + .output() + .unwrap(); + assert!( + init.status.success(), + "init: {}{}", + String::from_utf8_lossy(&init.stdout), + String::from_utf8_lossy(&init.stderr) + ); + let sync = Command::cargo_bin("gr") + .unwrap() + .arg("sync") + .current_dir(&workspace) + .output() + .unwrap(); + assert!( + sync.status.success(), + "baseline sync: {}{}", + String::from_utf8_lossy(&sync.stdout), + String::from_utf8_lossy(&sync.stderr) + ); + let space = workspace.join(".gitgrip/spaces/source-space"); + Fx { + _t: t, + source, + workspace, + space, + } +} + +fn apply(ws: &Path) -> (Option, String) { + let o = Command::cargo_bin("gr") + .unwrap() + .args(["link", "--apply"]) + .current_dir(ws) + .output() + .unwrap(); + ( + o.status.code(), + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ), + ) +} + +fn run_sync(ws: &Path) -> (Option, String) { + let o = Command::cargo_bin("gr") + .unwrap() + .arg("sync") + .current_dir(ws) + .output() + .unwrap(); + ( + o.status.code(), + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ), + ) +} + +fn recorded(space: &Path) -> String { + let o = StdCommand::new("git") + .args(["config", "--get", "gitgrip.requestedGripspaceRev"]) + .current_dir(space) + .output() + .unwrap(); + format!( + "exit={:?} value={:?}", + o.status.code(), + String::from_utf8_lossy(&o.stdout).trim() + ) +} + +#[test] +fn link_apply_refuses_an_attached_source_on_the_wrong_configured_branch() { + let fx = setup("main"); + assert_eq!( + std::fs::read_to_string(fx.workspace.join("OUT.md")).unwrap(), + "v1\n", + "precondition: the workspace starts from configured main" + ); + git(&fx.source, &["branch", "scratch", "HEAD"]); + git(&fx.space, &["checkout", "-qb", "scratch"]); + + std::fs::write(fx.source.join("SECTION.md"), "v2\n").unwrap(); + git(&fx.source, &["add", "-A"]); + git(&fx.source, &["commit", "-qm", "advance main"]); + + let (code, diag) = apply(&fx.workspace); + assert_eq!(code, Some(2), "wrong attached branch must refuse: {diag}"); + assert!( + diag.contains("is on branch 'scratch', configured revision 'main'"), + "wrong-branch diagnostic: {diag}" + ); + assert_eq!( + git(&fx.space, &["branch", "--show-current"]), + "scratch", + "refusal must not silently reattach the source" + ); + assert_eq!( + std::fs::read_to_string(fx.workspace.join("OUT.md")).unwrap(), + "v1\n", + "refusal must not compose stale content" + ); +} + +#[test] +fn control_attached_source_on_the_configured_current_branch_is_accepted() { + let fx = setup("main"); + assert_eq!( + git(&fx.space, &["branch", "--show-current"]), + "main", + "precondition: attached to configured main" + ); + + let (code, diag) = apply(&fx.workspace); + assert_eq!(code, Some(0), "current configured branch must pass: {diag}"); + assert_eq!( + std::fs::read_to_string(fx.workspace.join("OUT.md")).unwrap(), + "v1\n" + ); +} + +#[test] +fn link_apply_refuses_a_branch_configured_source_that_is_detached() { + let fx = setup("main"); + eprintln!("[A1] recorded after sync: {}", recorded(&fx.space)); + assert!( + git(&fx.space, &["branch", "--show-current"]) == "main", + "precondition: attached to main" + ); + git(&fx.space, &["checkout", "--detach", "HEAD"]); + assert!( + git(&fx.space, &["branch", "--show-current"]).is_empty(), + "precondition: detached" + ); + let (code, diag) = apply(&fx.workspace); + eprintln!("[A1] exit={:?}\n{}", code, diag); + assert_eq!(code, Some(2), "A1 must refuse"); + assert!( + diag.contains("detached from configured branch"), + "A1 diagnostic: {diag}" + ); + assert!( + git(&fx.space, &["branch", "--show-current"]).is_empty(), + "refusal must not silently reattach the source" + ); +} + +#[test] +fn link_apply_refuses_a_detached_source_without_recorded_provenance() { + let fx = setup("main"); + git( + &fx.space, + &["config", "--unset", "gitgrip.requestedGripspaceRev"], + ); + eprintln!("[A2a] recorded after unset: {}", recorded(&fx.space)); + git(&fx.space, &["checkout", "--detach", "HEAD"]); + let (code, diag) = apply(&fx.workspace); + eprintln!("[A2a] exit={:?}\n{}", code, diag); + assert_eq!(code, Some(2), "A2a must refuse"); + assert!( + diag.contains("provenance is unavailable"), + "A2a diagnostic: {diag}" + ); +} + +#[test] +fn link_apply_refuses_a_detached_source_with_an_unresolvable_named_revision() { + let fx = setup("main"); + git( + &fx.space, + &[ + "config", + "gitgrip.requestedGripspaceRev", + "v9.9.9-does-not-exist", + ], + ); + git(&fx.space, &["checkout", "--detach", "HEAD"]); + let (code, diag) = apply(&fx.workspace); + eprintln!("[A2b] exit={:?}\n{}", code, diag); + assert_eq!(code, Some(2), "A2b must refuse"); + assert!( + diag.contains("configured revision"), + "A2b diagnostic: {diag}" + ); +} + +#[test] +fn control_explicit_full_sha_pin_is_accepted() { + // Build with rev = the source's initial commit SHA. + let t = TempDir::new().unwrap(); + let root = t.path().to_path_buf(); + let source = repo( + &root, + "source-space", + &[ + ("SECTION.md", "v1\n"), + ("gripspace.yml", "version: 2\nrepos: {}\n"), + ], + ); + let pin = git(&source, &["rev-parse", "HEAD"]); + let dummy = repo(&root, "dummy-repo", &[("README.md", "d\n")]); + let manifest = repo( + &root, + "workspace-manifest", + &[( + "gripspace.yml", + &format!( + r#"version: 2 +gripspaces: + - url: "{}" + rev: {} +manifest: + url: "{}" + revision: main + composefile: + - dest: OUT.md + parts: + - gripspace: source-space + src: SECTION.md +repos: + dummy-repo: + url: "{}" + path: ./dummy-repo + revision: main +"#, + source.display(), + pin, + root.join("workspace-manifest").display(), + dummy.display() + ), + )], + ); + let workspace = root.join("workspace"); + let init = Command::cargo_bin("gr") + .unwrap() + .args([ + "init", + manifest.to_str().unwrap(), + "--path", + workspace.to_str().unwrap(), + "--no-interactive", + ]) + .output() + .unwrap(); + assert!( + init.status.success(), + "init: {}{}", + String::from_utf8_lossy(&init.stdout), + String::from_utf8_lossy(&init.stderr) + ); + let sync = Command::cargo_bin("gr") + .unwrap() + .arg("sync") + .current_dir(&workspace) + .output() + .unwrap(); + assert!( + sync.status.success(), + "sync: {}{}", + String::from_utf8_lossy(&sync.stdout), + String::from_utf8_lossy(&sync.stderr) + ); + let space = workspace.join(".gitgrip/spaces/source-space"); + eprintln!("[A3] recorded: {}", recorded(&space)); + assert!( + git(&space, &["branch", "--show-current"]).is_empty(), + "SHA pin must be detached" + ); + // Advance upstream: an immutable commit pin is NOT stale by definition. + std::fs::write(source.join("SECTION.md"), "v2\n").unwrap(); + git(&source, &["add", "-A"]); + git(&source, &["commit", "-qm", "advance"]); + let (code, diag) = apply(&workspace); + eprintln!("[A3] exit={:?}\n{}", code, diag); + assert_eq!(code, Some(0), "A3 control must be ACCEPTED: {diag}"); +} + +#[test] +fn moved_tag_is_refused_until_sync_updates_the_managed_clone() { + let t = TempDir::new().unwrap(); + let root = t.path().to_path_buf(); + let source = repo( + &root, + "source-space", + &[ + ("SECTION.md", "v1\n"), + ("gripspace.yml", "version: 2\nrepos: {}\n"), + ], + ); + git(&source, &["tag", "release"]); + let dummy = repo(&root, "dummy-repo", &[("README.md", "d\n")]); + let manifest = repo( + &root, + "workspace-manifest", + &[( + "gripspace.yml", + &format!( + r#"version: 2 +gripspaces: + - url: "{}" + rev: release +manifest: + url: "{}" + revision: main + composefile: + - dest: OUT.md + parts: + - gripspace: source-space + src: SECTION.md +repos: + dummy-repo: + url: "{}" + path: ./dummy-repo + revision: main +"#, + source.display(), + root.join("workspace-manifest").display(), + dummy.display() + ), + )], + ); + let workspace = root.join("workspace"); + let init = Command::cargo_bin("gr") + .unwrap() + .args([ + "init", + manifest.to_str().unwrap(), + "--path", + workspace.to_str().unwrap(), + "--no-interactive", + ]) + .output() + .unwrap(); + assert!( + init.status.success(), + "init: {}{}", + String::from_utf8_lossy(&init.stdout), + String::from_utf8_lossy(&init.stderr) + ); + let sync = Command::cargo_bin("gr") + .unwrap() + .arg("sync") + .current_dir(&workspace) + .output() + .unwrap(); + assert!( + sync.status.success(), + "sync: {}{}", + String::from_utf8_lossy(&sync.stdout), + String::from_utf8_lossy(&sync.stderr) + ); + let space = workspace.join(".gitgrip/spaces/source-space"); + eprintln!("[A4] recorded: {}", recorded(&space)); + let before = git(&space, &["rev-parse", "HEAD"]); + // Upstream moves the tag to new content. + std::fs::write(source.join("SECTION.md"), "v2-MOVED-TAG\n").unwrap(); + git(&source, &["add", "-A"]); + git(&source, &["commit", "-qm", "advance"]); + git(&source, &["tag", "-f", "release"]); + let upstream = git(&source, &["rev-parse", "release"]); + assert_ne!(before, upstream, "fixture: tag must have moved"); + let (code, diag) = apply(&workspace); + let after = git(&space, &["rev-parse", "HEAD"]); + let out = std::fs::read_to_string(workspace.join("OUT.md")).unwrap_or_default(); + eprintln!("[A4] local_before={before} upstream_tag={upstream} local_after={after}"); + eprintln!("[A4] exit={:?} OUT.md={:?}\n{}", code, out, diag); + eprintln!( + "[A4] local tag after fetch: {}", + git(&space, &["rev-parse", "release"]) + ); + assert_eq!(code, Some(2), "moved upstream tag must refuse: {diag}"); + assert!( + diag.contains("does not match configured revision 'release'"), + "moved-tag diagnostic: {diag}" + ); + assert_eq!(after, before, "refusal must not move the managed clone"); + assert_eq!(out, "v1\n", "refusal must not compose stale content again"); + + let (sync_code, sync_diag) = run_sync(&workspace); + assert_eq!(sync_code, Some(0), "gr sync must recover: {sync_diag}"); + assert_eq!( + git(&space, &["rev-parse", "HEAD"]), + upstream, + "sync must materialize the moved upstream tag" + ); + assert_eq!( + git(&space, &["rev-parse", "release"]), + upstream, + "sync must force-update the managed local tag" + ); + + let (apply_code, apply_diag) = apply(&workspace); + assert_eq!( + apply_code, + Some(0), + "apply after recovery must succeed: {apply_diag}" + ); + assert_eq!( + std::fs::read_to_string(workspace.join("OUT.md")).unwrap(), + "v2-MOVED-TAG\n", + "recovered apply must compose the upstream tag content" + ); +} From d6baa30dd6dcb2e16bb58918bc7b48c7ea2b1738 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 12:29:57 -0500 Subject: [PATCH 11/29] fix(gr2): Prototype 1 daemon never prompts, tolerates a failed tick, keeps inherit as inherit The first dogfood run against a real private remote hung on a username prompt: the machine collapsed an explicit empty git environment (the daemon's "inherit" mode) into its isolated default because {} is falsy, so the clone made with the host's credential helper was followed by an ls-remote without it. Three changes: - the machine distinguishes None (the isolated default) from {} (inherit), and names a failed or empty ls-remote SourceUnobservable (a RuntimeError subclass, raised before any state is touched) instead of a bare RuntimeError - the daemon sets GIT_TERMINAL_PROMPT=0 in both git environment modes, so a missing credential fails the tick instead of hanging the loop on a tty - run_loop prints and counts a tick whose git call fails (SourceUnobservable, CalledProcessError, OSError) and goes on; the next tick replays whatever the machine left pending, which is the machine's own kill-and-replay contract Witnesses: inherit reaches the machine as inherit (asserted at both ends), both modes carry the prompt override, a source moved away fails two ticks with tick-failed lines and no receipt, and the tick after it returns acknowledges. Four mutations (restore the `or`, drop the prompt override, stop catching SourceUnobservable, raise the bare RuntimeError again) each kill exactly their witnesses. Co-Authored-By: Claude --- gr2/prototypes/propagation_daemon.py | 74 ++++++++++++++++++--- gr2/prototypes/propagation_state_machine.py | 20 +++++- gr2/tests/test_propagation_daemon.py | 51 +++++++++++++- gr2/tests/test_propagation_state_machine.py | 19 ++++++ 4 files changed, 150 insertions(+), 14 deletions(-) diff --git a/gr2/prototypes/propagation_daemon.py b/gr2/prototypes/propagation_daemon.py index c169c1f..a5daee4 100644 --- a/gr2/prototypes/propagation_daemon.py +++ b/gr2/prototypes/propagation_daemon.py @@ -31,7 +31,16 @@ configuration (the Prototype 0 default); a real private remote needs the host's credential helper, so a declaration may say ``"git_env": "inherit"``. The daemon never creates a commit (fast-forward only), so inheriting the host configuration does not -invoke signing. +invoke signing. In both modes ``GIT_TERMINAL_PROMPT=0`` is set: a daemon never answers +a prompt, so a missing credential fails the tick instead of hanging the loop on a tty. +A tick whose git call fails is printed as ``propagation tick-failed`` and counted, and +the loop continues; the next tick replays whatever the machine left pending. + +The first dogfood run found the inherit seam the hard way: the machine collapsed an +explicit empty environment into its isolated default because ``{}`` is falsy, so the +clone that ``ensure_replica`` made with the host's credentials was followed by an +``ls-remote`` that prompted for a username. The machine now distinguishes ``None`` +from ``{}`` and the seam is witnessed at both ends. """ from __future__ import annotations @@ -57,6 +66,7 @@ Policy, Propagator, Receipt, + SourceUnobservable, State, ) from gr2.python_cli.events import EventType, emit @@ -104,7 +114,11 @@ def receipts_dir(self) -> Path: @property def git_env(self) -> dict[str, str]: - return {} if self.git_env_mode == "inherit" else dict(_ISOLATED_GIT_ENV) + # A daemon never answers a prompt: in both modes a missing credential must fail + # the git call (and the tick) rather than hang the loop on a tty. "inherit" carries + # ONLY that override, so the host's credential helper and config are in force. + base = {} if self.git_env_mode == "inherit" else dict(_ISOLATED_GIT_ENV) + return {**base, "GIT_TERMINAL_PROMPT": "0"} def destination(self) -> Destination: return Destination( @@ -410,10 +424,29 @@ def tick(declaration: Declaration, propagator: Propagator) -> TickResult: class LoopStats: ticks: int = 0 operations: int = 0 + failures: int = 0 by_state: dict[str, int] = field(default_factory=dict) last: TickResult | None = None +_TICK_FAILURES = (SourceUnobservable, subprocess.CalledProcessError, OSError) + + +def _git_failure_line(exc: BaseException) -> str: + if isinstance(exc, SourceUnobservable): + return str(exc) + if isinstance(exc, subprocess.CalledProcessError): + argv = ( + " ".join(str(part) for part in exc.cmd) + if isinstance(exc.cmd, list | tuple) + else exc.cmd + ) + stderr = (exc.stderr or "").strip().splitlines() + tail = stderr[-1] if stderr else "" + return f"git exited {exc.returncode} ({argv}): {tail}" + return f"{type(exc).__name__}: {exc}" + + def run_loop( declaration: Declaration, *, @@ -424,6 +457,14 @@ def run_loop( ) -> LoopStats: """Tick until ``stop()`` says so (or once). Every tick prints exactly one line. + A tick whose git call fails (the source unobservable, a credential refused, the mirror + fetch interrupted) is printed and counted and the loop goes on: nothing about the + declaration changed, and whatever the machine left pending is replayed on the next + tick, which is the machine's own kill-and-replay contract. The machine names the first + of those ``SourceUnobservable``, raised before any state is touched; the others arrive + as ``CalledProcessError`` / ``OSError`` from its git calls. Any other exception is a + defect in this module and propagates. + ``out`` defaults to the stdout in force at CALL time, not at import time, so a caller that redirects stdout (a test, a wrapper, a supervisor) gets the lines. """ @@ -432,14 +473,27 @@ def run_loop( propagator = make_propagator(declaration) stats = LoopStats() while True: - result = tick(declaration, propagator) - stats.ticks += 1 - stats.last = result - if result.receipt is not None: - stats.operations += 1 - key = str(result.receipt.state) - stats.by_state[key] = stats.by_state.get(key, 0) + 1 - print(f"{result.observed_at} {result.summary}", file=stream, flush=True) + observed_at = datetime.now(UTC).isoformat() + try: + result = tick(declaration, propagator) + except _TICK_FAILURES as exc: + stats.ticks += 1 + stats.failures += 1 + print( + f"{observed_at} propagation tick-failed: {declaration.destination_id} " + f"{_git_failure_line(exc)}; this tick left no receipt, the next tick " + f"replays whatever the machine left pending", + file=stream, + flush=True, + ) + else: + stats.ticks += 1 + stats.last = result + if result.receipt is not None: + stats.operations += 1 + key = str(result.receipt.state) + stats.by_state[key] = stats.by_state.get(key, 0) + 1 + print(f"{result.observed_at} {result.summary}", file=stream, flush=True) if once or (stop is not None and stop()): return stats sleep(declaration.interval_seconds) diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index 8c03426..21b7928 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -102,6 +102,15 @@ class DestinationUnreadable(RuntimeError): """A read against the destination returned an error instead of an answer.""" +class SourceUnobservable(RuntimeError): + """``git ls-remote`` against the source failed or advertised no such branch. + + Raised before any cursor, journal, or destination state is touched, so a caller that + loops (a daemon) can count it and try again on its next tick with nothing to repair. + A ``RuntimeError`` subclass, so a caller that caught the bare class still does. + """ + + class Direction(StrEnum): DOWN = "down" UP = "up" @@ -536,7 +545,9 @@ def __init__( self.policy = policy self.kill_after = kill_after self.after_apply_verb = after_apply_verb - self.git_env = git_env or _ISOLATED_GIT_ENV + # None means "the default, isolated from the host"; an explicit {} means "inherit the + # host environment" and must not collapse into the default because it is falsy + self.git_env = _ISOLATED_GIT_ENV if git_env is None else git_env self.journal = Journal(state_dir) self.mirror = state_dir / "mirror.git" @@ -550,8 +561,11 @@ def observe_source_at(self, cursor: str | None) -> Observation: env={**os.environ, **self.git_env}, ) if out.returncode != 0 or not out.stdout.strip(): - raise RuntimeError( - f"source {self.source_remote} has no branch {self.branch}: {out.stderr.strip()}" + stderr = out.stderr.strip().splitlines() + tail = stderr[-1] if stderr else "no output" + raise SourceUnobservable( + f"git ls-remote exited {out.returncode} for {self.source_remote} " + f"refs/heads/{self.branch}: {tail}" ) source_rev = out.stdout.split()[0] return Observation(source_rev=source_rev, cursor=cursor, is_new=(source_rev != cursor)) diff --git a/gr2/tests/test_propagation_daemon.py b/gr2/tests/test_propagation_daemon.py index c9eca76..8a2f655 100644 --- a/gr2/tests/test_propagation_daemon.py +++ b/gr2/tests/test_propagation_daemon.py @@ -198,8 +198,12 @@ def test_declaration_builds_a_downward_apply_coordinate_for_one_managed_replica( assert decl.git_env_mode == "isolated" assert decl.git_env["GIT_CONFIG_GLOBAL"] == os.devnull assert decl.git_env["GIT_CONFIG_NOSYSTEM"] == "1" + # a daemon never answers a prompt, in either mode + assert decl.git_env["GIT_TERMINAL_PROMPT"] == "0" inherit = declaration_from_dict(declaration_dict(synthetic, git_env="inherit")) - assert inherit.git_env == {} + # inherit carries ONLY the prompt override: the host's config and credential helper + # stay in force, which is the whole point of declaring it for a private remote + assert inherit.git_env == {"GIT_TERMINAL_PROMPT": "0"} def test_declaration_defaults_interval_and_policy_hash_when_absent(synthetic: Synthetic) -> None: @@ -288,6 +292,12 @@ def test_make_propagator_passes_the_source_url_through_as_the_exact_string( assert propagator.state_dir == decl.state_dir assert propagator.policy == decl.policy() assert propagator.git_env == decl.git_env + assert "GIT_CONFIG_GLOBAL" in propagator.git_env # isolated by default + # and the inherit mode reaches the machine as inherit: the host config is NOT masked + # (the first dogfood run lost the credential helper here and hung on a username prompt) + inherit = make_propagator(declaration_from_dict(declaration_dict(synthetic, git_env="inherit"))) + assert inherit.git_env == {"GIT_TERMINAL_PROMPT": "0"} + assert "GIT_CONFIG_GLOBAL" not in inherit.git_env # ----------------------------------------------------------------------- ensure_replica @@ -627,6 +637,45 @@ def test_run_loop_once_ensures_the_replica_ticks_once_and_prints_exactly_one_lin assert lines[0] == f"{stats.last.observed_at} {stats.last.summary}" +def test_a_tick_whose_git_call_fails_is_printed_counted_and_the_loop_goes_on( + synthetic: Synthetic, declaration: Declaration +) -> None: + ensure_replica(declaration) + # take the source away: the replica still names it as origin, so ensure_replica accepts + # the path, and the first git call of the tick (ls-remote) fails + moved = synthetic.remote.with_name("source.git.moved") + synthetic.remote.rename(moved) + out = io.StringIO() + slept: list[float] = [] + seen = {"n": 0} + + def stop() -> bool: + seen["n"] += 1 + return seen["n"] >= 2 + + stats = run_loop(declaration, stop=stop, sleep=slept.append, out=out) + assert stats.ticks == 2 + assert stats.failures == 2 + assert stats.operations == 0 + assert stats.by_state == {} + assert stats.last is None + assert slept == [7.5] # it kept going after the first failure + lines = out.getvalue().splitlines() + assert len(lines) == 2 + for line in lines: + assert "propagation tick-failed: config-main-replica git ls-remote exited" in line + assert "ls-remote" in line + assert "this tick left no receipt" in line + assert receipt_files(declaration) == [] + assert outbox_events(declaration.outbox_root) == [] + # put the source back: the very next tick is an ordinary operation, nothing to repair + moved.rename(synthetic.remote) + recovered = run_loop(declaration, once=True, sleep=slept.append, out=io.StringIO()) + assert recovered.failures == 0 + assert recovered.operations == 1 + assert recovered.by_state == {"acknowledged": 1} + + def test_run_loop_stops_when_told_and_sleeps_the_declared_interval_between_ticks( synthetic: Synthetic, declaration: Declaration ) -> None: diff --git a/gr2/tests/test_propagation_state_machine.py b/gr2/tests/test_propagation_state_machine.py index 952bd94..87a6cc4 100644 --- a/gr2/tests/test_propagation_state_machine.py +++ b/gr2/tests/test_propagation_state_machine.py @@ -669,6 +669,25 @@ def test_run_all_with_no_declared_targets_is_none(synthetic: Synthetic) -> None: assert propagator(synthetic).run_all([]) is None +def test_an_explicit_empty_git_env_is_inherit_and_not_the_isolated_default( + synthetic: Synthetic, +) -> None: + # None asks for the default (isolated from the host); {} asks to inherit the host + # environment. {} is falsy, so an `or` would silently turn inherit into isolated and a + # caller that needs the host's credential helper would lose it without any error. + assert "GIT_CONFIG_GLOBAL" in propagator(synthetic).git_env + assert propagator(synthetic, git_env=None).git_env == propagator(synthetic).git_env + assert propagator(synthetic, git_env={}).git_env == {} + explicit = {"GIT_TERMINAL_PROMPT": "0"} + assert propagator(synthetic, git_env=explicit).git_env == explicit + # and the inherit form still drives the machine: a change is applied under {} as well + dest = replica(synthetic, "replica") + new = synthetic.push_change("canon v2\n") + receipt = propagator(synthetic, git_env={}).run(coordinate(dest.destination_id), dest) + assert receipt is not None and receipt.state is State.ACKNOWLEDGED + assert git(dest.path, "rev-parse", "HEAD") == new + + def test_all_destinations_verifying_is_acknowledged_not_partial(synthetic: Synthetic) -> None: a = replica(synthetic, "replica-a") b = replica(synthetic, "replica-b") From ff479984d3bdf54bd438e7a58f5eae7f83814568 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 12:52:07 -0500 Subject: [PATCH 12/29] fix(gr2): the daemon loop survives a destination it cannot read; name the journal escape The second reviewer of this branch enumerated every raise site reachable from a tick and ran the one the loop still let escape: the machine wraps a failed destination read in DestinationUnreadable, which is raised at observe, plan, and verify (each a point the machine replays from), and the loop caught the two exceptions it wraps but not the wrapper, so a checkout removed or a volume unmounted mid-loop crashed the daemon on the next operation. - run_loop now catches DestinationUnreadable with the other environmental failures, prints it as a tick-failed line, counts it, and goes on; the catch list is derived from the machine's raise sites and says so - the machine's deliberate escape for a cursor the journal cannot account for is named JournalInconsistent (a RuntimeError subclass) so an escape that is meant is distinguishable from one that is not - the docstring lists what propagates by design Witness: the first tick acknowledges, the checkout is removed and a new source revision pushed, two ticks fail with "destination unreadable" lines and no new receipt, and a fresh loop re-ensures the replica and acknowledges. Mutations: dropping DestinationUnreadable from the catch list kills exactly that witness; raising the bare RuntimeError again kills exactly the corrupted-cursor witness. Co-Authored-By: Claude --- gr2/prototypes/propagation_daemon.py | 34 +++++++++++---- gr2/prototypes/propagation_state_machine.py | 11 ++++- gr2/tests/test_propagation_daemon.py | 47 +++++++++++++++++++++ gr2/tests/test_propagation_state_machine.py | 5 ++- 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/gr2/prototypes/propagation_daemon.py b/gr2/prototypes/propagation_daemon.py index a5daee4..bb41b98 100644 --- a/gr2/prototypes/propagation_daemon.py +++ b/gr2/prototypes/propagation_daemon.py @@ -61,6 +61,7 @@ Coordinate, Destination, DestinationKind, + DestinationUnreadable, Direction, Operation, Policy, @@ -429,12 +430,24 @@ class LoopStats: last: TickResult | None = None -_TICK_FAILURES = (SourceUnobservable, subprocess.CalledProcessError, OSError) +# Every way a tick can fail on the environment rather than on this module, derived from +# the machine's raise sites: the source cannot be observed (before any state is touched), +# the destination cannot be read (at observe, plan, or verify, each a point the machine +# replays from), or a git call failed / could not be spawned. DestinationUnreadable WRAPS +# the last two, so catching them without it would let the wrapped form escape. +_TICK_FAILURES = ( + SourceUnobservable, + DestinationUnreadable, + subprocess.CalledProcessError, + OSError, +) def _git_failure_line(exc: BaseException) -> str: if isinstance(exc, SourceUnobservable): return str(exc) + if isinstance(exc, DestinationUnreadable): + return f"destination unreadable: {exc}" if isinstance(exc, subprocess.CalledProcessError): argv = ( " ".join(str(part) for part in exc.cmd) @@ -457,13 +470,18 @@ def run_loop( ) -> LoopStats: """Tick until ``stop()`` says so (or once). Every tick prints exactly one line. - A tick whose git call fails (the source unobservable, a credential refused, the mirror - fetch interrupted) is printed and counted and the loop goes on: nothing about the - declaration changed, and whatever the machine left pending is replayed on the next - tick, which is the machine's own kill-and-replay contract. The machine names the first - of those ``SourceUnobservable``, raised before any state is touched; the others arrive - as ``CalledProcessError`` / ``OSError`` from its git calls. Any other exception is a - defect in this module and propagates. + A tick that fails on the environment (the source unobservable, the destination + unreadable because the checkout went away or a volume unmounted, a credential refused, + the mirror fetch interrupted) is printed and counted and the loop goes on: nothing + about the declaration changed, and whatever the machine left pending is replayed on + the next tick, which is the machine's own kill-and-replay contract. The machine names + the first two ``SourceUnobservable`` (raised before any state is touched) and + ``DestinationUnreadable`` (raised at observe, plan, or verify, each a replay point); + the rest arrive as ``CalledProcessError`` / ``OSError`` from its git calls. What + propagates, by design: ``JournalInconsistent`` and ``LookupError`` from the machine (a + journal that cannot account for its own cursor is corrupted sink state, and guessing + would hide it), ``DeclarationMismatch`` from ``ensure_replica`` at startup, and any + defect in this module. ``out`` defaults to the stdout in force at CALL time, not at import time, so a caller that redirects stdout (a test, a wrapper, a supervisor) gets the lines. diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index 21b7928..ef8f92e 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -102,6 +102,14 @@ class DestinationUnreadable(RuntimeError): """A read against the destination returned an error instead of an answer.""" +class JournalInconsistent(RuntimeError): + """The cursor names a revision the journal cannot account for. + + A corrupted sink state: the prototype says so and stops rather than guessing an + outcome. Raised deliberately and left to propagate by every caller, by name. + """ + + class SourceUnobservable(RuntimeError): """``git ls-remote`` against the source failed or advertised no such branch. @@ -658,7 +666,7 @@ def _terminal_receipt(self, coordinate: Coordinate, source_rev: str) -> Receipt: if not attempts or attempts[-1][-1].state is not State.ACKNOWLEDGED: # A cursor at a revision the journal never acknowledged is a corrupted # sink state, and the prototype says so rather than guessing an outcome. - raise RuntimeError( + raise JournalInconsistent( f"cursor for {coordinate.destination} is at {source_rev} but the journal " "carries no acknowledged attempt at that revision" ) @@ -1192,6 +1200,7 @@ def _receipt( "Destination", "DestinationKind", "DestinationUnreadable", + "JournalInconsistent", "Direction", "GateResult", "Journal", diff --git a/gr2/tests/test_propagation_daemon.py b/gr2/tests/test_propagation_daemon.py index 8a2f655..1d6d839 100644 --- a/gr2/tests/test_propagation_daemon.py +++ b/gr2/tests/test_propagation_daemon.py @@ -20,6 +20,9 @@ transition timestamps * ``run_loop`` ticks once under ``once``, stops when told, sleeps the declared interval, and prints exactly one line per tick +* a tick that fails on the environment (the source moved away, the destination checkout + removed mid-loop) is printed as ``tick-failed`` and counted, the loop goes on, and the + next loop over a restored environment completes the operation """ from __future__ import annotations @@ -27,6 +30,7 @@ import io import json import os +import shutil import subprocess from dataclasses import dataclass from datetime import datetime @@ -676,6 +680,49 @@ def stop() -> bool: assert recovered.by_state == {"acknowledged": 1} +def test_a_tick_whose_destination_vanished_is_printed_counted_and_the_loop_goes_on( + synthetic: Synthetic, declaration: Declaration +) -> None: + # the replica exists and ensure_replica has accepted it when the loop starts; the + # checkout goes away AFTER the first tick (a removed directory, an unmounted volume), + # and a new source revision makes every later tick an operation that must read it + out = io.StringIO() + slept: list[float] = [] + calls = {"n": 0} + + def stop() -> bool: + calls["n"] += 1 + if calls["n"] == 1: + shutil.rmtree(declaration.destination_path) + synthetic.push_change("canon v2\n") + return False + return calls["n"] >= 3 + + stats = run_loop(declaration, stop=stop, sleep=slept.append, out=out) + assert stats.ticks == 3 + assert stats.operations == 1 # the first tick, acknowledged before the checkout vanished + assert stats.failures == 2 + assert stats.by_state == {"acknowledged": 1} + assert slept == [7.5, 7.5] + lines = out.getvalue().splitlines() + assert len(lines) == 3 + assert "propagation acknowledged: config-main-replica" in lines[0] + for line in lines[1:]: + assert "propagation tick-failed: config-main-replica destination unreadable:" in line + assert "config-main-replica:" in line # the machine names the destination it could not read + assert len(receipt_files(declaration)) == 1 + assert len(outbox_events(declaration.outbox_root)) == 1 + # recovery: a fresh loop re-ensures the replica (absent path -> clone at the source + # revision) and the pending operation completes as an ordinary acknowledgement + recovered = run_loop(declaration, once=True, sleep=slept.append, out=io.StringIO()) + assert recovered.failures == 0 + assert recovered.operations == 1 + assert recovered.by_state == {"acknowledged": 1} + assert git(declaration.destination_path, "rev-parse", "HEAD") == git( + synthetic.author, "rev-parse", "HEAD" + ) + + def test_run_loop_stops_when_told_and_sleeps_the_declared_interval_between_ticks( synthetic: Synthetic, declaration: Declaration ) -> None: diff --git a/gr2/tests/test_propagation_state_machine.py b/gr2/tests/test_propagation_state_machine.py index 87a6cc4..37ed862 100644 --- a/gr2/tests/test_propagation_state_machine.py +++ b/gr2/tests/test_propagation_state_machine.py @@ -43,6 +43,7 @@ DestinationKind, Direction, Journal, + JournalInconsistent, Operation, Policy, Propagator, @@ -646,9 +647,9 @@ def test_a_cursor_the_journal_never_acknowledged_is_a_corrupted_sink_state_and_r Journal(synthetic.state_dir).advance_cursor(coord.key(), new, pending_id="forged") before = snapshot(dest.path) - with pytest.raises(RuntimeError, match="no acknowledged attempt"): + with pytest.raises(JournalInconsistent, match="no acknowledged attempt"): propagator(synthetic).run_all([(coord, dest)]) - with pytest.raises(RuntimeError, match="no acknowledged attempt"): + with pytest.raises(JournalInconsistent, match="no acknowledged attempt"): propagator(synthetic).run(coord, dest) assert snapshot(dest.path) == before, "a refusal to guess must not touch the destination" From 92fff14f54491aa5b515bbfbce305ad07838fc08 Mon Sep 17 00:00:00 2001 From: Atlas Date: Wed, 19 Aug 2026 13:13:48 -0500 Subject: [PATCH 13/29] fix(link): accept abbreviated commit pins during freshness checks --- src/cli/commands/link.rs | 6 +++--- tests/link_apply_detached_freshness.rs | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/cli/commands/link.rs b/src/cli/commands/link.rs index 90957fc..23105b5 100644 --- a/src/cli/commands/link.rs +++ b/src/cli/commands/link.rs @@ -221,8 +221,8 @@ fn remote_tag_commit(repo: &git2::Repository, rev: &str) -> anyhow::Result bool { - rev.len() == 40 && rev.bytes().all(|byte| byte.is_ascii_hexdigit()) +fn is_commit_id_like(rev: &str) -> bool { + rev.len() >= 4 && rev.bytes().all(|byte| byte.is_ascii_hexdigit()) } /// Refuse a manual apply when a gripspace is not at its configured revision. @@ -280,7 +280,7 @@ fn ensure_gripspace_sources_current( let pinned_oid = if let Some(tag_oid) = remote_tag_commit(&repo, &rev)? { tag_oid - } else if is_full_commit_id(&rev) { + } else if is_commit_id_like(&rev) { repo.revparse_single(&rev) .and_then(|object| object.peel_to_commit()) .map(|commit| commit.id()) diff --git a/tests/link_apply_detached_freshness.rs b/tests/link_apply_detached_freshness.rs index e244f19..ab0da87 100644 --- a/tests/link_apply_detached_freshness.rs +++ b/tests/link_apply_detached_freshness.rs @@ -295,8 +295,7 @@ fn link_apply_refuses_a_detached_source_with_an_unresolvable_named_revision() { ); } -#[test] -fn control_explicit_full_sha_pin_is_accepted() { +fn assert_explicit_sha_pin_is_accepted(pin_len: usize) { // Build with rev = the source's initial commit SHA. let t = TempDir::new().unwrap(); let root = t.path().to_path_buf(); @@ -308,7 +307,8 @@ fn control_explicit_full_sha_pin_is_accepted() { ("gripspace.yml", "version: 2\nrepos: {}\n"), ], ); - let pin = git(&source, &["rev-parse", "HEAD"]); + let full_pin = git(&source, &["rev-parse", "HEAD"]); + let pin = &full_pin[..pin_len]; let dummy = repo(&root, "dummy-repo", &[("README.md", "d\n")]); let manifest = repo( &root, @@ -372,7 +372,7 @@ repos: String::from_utf8_lossy(&sync.stderr) ); let space = workspace.join(".gitgrip/spaces/source-space"); - eprintln!("[A3] recorded: {}", recorded(&space)); + eprintln!("[A3/{pin_len}] recorded: {}", recorded(&space)); assert!( git(&space, &["branch", "--show-current"]).is_empty(), "SHA pin must be detached" @@ -382,10 +382,20 @@ repos: git(&source, &["add", "-A"]); git(&source, &["commit", "-qm", "advance"]); let (code, diag) = apply(&workspace); - eprintln!("[A3] exit={:?}\n{}", code, diag); + eprintln!("[A3/{pin_len}] exit={:?}\n{}", code, diag); assert_eq!(code, Some(0), "A3 control must be ACCEPTED: {diag}"); } +#[test] +fn control_explicit_full_sha_pin_is_accepted() { + assert_explicit_sha_pin_is_accepted(40); +} + +#[test] +fn explicit_short_sha_pin_is_accepted() { + assert_explicit_sha_pin_is_accepted(8); +} + #[test] fn moved_tag_is_refused_until_sync_updates_the_managed_clone() { let t = TempDir::new().unwrap(); From f792d6659d082dd7408c5f1a9cd33ca7d053063f Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 14:02:07 -0500 Subject: [PATCH 14/29] fix(gr2): the daemon's not-new shortcut no longer hides a corrupted cursor; witness what propagates Reviewing commit 3, the second reviewer widened the loop's catch list to swallow RuntimeError and LookupError and the suite stayed green: the docstring's claim about what propagates by design was unwitnessed. Writing the witness found a real gap behind it. The daemon's tick observed the source itself and, when the cursor already named the source revision, returned "current; not an operation" without asking the machine, so the machine's corrupted-sink check (a cursor at a revision the journal never acknowledged raises JournalInconsistent) never ran on the daemon path and "not an operation" would have hidden corrupted sink state on every tick forever. Found by the witness, not by the dogfood. - Propagator.run accepts an optional observation so a caller that already observed this tick can hand it over instead of asking the source twice; the cursor check runs on it all the same - the daemon hands its observation to the machine on both paths, so a corrupted cursor leaves the loop by name with no tick line and no receipt - witnesses: a forged cursor makes run_loop raise JournalInconsistent with nothing written; LookupError, a bare RuntimeError, and ValueError from a tick are not swallowed (the loop is bounded in the test so a wrong loop fails rather than hangs) Mutations, eight rows in one run, each killing exactly its witnesses: the six from before, plus restoring the not-new shortcut and widening the catch list to RuntimeError and LookupError. Co-Authored-By: Claude --- gr2/prototypes/propagation_daemon.py | 11 ++- gr2/prototypes/propagation_state_machine.py | 25 +++++-- gr2/tests/test_propagation_daemon.py | 76 +++++++++++++++++++-- 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/gr2/prototypes/propagation_daemon.py b/gr2/prototypes/propagation_daemon.py index bb41b98..43b75cc 100644 --- a/gr2/prototypes/propagation_daemon.py +++ b/gr2/prototypes/propagation_daemon.py @@ -379,6 +379,12 @@ def tick(declaration: Declaration, propagator: Propagator) -> TickResult: coordinate = declaration.coordinate observation = propagator.observe_source(coordinate) if not observation.is_new: + # Not an operation, but not a shortcut past the machine either: it must account + # for a cursor that sits at the source revision, and it raises JournalInconsistent + # for one the journal never acknowledged. An earlier version of this tick returned + # here without asking, and "current; not an operation" would have hidden corrupted + # sink state on every tick forever. Found by the witness, not by the dogfood. + propagator.run(coordinate, declaration.destination(), observation=observation) return TickResult( observed_at=observed_at, source_rev=observation.source_rev, @@ -391,9 +397,10 @@ def tick(declaration: Declaration, propagator: Propagator) -> TickResult: f"{_short(observation.source_rev)}; not an operation" ), ) - receipt = propagator.run(coordinate, declaration.destination()) + receipt = propagator.run(coordinate, declaration.destination(), observation=observation) if receipt is None: - # the source moved between observe and run, back to the cursor; nothing to do + # the machine's contract allows None for "not new"; with the observation shared it + # cannot answer that here, but a None is still not an operation and writes nothing return TickResult( observed_at=observed_at, source_rev=observation.source_rev, diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index ef8f92e..7b89cba 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -594,18 +594,33 @@ def _porcelain(self, destination: Destination) -> str: # -- driver - def run(self, coordinate: Coordinate, destination: Destination) -> Receipt | None: + def run( + self, + coordinate: Coordinate, + destination: Destination, + observation: Observation | None = None, + ) -> Receipt | None: """Drive one coordinate. ``None`` means no new source revision: not an operation. - Raises if the cursor sits at a revision the journal never acknowledged. + Raises ``JournalInconsistent`` if the cursor sits at a revision the journal never + acknowledged. A caller that has already observed the source this tick may pass + its ``observation`` so the source is not asked twice; the cursor check runs on it + all the same, because "nothing new" is exactly the answer that would hide a + corrupted sink state, whoever observed. """ - return self._run(coordinate, destination, report_current=False) + return self._run(coordinate, destination, report_current=False, observation=observation) def _run( - self, coordinate: Coordinate, destination: Destination, *, report_current: bool + self, + coordinate: Coordinate, + destination: Destination, + *, + report_current: bool, + observation: Observation | None = None, ) -> Receipt | None: key = coordinate.key() - observation = self.observe_source(coordinate) + if observation is None: + observation = self.observe_source(coordinate) if not observation.is_new: # The cursor is AT the source revision, which only happens after an # acknowledgement at that revision; _terminal_receipt RAISES if the diff --git a/gr2/tests/test_propagation_daemon.py b/gr2/tests/test_propagation_daemon.py index 1d6d839..33194cf 100644 --- a/gr2/tests/test_propagation_daemon.py +++ b/gr2/tests/test_propagation_daemon.py @@ -23,6 +23,9 @@ * a tick that fails on the environment (the source moved away, the destination checkout removed mid-loop) is printed as ``tick-failed`` and counted, the loop goes on, and the next loop over a restored environment completes the operation +* what the loop says propagates does propagate: a corrupted cursor (``JournalInconsistent``) + leaves the loop by name with no tick line and no receipt, and ``LookupError`` or a bare + ``RuntimeError`` from a tick is not swallowed either """ from __future__ import annotations @@ -55,6 +58,8 @@ from gr2.prototypes.propagation_state_machine import ( DestinationKind, Direction, + Journal, + JournalInconsistent, Operation, State, ) @@ -583,12 +588,12 @@ def test_a_dirty_replica_is_refused_with_a_receipt_left_untouched_then_applies_o def test_tick_writes_and_announces_nothing_when_the_machine_returns_none( declaration: Declaration, monkeypatch: pytest.MonkeyPatch ) -> None: - # the only way to reach this branch is a race (the source moved back to the cursor - # between observe and run), so the machine's answer is forced here; the claim under - # test is the daemon's: no receipt, no file, no event + # the machine's contract allows None ("not new"); since the daemon now hands the + # machine its own observation that answer cannot arise on this path, so it is forced + # here; the claim under test is the daemon's: no receipt, no file, no event ensure_replica(declaration) propagator = make_propagator(declaration) - monkeypatch.setattr(propagator, "run", lambda coordinate, destination: None) + monkeypatch.setattr(propagator, "run", lambda coordinate, destination, observation=None: None) result = tick(declaration, propagator) assert not result.was_operation assert result.receipt_path is None @@ -723,6 +728,69 @@ def stop() -> bool: ) +def test_a_corrupted_cursor_propagates_out_of_the_loop_by_name_instead_of_retrying_forever( + synthetic: Synthetic, declaration: Declaration +) -> None: + # The one state the journal can never produce on its own: a cursor AT the source + # revision with no acknowledged attempt behind it. The machine names it + # JournalInconsistent and the loop must NOT treat it as an environmental failure: a + # daemon that swallowed it would print tick-failed forever over corrupted sink state + # and nobody would learn. Found unwitnessed in review (widening the catch list to + # RuntimeError kept the suite green), so this is the witness. + ensure_replica(declaration) + propagator = make_propagator(declaration) + Journal(declaration.state_dir).advance_cursor( + declaration.coordinate.key(), synthetic.base, pending_id="forged" + ) + out = io.StringIO() + with pytest.raises(JournalInconsistent, match="no acknowledged attempt"): + run_loop(declaration, once=True, sleep=lambda _s: None, out=out) + assert out.getvalue() == "" # no tick line: the loop did not survive it + assert receipt_files(declaration) == [] + assert outbox_events(declaration.outbox_root) == [] + # and the same with the propagator handed in through tick() directly + with pytest.raises(JournalInconsistent): + tick(declaration, propagator) + + +@pytest.mark.parametrize( + "exc", + [ + pytest.param(LookupError("no transitions journaled"), id="LookupError"), + pytest.param(RuntimeError("a defect in the daemon"), id="bare-RuntimeError"), + pytest.param(ValueError("a defect in the daemon"), id="ValueError"), + ], +) +def test_the_loop_does_not_swallow_what_it_says_propagates( + declaration: Declaration, monkeypatch: pytest.MonkeyPatch, exc: BaseException +) -> None: + # The docstring lists what propagates by design; this pins the list from the other + # side. A "hardening" edit that widened _TICK_FAILURES to RuntimeError or LookupError + # would turn a corrupted-journal halt into an infinite retry, silently. + import gr2.prototypes.propagation_daemon as daemon_module + + calls = {"n": 0} + + def exploding_tick(decl, propagator): # noqa: ANN001 - signature of daemon.tick + calls["n"] += 1 + raise exc + + monkeypatch.setattr(daemon_module, "tick", exploding_tick) + out = io.StringIO() + # the stop predicate is BOUNDED so that a loop which wrongly swallows the exception + # terminates and fails here, instead of spinning forever and hanging the suite + rounds = {"n": 0} + + def stop_after_three() -> bool: + rounds["n"] += 1 + return rounds["n"] >= 3 + + with pytest.raises(type(exc)): + run_loop(declaration, stop=stop_after_three, sleep=lambda _s: None, out=out) + assert calls["n"] == 1 # it did not go round again + assert out.getvalue() == "" + + def test_run_loop_stops_when_told_and_sleeps_the_declared_interval_between_ticks( synthetic: Synthetic, declaration: Declaration ) -> None: From 2ca875081c6e727572df5be61efed92ba364f147 Mon Sep 17 00:00:00 2001 From: Atlas Date: Wed, 19 Aug 2026 14:04:44 -0500 Subject: [PATCH 15/29] fix(link): distinguish commit pins from stale refs --- src/cli/commands/link.rs | 26 +++-- tests/link_apply_detached_freshness.rs | 139 ++++++++++++------------- 2 files changed, 84 insertions(+), 81 deletions(-) diff --git a/src/cli/commands/link.rs b/src/cli/commands/link.rs index 23105b5..4e79593 100644 --- a/src/cli/commands/link.rs +++ b/src/cli/commands/link.rs @@ -225,6 +225,19 @@ fn is_commit_id_like(rev: &str) -> bool { rev.len() >= 4 && rev.bytes().all(|byte| byte.is_ascii_hexdigit()) } +/// Resolve a hexadecimal object-id prefix without interpreting it as a ref name. +/// +/// `revparse_single` gives refs precedence, so an all-hex tag deleted from +/// origin could otherwise resolve through the stale local tag and be mistaken +/// for an immutable commit pin. The object database answers the narrower +/// question this branch asks and rejects ambiguous prefixes itself. +fn commit_from_id_prefix(repo: &git2::Repository, rev: &str) -> anyhow::Result { + let prefix = git2::Oid::from_str(rev)?; + let oid = repo.odb()?.exists_prefix(prefix, rev.len())?; + repo.find_commit(oid)?; + Ok(oid) +} + /// Refuse a manual apply when a gripspace is not at its configured revision. /// /// A successful local composition proves that the files are internally @@ -281,14 +294,11 @@ fn ensure_gripspace_sources_current( let pinned_oid = if let Some(tag_oid) = remote_tag_commit(&repo, &rev)? { tag_oid } else if is_commit_id_like(&rev) { - repo.revparse_single(&rev) - .and_then(|object| object.peel_to_commit()) - .map(|commit| commit.id()) - .map_err(|error| { - CliOutcomeError::refusal(format!( - "Cannot verify gripspace source '{name}' at configured commit '{rev}': {error}" - )) - })? + commit_from_id_prefix(&repo, &rev).map_err(|error| { + CliOutcomeError::refusal(format!( + "Cannot verify gripspace source '{name}' at configured commit id '{rev}': {error}. Use a longer commit id if the prefix is ambiguous, or correct the configured revision." + )) + })? } else { return Err(CliOutcomeError::refusal(format!( "Cannot verify gripspace source '{name}' at configured revision '{rev}' against origin. Run `gr sync` before `gr link --apply`." diff --git a/tests/link_apply_detached_freshness.rs b/tests/link_apply_detached_freshness.rs index ab0da87..79f3458 100644 --- a/tests/link_apply_detached_freshness.rs +++ b/tests/link_apply_detached_freshness.rs @@ -51,6 +51,13 @@ struct Fx { /// Build a workspace whose gripspace source is materialized at `rev`. fn setup(rev: &str) -> Fx { + setup_with_source(rev, |_| {}) +} + +fn setup_with_source(rev: &str, prepare_source: F) -> Fx +where + F: FnOnce(&Path), +{ let t = TempDir::new().unwrap(); let root = t.path().to_path_buf(); let source = repo( @@ -61,6 +68,7 @@ fn setup(rev: &str) -> Fx { ("gripspace.yml", "version: 2\nrepos: {}\n"), ], ); + prepare_source(&source); let dummy = repo(&root, "dummy-repo", &[("README.md", "d\n")]); let manifest = repo( &root, @@ -295,6 +303,27 @@ fn link_apply_refuses_a_detached_source_with_an_unresolvable_named_revision() { ); } +#[test] +fn an_unresolvable_all_hex_commit_prefix_names_an_operator_remedy() { + let fx = setup("main"); + git( + &fx.space, + &[ + "config", + "gitgrip.requestedGripspaceRev", + "0000000000000000000000000000000000000000", + ], + ); + git(&fx.space, &["checkout", "--detach", "HEAD"]); + let (code, diag) = apply(&fx.workspace); + assert_eq!(code, Some(2), "unresolvable commit prefix must refuse"); + assert!( + diag.contains("Use a longer commit id if the prefix is ambiguous") + && diag.contains("correct the configured revision"), + "diagnostic must name remedies that can change the result: {diag}" + ); +} + fn assert_explicit_sha_pin_is_accepted(pin_len: usize) { // Build with rev = the source's initial commit SHA. let t = TempDir::new().unwrap(); @@ -398,79 +427,12 @@ fn explicit_short_sha_pin_is_accepted() { #[test] fn moved_tag_is_refused_until_sync_updates_the_managed_clone() { - let t = TempDir::new().unwrap(); - let root = t.path().to_path_buf(); - let source = repo( - &root, - "source-space", - &[ - ("SECTION.md", "v1\n"), - ("gripspace.yml", "version: 2\nrepos: {}\n"), - ], - ); - git(&source, &["tag", "release"]); - let dummy = repo(&root, "dummy-repo", &[("README.md", "d\n")]); - let manifest = repo( - &root, - "workspace-manifest", - &[( - "gripspace.yml", - &format!( - r#"version: 2 -gripspaces: - - url: "{}" - rev: release -manifest: - url: "{}" - revision: main - composefile: - - dest: OUT.md - parts: - - gripspace: source-space - src: SECTION.md -repos: - dummy-repo: - url: "{}" - path: ./dummy-repo - revision: main -"#, - source.display(), - root.join("workspace-manifest").display(), - dummy.display() - ), - )], - ); - let workspace = root.join("workspace"); - let init = Command::cargo_bin("gr") - .unwrap() - .args([ - "init", - manifest.to_str().unwrap(), - "--path", - workspace.to_str().unwrap(), - "--no-interactive", - ]) - .output() - .unwrap(); - assert!( - init.status.success(), - "init: {}{}", - String::from_utf8_lossy(&init.stdout), - String::from_utf8_lossy(&init.stderr) - ); - let sync = Command::cargo_bin("gr") - .unwrap() - .arg("sync") - .current_dir(&workspace) - .output() - .unwrap(); - assert!( - sync.status.success(), - "sync: {}{}", - String::from_utf8_lossy(&sync.stdout), - String::from_utf8_lossy(&sync.stderr) - ); - let space = workspace.join(".gitgrip/spaces/source-space"); + let fx = setup_with_source("release", |source| { + git(source, &["tag", "release"]); + }); + let source = &fx.source; + let workspace = &fx.workspace; + let space = &fx.space; eprintln!("[A4] recorded: {}", recorded(&space)); let before = git(&space, &["rev-parse", "HEAD"]); // Upstream moves the tag to new content. @@ -522,3 +484,34 @@ repos: "recovered apply must compose the upstream tag content" ); } + +#[test] +fn an_all_hex_tag_deleted_from_origin_is_not_reinterpreted_as_a_commit_prefix() { + let fx = setup_with_source("cafe", |source| { + git(source, &["tag", "cafe"]); + }); + let before = git(&fx.space, &["rev-parse", "HEAD"]); + assert_eq!( + git(&fx.space, &["rev-parse", "cafe"]), + before, + "fixture: the managed clone must retain its local tag" + ); + + git(&fx.source, &["tag", "-d", "cafe"]); + let (code, diag) = apply(&fx.workspace); + assert_eq!( + code, + Some(2), + "a tag absent from origin must not certify through its stale local ref: {diag}" + ); + assert_eq!( + git(&fx.space, &["rev-parse", "HEAD"]), + before, + "refusal must not move the managed clone" + ); + assert_eq!( + std::fs::read_to_string(fx.workspace.join("OUT.md")).unwrap(), + "v1\n", + "refusal must not compose again from the stale local tag" + ); +} From eb333082f19cd3ea0576a53606cb4ed9de9799f2 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 16:00:46 -0500 Subject: [PATCH 16/29] =?UTF-8?q?feat(gr2):=20Prototype=202=20=E2=80=94=20?= =?UTF-8?q?contributions=20land=20on=20a=20canonical=20remote=20by=20lease?= =?UTF-8?q?-guarded=20fast-forward=20push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contribution protocol on the Prototype 0 machine: one new destination kind and no new state. DestinationKind.CANONICAL is a bare remote that owns the branch; an operation with direction=up observes the child's branch, fetches it into the sink's mirror, plans against the owner's branch as read now, and lands by `git push --force-with-lease=:` from the mirror. The receiving repository enforces the compare-and-swap; the plan's fast-forward gate guarantees the lease never forces; a rejected lease is a REFUSAL carrying the revision the owner holds. Replanning is the author's act: the machine never rebases, merges, or forces on anyone's behalf. Machine changes: read_head reads the branch ref (never HEAD) for a canonical; the cleanliness gate is recorded NOT RUN for a bare destination rather than omitted; ahead/behind fetches the branch for a canonical; apply has a lease-push verb with a `before_apply_verb` test seam; verify names the canonical postcondition (branch-is-intended-after-and-tree-matches-digest, no worktree term). Six witnesses on a scratch parent + two subspace clones: the happy path with exact revisions and the not-run gate; the manufactured collision refused at plan with the observed base and nothing touched; the compare-and-swap race (plan, sink dies, the other child lands, resume) refused at apply; the lease refusing a move inside the check→push window; replan-by-the-author landing as a fresh attempt with both attempts in the journal; policy refusing `up` before any verb. Five mutations each kill their own witness (bare --force instead of the lease → the window witness; dropped apply-step head check → the race witness; fast-forward gate forced to pass → the collision witness, because a held lease would then force; porcelain on the bare remote → every landing; clean gate as pass instead of not-run → the happy path). Existing suites: 83 passed, 1 xfailed unchanged; with this file 89 + 1. Co-Authored-By: Claude --- gr2/prototypes/propagation_state_machine.py | 200 ++++++++-- gr2/tests/test_propagation_contribution.py | 420 ++++++++++++++++++++ 2 files changed, 592 insertions(+), 28 deletions(-) create mode 100644 gr2/tests/test_propagation_contribution.py diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index 7b89cba..18dbc12 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -60,7 +60,18 @@ Fault injection for tests: ``kill_after`` raises :class:`SinkKilled` right after the named state is journaled (or right after the apply verb ran, before the read back, with ``KILL_AFTER_APPLY_VERB``); ``after_apply_verb`` runs a callable -between the verb and the read back so a destination can be made unreadable. +between the verb and the read back so a destination can be made unreadable; +``before_apply_verb`` runs a callable after the apply step's own head check and +before the verb, so a destination can be moved inside that window. + +Prototype 2 (the contribution protocol) adds ONE destination kind and no new +state: :attr:`DestinationKind.CANONICAL` is a bare remote that OWNS the branch, and +an operation with ``direction=up`` lands on it by a fast-forward push guarded by +``--force-with-lease`` on the expected base. The receiving repository enforces the +compare-and-swap; the plan's fast-forward gate guarantees the lease never forces; +a rejected lease is a REFUSAL carrying the revision the owner holds, not an error. +Replanning a refused contribution is the author's act — the machine never rebases, +merges, or forces on anyone's behalf. """ from __future__ import annotations @@ -84,6 +95,10 @@ "destination.fast-forward", ) _POSTCONDITION = "head-is-intended-after-and-tree-matches-digest" +# A canonical destination is bare: the postcondition is about its BRANCH ref, and +# "clean" is not part of it because there is no worktree to be clean. +_POSTCONDITION_CANONICAL = "branch-is-intended-after-and-tree-matches-digest" +_GATE_NOT_RUN = "not-run" # The prototype runs git without the user's global or system configuration so # that signing, hook paths, and identity settings on the host cannot reach the @@ -134,6 +149,11 @@ class Operation(StrEnum): class DestinationKind(StrEnum): REPLICA = "replica" AUTHORING = "authoring" + # Prototype 2 (the contribution protocol): a bare remote that OWNS the branch and + # accepts a contribution by a fast-forward push guarded by ``--force-with-lease`` on the + # expected base — compare-and-swap enforced by the receiving repository, not by + # this process. It has no worktree, so cleanliness is not a property it has. + CANONICAL = "canonical" class State(StrEnum): @@ -545,6 +565,7 @@ def __init__( *, kill_after: State | str | None = None, after_apply_verb: Callable[[], None] | None = None, + before_apply_verb: Callable[[], None] | None = None, git_env: dict[str, str] | None = None, ) -> None: self.source_remote = source_remote @@ -553,6 +574,10 @@ def __init__( self.policy = policy self.kill_after = kill_after self.after_apply_verb = after_apply_verb + # Prototype 2 test seam: runs AFTER the apply step's own head check and BEFORE + # the verb, so a test can move a canonical destination inside the window the + # process-side check cannot see. The lease is what must catch that move. + self.before_apply_verb = before_apply_verb # None means "the default, isolated from the host"; an explicit {} means "inherit the # host environment" and must not collapse into the default because it is falsy self.git_env = _ISOLATED_GIT_ENV if git_env is None else git_env @@ -584,14 +609,26 @@ def observe_source(self, coordinate: Coordinate) -> Observation: # -- destination reads (never writes before apply) def read_head(self, destination: Destination) -> str: + # A canonical destination is read at its BRANCH ref, never HEAD: a bare + # repository's HEAD is a symref that may point anywhere, and the thing a + # contribution lands on is the branch the owner declared. + ref = ( + f"refs/heads/{self.branch}" if destination.kind is DestinationKind.CANONICAL else "HEAD" + ) try: - return _git(destination.path, "rev-parse", "HEAD", env=self.git_env) + return _git(destination.path, "rev-parse", "--verify", ref, env=self.git_env) except (subprocess.CalledProcessError, OSError) as exc: raise DestinationUnreadable(f"{destination.destination_id}: {exc}") from exc def _porcelain(self, destination: Destination) -> str: return _git(destination.path, "status", "--porcelain", env=self.git_env) + def _observed_ref_source(self, destination: Destination) -> str: + """The ref the mirror fetches for ahead/behind: the branch for a canonical, else HEAD.""" + if destination.kind is DestinationKind.CANONICAL: + return f"refs/heads/{self.branch}" + return "HEAD" + # -- driver def run( @@ -879,6 +916,19 @@ def _gate(self, gate_id: str, passed: bool, detail: str) -> GateResult: result_hash=_sha256_json({"gate": gate_id, "result": result, "detail": detail}), ) + def _gate_not_run(self, gate_id: str, detail: str) -> GateResult: + """A gate judged inapplicable to this destination kind, recorded as such. + + ``not-run`` is neither pass nor fail: it does not refuse, and it does not let a + reader mistake "the list was empty" for "every gate passed". + """ + return GateResult( + gate_id=gate_id, + result=_GATE_NOT_RUN, + detail=detail, + result_hash=_sha256_json({"gate": gate_id, "result": _GATE_NOT_RUN, "detail": detail}), + ) + def _plan(self, op: _Op) -> bool: assert op.expected_base is not None and op.digest is not None gates: list[GateResult] = [] @@ -902,19 +952,32 @@ def _plan(self, op: _Op) -> bool: ) ) - porcelain = self._porcelain(op.destination) - gates.append( - self._gate( - "destination.clean", - porcelain == "", - "worktree and index clean" - if porcelain == "" - else f"dirty: {len(porcelain.splitlines())} path(s)", + if op.destination.kind is DestinationKind.CANONICAL: + # a bare repository has no worktree; the gate is recorded as NOT RUN rather + # than omitted, so a receipt with an empty gate list and one where this + # gate was judged inapplicable remain distinguishable (a gate list must say + # which gates ran, individually; an aggregate cannot) + gates.append( + self._gate_not_run( + "destination.clean", + "bare canonical destination: cleanliness is not a property it has", + ) + ) + else: + porcelain = self._porcelain(op.destination) + gates.append( + self._gate( + "destination.clean", + porcelain == "", + "worktree and index clean" + if porcelain == "" + else f"dirty: {len(porcelain.splitlines())} path(s)", + ) ) - ) - # the destination's HEAD is fetched INTO the mirror (a read of the destination), - # so ahead/behind is computed in the sink's own store + # the destination's branch (canonical) or HEAD (clone) is fetched INTO the + # mirror (a read of the destination), so ahead/behind is computed in the + # sink's own store observed_tag = hashlib.sha256(op.destination.destination_id.encode()).hexdigest()[:16] observed_ref = f"refs/observed/{observed_tag}" _git( @@ -922,7 +985,7 @@ def _plan(self, op: _Op) -> bool: "fetch", "-q", str(op.destination.path), - f"+HEAD:{observed_ref}", + f"+{self._observed_ref_source(op.destination)}:{observed_ref}", env=self.git_env, ) counts = _git( @@ -1028,15 +1091,90 @@ def _apply(self, op: _Op) -> bool: self._note( op, "apply-verb-started", {"expected_base": head, "intended_after": op.intended_after} ) - _git( - op.destination.path, - "fetch", - "-q", - str(self.mirror), - f"refs/remotes/source/{self.branch}", - env=self.git_env, - ) - _git(op.destination.path, "merge", "-q", "--ff-only", op.intended_after, env=self.git_env) + if self.before_apply_verb is not None: + self.before_apply_verb() + if op.destination.kind is DestinationKind.CANONICAL: + # Prototype 2 (the contribution protocol): the contribution lands by a push FROM + # the mirror (which holds the fetched source objects) TO the owner's bare remote, as a + # fast-forward of ``expected_base`` guarded by ``--force-with-lease`` on + # that same base. The receiving repository enforces the compare-and-swap: + # if its branch is no longer at ``expected_base`` when the push arrives — + # including a move inside the window after this process's own head check — + # the push is rejected and nothing lands. That rejection is a REFUSAL with + # the observed base, not an exception: it is the protocol doing its job. + # Why ``--force-with-lease`` can never FORCE here: the plan's + # ``destination.fast-forward`` gate already established that ``intended_after`` + # descends from ``expected_base`` (ahead == 0), so when the lease holds the + # update is a genuine fast-forward, and when the owner moved the lease rejects + # it before any update. Mutation-checked: forcing the gate to pass lets a stale + # contribution overwrite the owner (W2b goes red); dropping the lease for a bare + # ``--force`` lets a move inside the check→push window be overwritten (W2d goes + # red). The two halves are one mechanism and neither is sufficient alone. + lease = f"--force-with-lease=refs/heads/{self.branch}:{op.expected_base}" + push = subprocess.run( + [ + "git", + "-C", + str(self.mirror), + "push", + "-q", + lease, + str(op.destination.path), + f"{op.intended_after}:refs/heads/{self.branch}", + ], + capture_output=True, + text=True, + env={**os.environ, **self.git_env}, + ) + if push.returncode != 0: + try: + moved_to = self.read_head(op.destination) + except DestinationUnreadable as exc: + self._record( + op, + State.UNVERIFIABLE, + { + "detail": ( + "the lease push returned non-zero and the destination could " + f"not be read back: {exc}" + ), + "observed_base": None, + }, + ) + return False + if moved_to != op.expected_base: + self._record( + op, + State.REFUSED, + { + "refusal_reason": ( + f"destination.lease-refused: the owner's branch moved to " + f"{moved_to} after the base check; expected {op.expected_base}" + ), + "observed_base": moved_to, + "established_by": ( + "the receiving repository rejected the lease; read back names " + "the revision it holds" + ), + }, + ) + return False + raise RuntimeError( + "lease push failed although the owner's branch is still at the expected " + f"base: {push.stderr.strip()}" + ) + else: + _git( + op.destination.path, + "fetch", + "-q", + str(self.mirror), + f"refs/remotes/source/{self.branch}", + env=self.git_env, + ) + _git( + op.destination.path, "merge", "-q", "--ff-only", op.intended_after, env=self.git_env + ) if self.after_apply_verb is not None: self.after_apply_verb() if self.kill_after == KILL_AFTER_APPLY_VERB: @@ -1088,19 +1226,25 @@ def _verify(self, op: _Op) -> bool: assert op.intended_after is not None and op.digest is not None head = self.read_head(op.destination) tree = tree_digest(op.destination.path, head, env=self.git_env) - porcelain = self._porcelain(op.destination) - holds = head == op.intended_after and tree == op.digest and porcelain == "" - detail = f"head={head} tree={tree} clean={porcelain == ''}" + if op.destination.kind is DestinationKind.CANONICAL: + postcondition = _POSTCONDITION_CANONICAL + holds = head == op.intended_after and tree == op.digest + detail = f"branch={head} tree={tree}" + else: + postcondition = _POSTCONDITION + porcelain = self._porcelain(op.destination) + holds = head == op.intended_after and tree == op.digest and porcelain == "" + detail = f"head={head} tree={tree} clean={porcelain == ''}" if not holds: self._note( - op, "postcondition-failed", {"postcondition": _POSTCONDITION, "detail": detail} + op, "postcondition-failed", {"postcondition": postcondition, "detail": detail} ) return False self._record( op, State.VERIFIED, { - "postcondition_checked": _POSTCONDITION, + "postcondition_checked": postcondition, "holds": True, "detail": detail, "established_by": "the named postcondition re-read from the destination", diff --git a/gr2/tests/test_propagation_contribution.py b/gr2/tests/test_propagation_contribution.py new file mode 100644 index 0000000..fa26bbc --- /dev/null +++ b/gr2/tests/test_propagation_contribution.py @@ -0,0 +1,420 @@ +"""Prototype 2: the contribution protocol on the Prototype 0 machine. + +Everything here runs against throwaway repositories under ``tmp_path``. No real +workspace, remote, or authoring clone is touched. The synthetic topology is a +scratch PARENT and two SUBSPACES: + +* one bare ``owner.git`` — the parent layer's CANONICAL remote for ``main``, the + surface the children do not own and may only change by contributing +* two child clones, ``child-a`` and ``child-b``, each an independent authoring + clone of that canonical (independent clones are the product) + +A contribution is a state-machine operation with ``direction=up``: the machine observes the +child's branch, fetches it into the sink's mirror, plans against the owner's +branch as read NOW, and lands it by a fast-forward push guarded by +``--force-with-lease`` on the expected base — compare-and-swap enforced by the +receiving repository, not by this process. What the witnesses prove: + +* W2a a contribution walks observed -> fetched -> planned -> applied -> verified + -> acknowledged; the owner's branch reports the child's revision; the + receipt names exact revisions; the cleanliness gate is recorded NOT RUN + (a bare remote has no worktree), never silently omitted +* W2b the manufactured collision: after A lands, B's change (authored against + the OLD base) is REFUSED at plan — not a fast-forward of the observed base + — with the observed base in the receipt; the owner's refs and B's clone are + byte-for-byte untouched +* W2c the compare-and-swap race: B plans while the owner is at the old base, the + sink dies, A lands, B resumes — and B is REFUSED at apply because the + expected base moved; nothing of B's reaches the owner +* W2d the lease closes the window the process-side check cannot see: the owner + moves AFTER B's apply-step head check and BEFORE B's push; the receiving + repository rejects the lease; B is REFUSED with the revision the owner + holds; B's bytes never land +* W2e replan is the AUTHOR's act: B rebases onto the owner's branch and + re-contributes; the new revision is a fresh attempt and lands; the journal + carries the refused attempt and the acknowledged one, both by name +* W2f policy governs ``up`` exactly as it governs ``down``: a policy that does + not allow ``up`` refuses at ``policy.direction`` before any verb +""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest +from gr2.prototypes.propagation_state_machine import ( + Coordinate, + Destination, + DestinationKind, + Direction, + Operation, + Policy, + Propagator, + SinkKilled, + State, +) + +_GIT_ENV = { + "GIT_AUTHOR_NAME": "prototype", + "GIT_AUTHOR_EMAIL": "prototype@example.invalid", + "GIT_COMMITTER_NAME": "prototype", + "GIT_COMMITTER_EMAIL": "prototype@example.invalid", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", +} + + +def git(repo: Path, *args: str) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + return proc.stdout.strip() + + +def bare_refs(repo: Path) -> str: + return git(repo, "for-each-ref", "--format=%(refname) %(objectname)") + + +def clone_snapshot(repo: Path) -> dict[str, str]: + return { + "refs": git(repo, "for-each-ref", "--format=%(refname) %(objectname)"), + "head": git(repo, "rev-parse", "HEAD"), + "porcelain": git(repo, "status", "--porcelain"), + "canon": (repo / "canon.md").read_text(), + "reflog": git(repo, "reflog", "show", "--format=%gs", "HEAD"), + } + + +@dataclass +class Parent: + """A scratch parent layer: its canonical remote plus two subspace clones.""" + + owner: Path + base: str + root: Path + + def child(self, name: str) -> Path: + path = self.root / name + subprocess.run( + ["git", "clone", "-q", str(self.owner), str(path)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + return path + + def sink(self, name: str) -> Path: + return self.root / f"sink-{name}" + + +def author(child: Path, text: str, *, path: str = "canon.md") -> str: + (child / path).write_text(text) + git(child, "add", path) + git(child, "commit", "-q", "-m", f"{path}: {text.strip()[:40]}") + return git(child, "rev-parse", "HEAD") + + +@pytest.fixture +def parent(tmp_path: Path) -> Parent: + owner = tmp_path / "owner.git" + subprocess.run( + ["git", "init", "-q", "--bare", "--initial-branch=main", str(owner)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + seed = tmp_path / "seed" + subprocess.run( + ["git", "clone", "-q", str(owner), str(seed)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + git(seed, "switch", "-q", "-c", "main") + (seed / "canon.md").write_text("canon v1\n") + (seed / "other.md").write_text("other v1\n") + git(seed, "add", "canon.md", "other.md") + git(seed, "commit", "-q", "-m", "canon v1") + git(seed, "push", "-q", "-u", "origin", "main") + base = git(seed, "rev-parse", "HEAD") + return Parent(owner=owner, base=base, root=tmp_path) + + +POLICY_UP = Policy(policy_hash="policy-prototype-2", allowed_directions=frozenset({Direction.UP})) +POLICY_DOWN_ONLY = Policy( + policy_hash="policy-prototype-2-down-only", allowed_directions=frozenset({Direction.DOWN}) +) + + +def contribution(child_id: str) -> Coordinate: + return Coordinate( + source=child_id, + destination="owner-canonical", + layer="layer-parent", + direction=Direction.UP, + operation=Operation.CONTRIBUTE, + artifact_class="config", + ) + + +def canonical(parent: Parent) -> Destination: + return Destination( + destination_id="owner-canonical", path=parent.owner, kind=DestinationKind.CANONICAL + ) + + +def contributor( + parent: Parent, child: Path, name: str, *, policy: Policy = POLICY_UP, **kw +) -> Propagator: + """The sink that carries ONE child's contributions: its source is that child's clone.""" + return Propagator( + source_remote=child, branch="main", state_dir=parent.sink(name), policy=policy, **kw + ) + + +def owner_main(parent: Parent) -> str: + return git(parent.owner, "rev-parse", "--verify", "refs/heads/main") + + +# --------------------------------------------------------------------------- W2a + + +def test_a_contribution_lands_on_the_owner_by_lease_push_and_names_exact_revisions( + parent: Parent, +) -> None: + child_a = parent.child("child-a") + new = author(child_a, "canon v2 (a)\n") + + receipt = contributor(parent, child_a, "a").run(contribution("child-a"), canonical(parent)) + + assert receipt is not None + assert receipt.state is State.ACKNOWLEDGED + assert [t.state for t in receipt.transitions] == [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + State.APPLIED, + State.VERIFIED, + State.ACKNOWLEDGED, + ] + # the owner's BRANCH reports the child's revision: read back from the owner, not the verb + assert owner_main(parent) == new + assert receipt.source_rev == new + assert receipt.expected_base == parent.base + assert receipt.after == new + # the cleanliness gate is recorded NOT RUN for a bare destination, never omitted + by_id = {g.gate_id: g for g in receipt.gate_results} + assert by_id["destination.clean"].result == "not-run" + assert by_id["destination.base-unmoved"].result == "pass" + assert by_id["destination.fast-forward"].result == "pass" + assert by_id["policy.direction"].result == "pass" + verified = next(t for t in receipt.transitions if t.state is State.VERIFIED) + assert verified.observation["postcondition_checked"] == ( + "branch-is-intended-after-and-tree-matches-digest" + ) + applied = next(t for t in receipt.transitions if t.state is State.APPLIED) + assert applied.observation["verb_ran_now"] is True + + +# --------------------------------------------------------------------------- W2b + + +def test_the_manufactured_collision_refuses_the_stale_contribution_and_touches_nothing( + parent: Parent, +) -> None: + child_a = parent.child("child-a") + child_b = parent.child("child-b") + a_new = author(child_a, "canon v2 (a)\n") + b_new = author(child_b, "canon v2 (b)\n") # same logical path, authored against the OLD base + + assert ( + contributor(parent, child_a, "a").run(contribution("child-a"), canonical(parent)).state + is State.ACKNOWLEDGED + ) + assert owner_main(parent) == a_new + + owner_before = bare_refs(parent.owner) + b_before = clone_snapshot(child_b) + + receipt = contributor(parent, child_b, "b").run(contribution("child-b"), canonical(parent)) + + assert receipt is not None + assert receipt.state is State.REFUSED + # refused at PLAN: the owner's branch is not an ancestor of B's revision + assert [t.state for t in receipt.transitions] == [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + State.REFUSED, + ] + refused = receipt.transitions[-1] + assert "destination.fast-forward" in refused.observation["failed_gates"] + assert refused.observation["observed_base"] == a_new + assert receipt.source_rev == b_new + # nothing merged, nothing forced, nothing written: owner refs and B's clone unchanged + assert bare_refs(parent.owner) == owner_before + assert clone_snapshot(child_b) == b_before + assert owner_main(parent) == a_new + + +# --------------------------------------------------------------------------- W2c + + +def test_compare_and_swap_race_refuses_at_apply_when_the_base_moved_after_plan( + parent: Parent, +) -> None: + child_a = parent.child("child-a") + child_b = parent.child("child-b") + a_new = author(child_a, "canon v2 (a)\n") + b_new = author(child_b, "other v2 (b)\n", path="other.md") # DISJOINT paths, same base + + # B plans while the owner is still at the base, then the sink dies + with pytest.raises(SinkKilled): + contributor(parent, child_b, "b", kill_after=State.PLANNED).run( + contribution("child-b"), canonical(parent) + ) + # A lands in between + assert ( + contributor(parent, child_a, "a").run(contribution("child-a"), canonical(parent)).state + is State.ACKNOWLEDGED + ) + assert owner_main(parent) == a_new + + # B resumes the SAME operation from planned: apply re-reads the base and refuses + receipt = contributor(parent, child_b, "b").run(contribution("child-b"), canonical(parent)) + + assert receipt is not None + assert receipt.state is State.REFUSED + refused = receipt.transitions[-1] + assert str(refused.observation["refusal_reason"]).startswith("expected_base moved") + assert refused.observation["observed_base"] == a_new + assert receipt.expected_base == parent.base + assert receipt.source_rev == b_new + # the owner still holds A's revision and nothing of B's is reachable from it + assert owner_main(parent) == a_new + contains = subprocess.run( + ["git", "-C", str(parent.owner), "merge-base", "--is-ancestor", b_new, "refs/heads/main"], + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + assert contains.returncode != 0 # B's revision is NOT an ancestor of the owner's branch + + +# --------------------------------------------------------------------------- W2d + + +def test_the_lease_refuses_a_move_inside_the_window_after_the_head_check(parent: Parent) -> None: + child_a = parent.child("child-a") + child_b = parent.child("child-b") + a_new = author(child_a, "canon v2 (a)\n") + b_new = author(child_b, "other v2 (b)\n", path="other.md") + + def another_writer_lands() -> None: + # between B's apply-step head check and B's push, A pushes straight to the owner + git(child_a, "push", "-q", "origin", "main") + assert owner_main(parent) == a_new + + receipt = contributor(parent, child_b, "b", before_apply_verb=another_writer_lands).run( + contribution("child-b"), canonical(parent) + ) + + assert receipt is not None + assert receipt.state is State.REFUSED + refused = receipt.transitions[-1] + assert str(refused.observation["refusal_reason"]).startswith("destination.lease-refused") + assert refused.observation["observed_base"] == a_new + # the plan was computed against the base, the apply verb was attempted, the lease held the line + assert receipt.expected_base == parent.base + assert [t.state for t in receipt.transitions] == [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + State.REFUSED, + ] + assert owner_main(parent) == a_new + contains = subprocess.run( + ["git", "-C", str(parent.owner), "merge-base", "--is-ancestor", b_new, "refs/heads/main"], + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + assert contains.returncode != 0 + + +# --------------------------------------------------------------------------- W2e + + +def test_replan_is_the_authors_act_and_the_rebased_contribution_lands_as_a_fresh_attempt( + parent: Parent, +) -> None: + child_a = parent.child("child-a") + child_b = parent.child("child-b") + a_new = author(child_a, "canon v2 (a)\n") + b_stale = author(child_b, "other v2 (b)\n", path="other.md") + assert ( + contributor(parent, child_a, "a").run(contribution("child-a"), canonical(parent)).state + is State.ACKNOWLEDGED + ) + + sink_b = contributor(parent, child_b, "b") + first = sink_b.run(contribution("child-b"), canonical(parent)) + assert first is not None and first.state is State.REFUSED + assert first.source_rev == b_stale + + # the AUTHOR replans: rebase B onto the owner's branch (the machine never did this) + git(child_b, "fetch", "-q", "origin") + git(child_b, "rebase", "-q", "origin/main") + b_rebased = git(child_b, "rev-parse", "HEAD") + assert b_rebased != b_stale + assert git(child_b, "merge-base", "--is-ancestor", a_new, b_rebased) == "" + + second = sink_b.run(contribution("child-b"), canonical(parent)) + + assert second is not None + assert second.state is State.ACKNOWLEDGED + assert second.source_rev == b_rebased + assert second.expected_base == a_new + assert owner_main(parent) == b_rebased + # both attempts are in the journal by name: the refused one at the stale revision, + # the acknowledged one at the rebased revision — a refusal describes a moment + assert ( + sink_b.journal.find(contribution("child-b").key(), b_stale)[-1][-1].state is State.REFUSED + ) + assert ( + sink_b.journal.find(contribution("child-b").key(), b_rebased)[-1][-1].state + is State.ACKNOWLEDGED + ) + # the owner's tree now carries BOTH changes, because the author's rebase composed them + assert git(parent.owner, "show", "refs/heads/main:canon.md") == "canon v2 (a)" + assert git(parent.owner, "show", "refs/heads/main:other.md") == "other v2 (b)" + + +# --------------------------------------------------------------------------- W2f + + +def test_policy_that_does_not_allow_up_refuses_before_any_verb(parent: Parent) -> None: + child_a = parent.child("child-a") + author(child_a, "canon v2 (a)\n") + owner_before = bare_refs(parent.owner) + + receipt = contributor(parent, child_a, "a", policy=POLICY_DOWN_ONLY).run( + contribution("child-a"), canonical(parent) + ) + + assert receipt is not None + assert receipt.state is State.REFUSED + refused = receipt.transitions[-1] + assert "policy.direction" in refused.observation["failed_gates"] + assert bare_refs(parent.owner) == owner_before + assert owner_main(parent) == parent.base From e2b1f1848718a1581fe42501855a560d2d735deb Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 16:19:54 -0500 Subject: [PATCH 17/29] =?UTF-8?q?feat(gr2):=20Prototype=202=20protocol=20?= =?UTF-8?q?=E2=80=94=20ownership=20as=20a=20recorded=20fact,=20contributio?= =?UTF-8?q?n=20sets,=20retire=20refusal,=20append=20surfaces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The machine lands ONE contribution. This adds the protocol around it, each piece the smallest shape its witness needs: - the state machine's run() takes stop_after=PLANNED: drive an operation through its gates and stop BEFORE any verb, journaled at planned; a later run resumes it. This is how a set prepares every member against its owner's current base before landing any of them - ResolvedManifest / ResolvedEntry: every resolved entry carries declared_by and overridden_by; owner is the override or the declaration; classify() answers by longest declared prefix and refuses a path under no entry; two layers declaring the same entry without override is a NAMED ResolutionCollision, never a silent precedence. Today's resolver flattens this away, so the dataclass is the field the resolver must grow and the witnesses run against the stub - ContributionSet: prepare() every member (no verb), land() in declared order, STOP at the first member that does not acknowledge; the set receipt names the landed, the refusing, and the not-attempted members; nothing is rolled back (history is forward-only; a rollback would be a new forward operation) - Subspace.retire() refuses while any contribution is open (not acknowledged and not explicitly abandoned) and lists their operation ids; abandon() is a note in the contribution's own journal, so a change can never simply evaporate - AppendSurface: one guarded append point (exclusive lock across write, flush, fsync), arrival-ordered sequence numbers, no expected base because appends commute BY DECLARATION; file-level only, never a git-tracked path Eleven witnesses in gr2/tests/test_contribution_protocol.py (W1 x3, W4 x4, W5 x2, W6 x2). Eight mutations, each killed by its own witness: refused-is-terminal, retire-ignores-open, land-never-stops, prepare-lands, collision-by-precedence, owner-ignores-override, append-caches-seq, classify-first-match. Two of those needed the harness tightened first: one mutation had not actually applied (the harness now asserts the mutated file differs from the saved copy before running), and the classify fixture's declaration order made first-match and longest-prefix agree, so it was reordered until they disagree. All four propagation suites: 100 passed, 1 xfailed. Co-Authored-By: Claude --- gr2/prototypes/contribution_protocol.py | 529 ++++++++++++++++++++ gr2/prototypes/propagation_state_machine.py | 22 +- gr2/tests/test_contribution_protocol.py | 466 +++++++++++++++++ 3 files changed, 1014 insertions(+), 3 deletions(-) create mode 100644 gr2/prototypes/contribution_protocol.py create mode 100644 gr2/tests/test_contribution_protocol.py diff --git a/gr2/prototypes/contribution_protocol.py b/gr2/prototypes/contribution_protocol.py new file mode 100644 index 0000000..64bc12d --- /dev/null +++ b/gr2/prototypes/contribution_protocol.py @@ -0,0 +1,529 @@ +"""Prototype 2: the contribution protocol around the propagation state machine. + +The machine (``propagation_state_machine``) lands ONE contribution on a canonical +remote by compare-and-swap. This module is the protocol that decides WHICH owner a +change is proposed to, lands SEVERAL contributions that must go together, refuses +to RETIRE a workspace that still holds open contributions, and gives declared +append-only surfaces the one behaviour they are allowed that everything else is +not: two writers, both land, neither replans. + +Everything here is neutral: layer refs, destination ids, and surface names are +opaque strings the caller resolves. The module holds no notion of who an agent or +an org is, and who MAY contribute where is a policy it is handed, never derives. + +Four pieces, each the smallest shape that its witness needs: + +``ResolvedManifest`` / ``ResolvedEntry`` + Ownership is a RECORDED fact, never a guess over path shape. Every resolved + entry carries ``declared_by`` (the layer whose declaration put it in the + workspace), ``overridden_by`` (the layer whose override currently governs it, + if any) and ``write_mode`` (``own`` / ``contribute`` / ``append`` / ``read``). + ``owner()`` is ``overridden_by or declared_by``; ``classify()`` answers, for a + path, whether a change is local authoring, a contribution (and to whom), an + append, or refused. Two layers declaring the same entry without an override is + a NAMED collision at resolution, not a silent precedence. + + Today's resolver FLATTENS this information away (it merges repos by name and + records no provenance), so this dataclass is the specification of the field + the resolver must grow, and the witnesses run against the stub. + +``ContributionSet`` + Several contributions that must land together land in DECLARED ORDER with one + receipt per member and one set receipt naming what landed — all-or-REPORT, + not all-or-nothing. ``prepare()`` drives every member through its gates and + stops before any verb (``stop_after=PLANNED``); if any member refuses there, + nothing has landed and the set reports it. ``land()`` then resumes each member + in order and STOPS at the first member that does not reach acknowledged, + leaving earlier members landed (git history is forward-only; a rollback would + be a new forward operation, never an un-push) and naming the landed, the + refusing, and the not-attempted members in the set receipt. + +``Subspace`` + A workspace that holds contributions. ``retire()`` REFUSES while any of its + contributions is not terminal — not acknowledged and not explicitly + abandoned — and lists their operation ids. ``abandon()`` writes an + ``abandoned`` note to the contribution's own journal, so a workspace can never + be retired with a change that simply evaporates. + +``AppendSurface`` + A declared append-only file: one guarded append point (an exclusive lock held + across write + flush + fsync), each record numbered in arrival order. Two + writers both land; no expected base exists because the operation commutes. + This is commutative BY DECLARATION, never inferred from the shape of a change, + and it is a file-level surface: a git-tracked path is never an append surface + to this protocol, because landing two appends there would require merging on + an author's behalf. +""" + +from __future__ import annotations + +import fcntl +import json +import os +from collections.abc import Callable, Iterable, Sequence +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from enum import StrEnum +from pathlib import Path + +from gr2.prototypes.propagation_state_machine import ( + Coordinate, + Destination, + Propagator, + Receipt, + State, +) + +ABANDONED_NOTE = "abandoned" + + +class WriteMode(StrEnum): + OWN = "own" + CONTRIBUTE = "contribute" + APPEND = "append" + READ = "read" + + +class ResolutionCollision(ValueError): + """Two layers declared the same entry and neither said ``override``. + + Named and refused at resolution; never resolved by precedence silently. + """ + + +@dataclass(frozen=True) +class ResolvedEntry: + """One surface of a materialized workspace, with its provenance recorded.""" + + name: str + path: str + declared_by: str + overridden_by: str | None + write_mode: WriteMode + + @property + def owner(self) -> str: + return self.overridden_by or self.declared_by + + +@dataclass(frozen=True) +class Classification: + """What a change to ``path`` is, under this workspace's recorded ownership.""" + + entry: ResolvedEntry | None + mode: WriteMode + owner: str | None + reason: str + + +@dataclass(frozen=True) +class Declaration: + """One layer's declaration of one entry, as the resolver sees it before merging.""" + + layer: str + name: str + path: str + overrides: bool = False + + +class ResolvedManifest: + """Ownership as a lookup. Built by ``resolve``; read by ``classify``.""" + + def __init__(self, entries: Iterable[ResolvedEntry]) -> None: + self._entries = {e.name: e for e in entries} + + @classmethod + def resolve( + cls, this_layer: str, declarations: Sequence[Declaration], *, appends: Iterable[str] = () + ) -> ResolvedManifest: + """Merge layer declarations (ancestors first) with provenance kept. + + A later declaration of a name already declared is a ``ResolutionCollision`` + unless it says ``overrides``; an override records ``overridden_by`` and keeps + ``declared_by``. Names in ``appends`` are declared append-only by their owner. + """ + merged: dict[str, ResolvedEntry] = {} + append_names = set(appends) + for d in declarations: + prior = merged.get(d.name) + if prior is None: + merged[d.name] = ResolvedEntry( + name=d.name, + path=d.path, + declared_by=d.layer, + overridden_by=None, + write_mode=WriteMode.READ, + ) + continue + if not d.overrides: + raise ResolutionCollision( + f"{d.name!r} declared by {prior.declared_by!r} at {prior.path!r} and again by " + f"{d.layer!r} at {d.path!r} without override" + ) + merged[d.name] = ResolvedEntry( + name=d.name, + path=d.path, + declared_by=prior.declared_by, + overridden_by=d.layer, + write_mode=WriteMode.READ, + ) + entries = [] + for e in merged.values(): + if e.name in append_names: + mode = WriteMode.APPEND + elif e.owner == this_layer: + mode = WriteMode.OWN + else: + mode = WriteMode.CONTRIBUTE + entries.append( + ResolvedEntry( + name=e.name, + path=e.path, + declared_by=e.declared_by, + overridden_by=e.overridden_by, + write_mode=mode, + ) + ) + return cls(entries) + + def entry(self, name: str) -> ResolvedEntry: + return self._entries[name] + + def owner(self, name: str) -> str: + return self._entries[name].owner + + def classify(self, path: str) -> Classification: + """The longest declared path prefix wins; a path under no entry is refused (READ).""" + best: ResolvedEntry | None = None + for e in self._entries.values(): + root = e.path.rstrip("/") + "/" + if (path == e.path or path.startswith(root)) and ( + best is None or len(e.path) > len(best.path) + ): + best = e + if best is None: + return Classification( + entry=None, + mode=WriteMode.READ, + owner=None, + reason=f"{path!r} is under no declared entry", + ) + return Classification( + entry=best, + mode=best.write_mode, + owner=best.owner, + reason=f"{path!r} is under {best.name!r} declared_by={best.declared_by!r} " + f"overridden_by={best.overridden_by!r}", + ) + + +# --------------------------------------------------------------------------- sets + + +@dataclass(frozen=True) +class SetMember: + member_id: str + coordinate: Coordinate + destination: Destination + propagator: Propagator + + +@dataclass(frozen=True) +class MemberOutcome: + member_id: str + state: str # a State value, or "not-attempted" + receipt: Receipt | None + + def as_dict(self) -> dict[str, object]: + return { + "member_id": self.member_id, + "state": self.state, + "receipt": self.receipt.as_dict() if self.receipt is not None else None, + } + + +@dataclass(frozen=True) +class SetReceipt: + set_id: str + phase: str # "prepared" | "refused-at-prepare" | "landed" | "stopped" + order: tuple[str, ...] + outcomes: tuple[MemberOutcome, ...] + timestamp: str + + @property + def landed(self) -> tuple[str, ...]: + return tuple(o.member_id for o in self.outcomes if o.state == str(State.ACKNOWLEDGED)) + + @property + def refused(self) -> tuple[str, ...]: + return tuple( + o.member_id + for o in self.outcomes + if o.state in (str(State.REFUSED), str(State.UNVERIFIABLE)) + ) + + @property + def not_attempted(self) -> tuple[str, ...]: + return tuple(o.member_id for o in self.outcomes if o.state == "not-attempted") + + def as_dict(self) -> dict[str, object]: + return { + "set_id": self.set_id, + "phase": self.phase, + "order": list(self.order), + "landed": list(self.landed), + "refused": list(self.refused), + "not_attempted": list(self.not_attempted), + "outcomes": [o.as_dict() for o in self.outcomes], + "timestamp": self.timestamp, + } + + +def _now() -> str: + return datetime.now(UTC).isoformat(timespec="microseconds") + + +class ContributionSet: + """Several contributions that must land together: prepare all, land in order, report.""" + + def __init__(self, set_id: str, members: Sequence[SetMember]) -> None: + if not members: + raise ValueError("a contribution set needs at least one member") + ids = [m.member_id for m in members] + if len(set(ids)) != len(ids): + raise ValueError(f"duplicate member ids in set {set_id!r}: {ids}") + self.set_id = set_id + self.members = tuple(members) + + @property + def order(self) -> tuple[str, ...]: + return tuple(m.member_id for m in self.members) + + def prepare(self) -> SetReceipt: + """Drive every member to ``planned`` without running a verb. + + If any member refuses at plan, the set is refused BEFORE anything lands and the + receipt carries every member's evidence. A member whose source has nothing new + (``run`` returned ``None``) is reported as not-attempted: it is not part of this + set's landing. + """ + outcomes: list[MemberOutcome] = [] + for m in self.members: + receipt = m.propagator.run(m.coordinate, m.destination, stop_after=State.PLANNED) + if receipt is None: + outcomes.append(MemberOutcome(m.member_id, "not-attempted", None)) + else: + outcomes.append(MemberOutcome(m.member_id, str(receipt.state), receipt)) + any_refused = any(o.state == str(State.REFUSED) for o in outcomes) + return SetReceipt( + set_id=self.set_id, + phase="refused-at-prepare" if any_refused else "prepared", + order=self.order, + outcomes=tuple(outcomes), + timestamp=_now(), + ) + + def land(self, *, on_member_landed: Callable[[str], None] | None = None) -> SetReceipt: + """Resume each prepared member in order; STOP at the first that does not acknowledge. + + Earlier members stay landed. The receipt names landed / refused / not-attempted + members so the author can resolve forward. Nothing is rolled back. + """ + outcomes: list[MemberOutcome] = [] + stopped = False + for m in self.members: + if stopped: + outcomes.append(MemberOutcome(m.member_id, "not-attempted", None)) + continue + receipt = m.propagator.run(m.coordinate, m.destination) + if receipt is None: + outcomes.append(MemberOutcome(m.member_id, "not-attempted", None)) + continue + outcomes.append(MemberOutcome(m.member_id, str(receipt.state), receipt)) + if receipt.state is State.ACKNOWLEDGED: + if on_member_landed is not None: + on_member_landed(m.member_id) + else: + stopped = True + return SetReceipt( + set_id=self.set_id, + phase="stopped" if stopped else "landed", + order=self.order, + outcomes=tuple(outcomes), + timestamp=_now(), + ) + + +# --------------------------------------------------------------------------- subspaces + + +@dataclass(frozen=True) +class OpenContribution: + coordinate_key: str + source_rev: str + last_state: str + operation_id: str | None + + +class RetireRefused(RuntimeError): + """The subspace holds contributions that are neither acknowledged nor abandoned.""" + + def __init__(self, subspace: str, open_contributions: Sequence[OpenContribution]) -> None: + self.subspace = subspace + self.open_contributions = tuple(open_contributions) + listed = ", ".join( + f"{o.coordinate_key} @ {o.source_rev[:12]} ({o.last_state})" for o in open_contributions + ) + super().__init__(f"{subspace}: {len(open_contributions)} open contribution(s): {listed}") + + +@dataclass +class Subspace: + """A workspace and the contributions authored in it, each carried by its own sink.""" + + name: str + contributions: list[tuple[Coordinate, Propagator]] = field(default_factory=list) + retired: bool = False + + def open_contributions(self) -> list[OpenContribution]: + """Every (coordinate, source revision) attempt whose last state is not terminal. + + Terminal means ACKNOWLEDGED, or REFUSED-and-then-ABANDONED by an explicit note. + A refused attempt with no abandon note is OPEN: the refusal described a moment and + the author has not said what becomes of the change. + """ + found: list[OpenContribution] = [] + for coordinate, propagator in self.contributions: + key = coordinate.key() + revs: dict[str, list[dict[str, object]]] = {} + for row in propagator.journal._rows(): + if row.get("coordinate_key") != key: + continue + revs.setdefault(str(row["source_rev"]), []).append(row) + for source_rev, rows in revs.items(): + states = [str(r["state"]) for r in rows if "state" in r] + last_state = states[-1] if states else "(no transition)" + if last_state == str(State.ACKNOWLEDGED): + continue + if any(r.get("note") == ABANDONED_NOTE for r in rows): + continue + op_ids = [str(r["operation_id"]) for r in rows if r.get("operation_id")] + found.append( + OpenContribution( + coordinate_key=key, + source_rev=source_rev, + last_state=last_state, + operation_id=op_ids[-1] if op_ids else None, + ) + ) + return found + + def abandon(self, coordinate: Coordinate, source_rev: str, *, reason: str) -> None: + """Explicitly give up a contribution: an ``abandoned`` note in its own journal.""" + for c, propagator in self.contributions: + if c.key() != coordinate.key(): + continue + attempts = propagator.journal.find(coordinate.key(), source_rev) + attempt = len(attempts) if attempts else 1 + pending_id = f"{coordinate.key()}@{source_rev}" + if attempts and attempts[-1][-1].operation_id: + pending_id = str(attempts[-1][-1].operation_id) + propagator.journal.note( + pending_id=pending_id, + attempt=attempt, + coordinate_key=coordinate.key(), + source_rev=source_rev, + note=ABANDONED_NOTE, + data={"reason": reason}, + ) + return + raise KeyError(f"{self.name}: no contribution at {coordinate.key()}") + + def retire(self) -> None: + """Refuse while any contribution is open; otherwise mark retired.""" + open_ = self.open_contributions() + if open_: + raise RetireRefused(self.name, open_) + self.retired = True + + +# --------------------------------------------------------------------------- append surfaces + + +@dataclass(frozen=True) +class AppendRecord: + seq: int + writer: str + payload: dict[str, object] + timestamp: str + + +class AppendSurface: + """A declared append-only file with one guarded append point. + + ``append`` takes an exclusive lock on the file, reads the last sequence number, + writes ``seq + 1`` with the record, flushes, fsyncs, and releases. Two writers + interleave in ARRIVAL order and both land; there is no expected base to refuse on + because appends commute. Nothing here ever rewrites an earlier record. + """ + + def __init__(self, path: Path) -> None: + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.touch(exist_ok=True) + + def append(self, writer: str, payload: dict[str, object]) -> AppendRecord: + with open(self.path, "a+", encoding="utf-8") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + handle.seek(0) + last = 0 + for line in handle: + line = line.strip() + if line: + last = int(json.loads(line)["seq"]) + record = AppendRecord( + seq=last + 1, writer=writer, payload=payload, timestamp=_now() + ) + handle.seek(0, os.SEEK_END) + handle.write(json.dumps(asdict(record), sort_keys=True) + "\n") + handle.flush() + os.fsync(handle.fileno()) + return record + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + def records(self) -> list[AppendRecord]: + out: list[AppendRecord] = [] + for line in self.path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + data = json.loads(line) + out.append( + AppendRecord( + seq=int(data["seq"]), + writer=str(data["writer"]), + payload=dict(data["payload"]), + timestamp=str(data["timestamp"]), + ) + ) + return out + + +__all__ = [ + "ABANDONED_NOTE", + "AppendRecord", + "AppendSurface", + "Classification", + "ContributionSet", + "Declaration", + "MemberOutcome", + "OpenContribution", + "ResolutionCollision", + "ResolvedEntry", + "ResolvedManifest", + "RetireRefused", + "SetMember", + "SetReceipt", + "Subspace", + "WriteMode", +] diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index 18dbc12..09a3ec8 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -636,6 +636,8 @@ def run( coordinate: Coordinate, destination: Destination, observation: Observation | None = None, + *, + stop_after: State | None = None, ) -> Receipt | None: """Drive one coordinate. ``None`` means no new source revision: not an operation. @@ -644,8 +646,19 @@ def run( its ``observation`` so the source is not asked twice; the cursor check runs on it all the same, because "nothing new" is exactly the answer that would hide a corrupted sink state, whoever observed. + + ``stop_after=State.PLANNED`` drives the operation through its gates and stops + BEFORE the apply verb, journaled at ``planned``; a later ``run`` resumes it from + there. This is how a contribution SET prepares every member against its owner's + current base before landing any of them (Prototype 2). """ - return self._run(coordinate, destination, report_current=False, observation=observation) + return self._run( + coordinate, + destination, + report_current=False, + observation=observation, + stop_after=stop_after, + ) def _run( self, @@ -654,6 +667,7 @@ def _run( *, report_current: bool, observation: Observation | None = None, + stop_after: State | None = None, ) -> Receipt | None: key = coordinate.key() if observation is None: @@ -709,7 +723,7 @@ def _run( else: op = self._fresh(coordinate, destination, source_rev, observation.cursor, attempt=1) - self._drive(op) + self._drive(op, stop_after=stop_after) return self._receipt(coordinate, source_rev, op.attempt, replayed=False) def _terminal_receipt(self, coordinate: Coordinate, source_rev: str) -> Receipt: @@ -809,7 +823,7 @@ def _resume( # -- the state machine - def _drive(self, op: _Op) -> None: + def _drive(self, op: _Op, *, stop_after: State | None = None) -> None: last = op.transitions[-1].state if op.transitions else None if last is None: self._observe(op) @@ -821,6 +835,8 @@ def _drive(self, op: _Op) -> None: if not self._plan(op): return # refused at plan; journaled last = State.PLANNED + if stop_after is State.PLANNED and last is State.PLANNED: + return # prepared: gates evaluated and journaled, no verb has run if last in (State.PLANNED, State.UNVERIFIABLE): if not self._apply(op): return # refused or unverifiable; journaled diff --git a/gr2/tests/test_contribution_protocol.py b/gr2/tests/test_contribution_protocol.py new file mode 100644 index 0000000..ef201de --- /dev/null +++ b/gr2/tests/test_contribution_protocol.py @@ -0,0 +1,466 @@ +"""Prototype 2, the protocol around the machine: ownership, sets, retirement, appends. + +Everything runs against throwaway repositories and files under ``tmp_path``. The +topology is the W2 one (a scratch parent with a bare canonical remote and child +clones) plus, where a SET needs them, further bare owners so that one set spans +several destinations. What the witnesses prove: + +* W1 ownership is a RECORDED fact: every resolved entry carries ``declared_by`` + and ``overridden_by``; ``owner`` is the override or the declaration; a path + classifies to its longest declared prefix; a path under no entry is refused; + two layers declaring the same entry without ``override`` is a NAMED + collision at resolution, never a silent precedence +* W4 a contribution SET prepares EVERY member through its gates before any verb + runs, then lands in declared order; a refusal at prepare lands nothing; a + refusal MID-SET stops the set and the set receipt names the landed, the + refusing, and the not-attempted members; nothing is rolled back +* W5 a subspace REFUSES to retire while any of its contributions is open; the + refusal lists operation ids; a refused contribution stays open until the + author abandons it explicitly, and the abandon is a note in the + contribution's own journal +* W6 a declared append-only surface: two writers both land, in arrival order, + with no expected base to refuse on; earlier records are never rewritten; + a second handle on the same file sees the first handle's records +""" + +from __future__ import annotations + +import os +import subprocess +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest +from gr2.prototypes.contribution_protocol import ( + ABANDONED_NOTE, + AppendSurface, + ContributionSet, + Declaration, + ResolutionCollision, + ResolvedManifest, + RetireRefused, + SetMember, + Subspace, + WriteMode, +) +from gr2.prototypes.propagation_state_machine import ( + Coordinate, + Destination, + DestinationKind, + Direction, + Operation, + State, +) +from gr2.tests.test_propagation_contribution import ( + _GIT_ENV, + Parent, + author, + canonical, + contribution, + contributor, + git, + owner_main, +) + +# --------------------------------------------------------------------------- helpers + + +def make_owner(root: Path, name: str) -> Parent: + """Another bare canonical with the same seed shape, so one set can span owners.""" + owner = root / f"{name}.git" + subprocess.run( + ["git", "init", "-q", "--bare", "--initial-branch=main", str(owner)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + seed = root / f"seed-{name}" + subprocess.run( + ["git", "clone", "-q", str(owner), str(seed)], + check=True, + capture_output=True, + text=True, + env={**os.environ, **_GIT_ENV}, + ) + git(seed, "switch", "-q", "-c", "main") + (seed / "canon.md").write_text(f"{name} canon v1\n") + (seed / "other.md").write_text(f"{name} other v1\n") + git(seed, "add", "canon.md", "other.md") + git(seed, "commit", "-q", "-m", f"{name} canon v1") + git(seed, "push", "-q", "-u", "origin", "main") + return Parent(owner=owner, base=git(seed, "rev-parse", "HEAD"), root=root) + + +@pytest.fixture +def parent(tmp_path: Path) -> Parent: + """The W2 parent shape, built by the same helper the extra owners use.""" + return make_owner(tmp_path, "owner") + + +def coordinate(child_id: str, destination_id: str) -> Coordinate: + return Coordinate( + source=child_id, + destination=destination_id, + layer="layer-parent", + direction=Direction.UP, + operation=Operation.CONTRIBUTE, + artifact_class="config", + ) + + +def member(owner: Parent, child: Path, member_id: str) -> SetMember: + """One set member: its own sink carrying ``child``'s branch to ``owner``'s canonical.""" + destination = Destination( + destination_id=f"canonical-{member_id}", path=owner.owner, kind=DestinationKind.CANONICAL + ) + return SetMember( + member_id=member_id, + coordinate=coordinate(f"child-{member_id}", destination.destination_id), + destination=destination, + propagator=contributor(owner, child, member_id), + ) + + +# --------------------------------------------------------------------------- W1 + + +def _declarations() -> list[Declaration]: + # ancestors first; the SHORTER path is declared before the longer one nested under it, + # so a first-match classifier and a longest-prefix classifier give different answers + return [ + Declaration(layer="org", name="canon", path="config/canon.md"), + Declaration(layer="org", name="notes", path="team"), + Declaration(layer="team", name="ledger", path="team/ledger"), + Declaration(layer="team", name="canon", path="config/canon.md", overrides=True), + ] + + +def test_w1_ownership_is_recorded_provenance_and_owner_is_override_or_declaration() -> None: + as_agent = ResolvedManifest.resolve("agent", _declarations()) + + canon = as_agent.entry("canon") + assert canon.declared_by == "org" + assert canon.overridden_by == "team" + assert canon.owner == "team" # the override governs; the declaration is still recorded + assert canon.write_mode is WriteMode.CONTRIBUTE + notes = as_agent.entry("notes") + assert (notes.declared_by, notes.overridden_by, notes.owner) == ("org", None, "org") + assert as_agent.owner("ledger") == "team" + + # the SAME declarations resolved AS the team: what the team owns is authored, not contributed + as_team = ResolvedManifest.resolve("team", _declarations()) + assert as_team.entry("canon").write_mode is WriteMode.OWN + assert as_team.entry("ledger").write_mode is WriteMode.OWN + assert as_team.entry("notes").write_mode is WriteMode.CONTRIBUTE + + # append-only is DECLARED by name, never inferred + with_append = ResolvedManifest.resolve("agent", _declarations(), appends=["ledger"]) + assert with_append.entry("ledger").write_mode is WriteMode.APPEND + assert as_agent.entry("ledger").write_mode is WriteMode.CONTRIBUTE + + +def test_w1_classify_picks_the_longest_declared_prefix_and_refuses_undeclared_paths() -> None: + manifest = ResolvedManifest.resolve("agent", _declarations()) + + canon = manifest.classify("config/canon.md") + assert canon.mode is WriteMode.CONTRIBUTE + assert canon.owner == "team" + assert canon.entry is not None and canon.entry.name == "canon" + + nested = manifest.classify("team/ledger/2026.jsonl") # under BOTH team/ and team/ledger + assert nested.entry is not None and nested.entry.name == "ledger" + assert nested.owner == "team" + sibling = manifest.classify("team/readme.md") + assert sibling.entry is not None and sibling.entry.name == "notes" + assert sibling.owner == "org" + + # a path that merely shares a prefix STRING is not under the entry + assert manifest.classify("teamwork.md").entry is None + unknown = manifest.classify("elsewhere/file.md") + assert unknown.mode is WriteMode.READ + assert unknown.owner is None + assert "under no declared entry" in unknown.reason + + +def test_w1_two_declarations_without_override_are_a_named_collision_not_precedence() -> None: + colliding = [ + Declaration(layer="org", name="canon", path="config/canon.md"), + Declaration(layer="team", name="canon", path="team/canon.md"), + ] + with pytest.raises(ResolutionCollision) as refused: + ResolvedManifest.resolve("agent", colliding) + message = str(refused.value) + # the refusal NAMES both layers and both paths: a reader can act on it + for fragment in ("'org'", "'team'", "config/canon.md", "team/canon.md", "without override"): + assert fragment in message + + # positive control: the same pair WITH an override resolves, provenance intact + resolved = ResolvedManifest.resolve( + "agent", + [ + colliding[0], + Declaration(layer="team", name="canon", path="team/canon.md", overrides=True), + ], + ) + entry = resolved.entry("canon") + assert (entry.declared_by, entry.overridden_by, entry.path) == ("org", "team", "team/canon.md") + + +# --------------------------------------------------------------------------- W4 + + +def test_w4_a_set_prepares_every_member_before_any_verb_then_lands_in_declared_order( + parent: Parent, tmp_path: Path +) -> None: + owner2 = make_owner(tmp_path, "owner2") + a1 = parent.child("a1") + a2 = owner2.child("a2") + new1 = author(a1, "canon v2 (set m1)\n") + new2 = author(a2, "owner2 canon v2 (set m2)\n") + contribution_set = ContributionSet( + "set-1", [member(parent, a1, "m1"), member(owner2, a2, "m2")] + ) + + prepared = contribution_set.prepare() + + assert prepared.phase == "prepared" + assert [o.state for o in prepared.outcomes] == [str(State.PLANNED), str(State.PLANNED)] + # prepared, NOT landed: both owners still at their bases + assert owner_main(parent) == parent.base + assert owner_main(owner2) == owner2.base + for outcome in prepared.outcomes: + assert outcome.receipt is not None + assert [t.state for t in outcome.receipt.transitions] == [ + State.OBSERVED, + State.FETCHED, + State.PLANNED, + ] + + landed_order: list[str] = [] + landed = contribution_set.land(on_member_landed=landed_order.append) + + assert landed.phase == "landed" + assert landed.landed == ("m1", "m2") + assert landed_order == ["m1", "m2"] # declared order, not arrival order + assert landed.refused == () and landed.not_attempted == () + assert owner_main(parent) == new1 + assert owner_main(owner2) == new2 + # each member's receipt is the RESUMED attempt: one journaled attempt, planned then applied + for outcome in landed.outcomes: + assert outcome.receipt is not None and outcome.receipt.attempt == 1 + assert outcome.receipt.state is State.ACKNOWLEDGED + as_dict = landed.as_dict() + assert as_dict["order"] == ["m1", "m2"] and as_dict["landed"] == ["m1", "m2"] + + +def test_w4_a_refusal_at_prepare_lands_nothing_even_for_the_members_that_were_fine( + parent: Parent, tmp_path: Path +) -> None: + owner2 = make_owner(tmp_path, "owner2") + a1 = parent.child("a1") + a2 = owner2.child("a2") + author(a1, "canon v2 (stale m1)\n") + new2 = author(a2, "owner2 canon v2 (fine m2)\n") + # an interloper lands on owner 1 first, so m1 is stale against the owner's CURRENT base + interloper = parent.child("interloper") + interloper_rev = author(interloper, "canon v2 (interloper)\n") + assert ( + contributor(parent, interloper, "x") + .run(contribution("interloper"), canonical(parent)) + .state + is State.ACKNOWLEDGED + ) # type: ignore[union-attr] + + contribution_set = ContributionSet( + "set-2", [member(parent, a1, "m1"), member(owner2, a2, "m2")] + ) + prepared = contribution_set.prepare() + + assert prepared.phase == "refused-at-prepare" + assert prepared.refused == ("m1",) + m1, m2 = prepared.outcomes + assert m1.state == str(State.REFUSED) and m1.receipt is not None + assert "destination.fast-forward" in [ + g.gate_id for g in m1.receipt.gate_results if g.result == "fail" + ] + assert m1.receipt.observed_base == interloper_rev + assert m2.state == str(State.PLANNED) # m2 was fine, and it did NOT land + assert owner_main(owner2) == owner2.base + assert owner_main(parent) == interloper_rev + assert new2 != owner2.base + + +def test_w4_a_mid_set_refusal_stops_the_set_and_reports_landed_refused_not_attempted( + parent: Parent, tmp_path: Path +) -> None: + owner2 = make_owner(tmp_path, "owner2") + owner3 = make_owner(tmp_path, "owner3") + a1, a2, a3 = parent.child("a1"), owner2.child("a2"), owner3.child("a3") + new1 = author(a1, "canon v2 (m1)\n") + author(a2, "owner2 canon v2 (m2)\n") + new3 = author(a3, "owner3 canon v2 (m3)\n") + contribution_set = ContributionSet( + "set-3", + [member(parent, a1, "m1"), member(owner2, a2, "m2"), member(owner3, a3, "m3")], + ) + assert contribution_set.prepare().phase == "prepared" + + # between prepare and land, owner 2 moves: the compare-and-swap at apply will refuse m2 + interloper = owner2.child("interloper2") + moved_to = author(interloper, "owner2 canon v2 (interloper)\n") + assert ( + contributor(owner2, interloper, "x2") + .run( + coordinate("interloper2", "canonical-x2"), + Destination( + destination_id="canonical-x2", path=owner2.owner, kind=DestinationKind.CANONICAL + ), + ) + .state + is State.ACKNOWLEDGED + ) # type: ignore[union-attr] + + landed = contribution_set.land() + + assert landed.phase == "stopped" + assert landed.landed == ("m1",) + assert landed.refused == ("m2",) + assert landed.not_attempted == ("m3",) + m1, m2, m3 = landed.outcomes + assert m2.receipt is not None and m2.receipt.state is State.REFUSED + assert "expected_base moved" in (m2.receipt.refusal_reason or "") + assert m2.receipt.observed_base == moved_to + assert m3.receipt is None + # nothing rolled back: m1 stays landed; m3 never reached its owner + assert owner_main(parent) == new1 + assert owner_main(owner2) == moved_to + assert owner_main(owner3) == owner3.base + assert new3 != owner3.base + + +def test_w4_a_set_refuses_to_exist_with_no_members_or_duplicate_member_ids( + parent: Parent, +) -> None: + with pytest.raises(ValueError): + ContributionSet("empty", []) + a1 = parent.child("a1") + with pytest.raises(ValueError): + ContributionSet("dup", [member(parent, a1, "m1"), member(parent, a1, "m1")]) + + +# --------------------------------------------------------------------------- W5 + + +def test_w5_retire_refuses_while_a_contribution_is_open_and_names_its_operation_id( + parent: Parent, +) -> None: + child_a = parent.child("child-a") + author(child_a, "canon v2 (a)\n") + sink = contributor(parent, child_a, "a") + coord = contribution("child-a") + prepared = sink.run(coord, canonical(parent), stop_after=State.PLANNED) + assert prepared is not None and prepared.state is State.PLANNED + subspace = Subspace("child-a", [(coord, sink)]) + + with pytest.raises(RetireRefused) as refused: + subspace.retire() + assert subspace.retired is False + [open_] = refused.value.open_contributions + assert open_.coordinate_key == coord.key() + assert open_.source_rev == prepared.source_rev + assert open_.last_state == str(State.PLANNED) + assert open_.operation_id == prepared.operation_id is not None + assert prepared.operation_id in str(refused.value) or open_.source_rev[:12] in str( + refused.value + ) + + # landing the contribution closes it; retire now succeeds + landed = sink.run(coord, canonical(parent)) + assert landed is not None and landed.state is State.ACKNOWLEDGED + assert subspace.open_contributions() == [] + subspace.retire() + assert subspace.retired is True + + +def test_w5_a_refused_contribution_stays_open_until_abandoned_by_an_explicit_note( + parent: Parent, +) -> None: + child_a = parent.child("child-a") + child_b = parent.child("child-b") + author(child_a, "canon v2 (a)\n") + stale = author(child_b, "canon v2 (b)\n") + assert ( + contributor(parent, child_a, "a").run(contribution("child-a"), canonical(parent)).state + is State.ACKNOWLEDGED + ) # type: ignore[union-attr] + sink_b = contributor(parent, child_b, "b") + coord_b = contribution("child-b") + refused = sink_b.run(coord_b, canonical(parent)) + assert refused is not None and refused.state is State.REFUSED + subspace = Subspace("child-b", [(coord_b, sink_b)]) + + # refused is NOT terminal: the author has not said what becomes of the change + with pytest.raises(RetireRefused) as blocked: + subspace.retire() + [open_] = blocked.value.open_contributions + assert (open_.source_rev, open_.last_state) == (stale, str(State.REFUSED)) + + subspace.abandon(coord_b, stale, reason="superseded by child-a's change") + assert sink_b.journal.notes(coord_b.key(), stale, ABANDONED_NOTE) == 1 + assert subspace.open_contributions() == [] + subspace.retire() + assert subspace.retired is True + + # abandoning something the subspace does not hold is an error, not a silent no-op + with pytest.raises(KeyError): + subspace.abandon(contribution("child-z"), stale, reason="nope") + + +# --------------------------------------------------------------------------- W6 + + +def test_w6_two_writers_both_land_in_arrival_order_and_earlier_records_are_never_rewritten( + tmp_path: Path, +) -> None: + path = tmp_path / "ledger" / "events.jsonl" + first_handle = AppendSurface(path) + second_handle = AppendSurface(path) # a second writer on the SAME file, no shared state + + r1 = first_handle.append("a", {"n": 1}) + r2 = second_handle.append("b", {"n": 1}) + r3 = first_handle.append("a", {"n": 2}) + assert [r.seq for r in (r1, r2, r3)] == [1, 2, 3] + assert [(r.writer, r.payload["n"]) for r in first_handle.records()] == [ + ("a", 1), + ("b", 1), + ("a", 2), + ] + # the second handle reads the first handle's records: the file is the state + assert second_handle.records() == first_handle.records() + + # earlier records are a PREFIX of the file after any later append + before = path.read_bytes() + second_handle.append("b", {"n": 2}) + after = path.read_bytes() + assert after.startswith(before) and len(after) > len(before) + assert [r.seq for r in first_handle.records()] == [1, 2, 3, 4] + + +def test_w6_concurrent_writers_produce_one_gapless_sequence(tmp_path: Path) -> None: + path = tmp_path / "ledger" / "events.jsonl" + writers, each = 4, 25 + + def write_many(writer: str) -> None: + surface = AppendSurface(path) + for n in range(each): + surface.append(writer, {"n": n}) + + with ThreadPoolExecutor(max_workers=writers) as pool: + list(pool.map(write_many, [f"w{i}" for i in range(writers)])) + + records = AppendSurface(path).records() + assert [r.seq for r in records] == list(range(1, writers * each + 1)) + for i in range(writers): + mine = [r.payload["n"] for r in records if r.writer == f"w{i}"] + assert mine == list(range(each)) # each writer's own order is preserved From 47937968e251946db33c9dd50869be1e5915af0f Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Wed, 19 Aug 2026 16:23:51 -0500 Subject: [PATCH 18/29] measure(gr2): per-state latency read from the receipts' own timestamps, printed for one landing and a two-member set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit state_latencies(receipt) returns (state, seconds since the previous transition) rows from the receipt's timestamps — the number comes from the artifact, not from a stopwatch around it. One test prints the table (visible with -s, written to a file) for a single contribution and both members of a set. On this host a single contribution lands in ~0.23 s end to end (fetch ~0.09, plan ~0.05, lease push ~0.07, verify ~0.02); for set members the applied row spans prepare -> land by construction, so it reads as the time the prepared base sat, not the push alone. Co-Authored-By: Claude --- gr2/prototypes/contribution_protocol.py | 18 ++++++++ gr2/tests/test_contribution_protocol.py | 55 +++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/gr2/prototypes/contribution_protocol.py b/gr2/prototypes/contribution_protocol.py index 64bc12d..b383a2e 100644 --- a/gr2/prototypes/contribution_protocol.py +++ b/gr2/prototypes/contribution_protocol.py @@ -283,6 +283,23 @@ def _now() -> str: return datetime.now(UTC).isoformat(timespec="microseconds") +def state_latencies(receipt: Receipt) -> list[tuple[str, float]]: + """Per-state latency from the receipt's OWN timestamps, in seconds. + + Each row is ``(state, seconds since the previous transition)``; the first row is + ``(state, 0.0)``. A measurement, not a witness: the receipts already carry the + timestamps, so the number comes from the artifact and not from a stopwatch around + it. Printed so a daemon's ``up`` budget can start from data. + """ + rows: list[tuple[str, float]] = [] + previous: datetime | None = None + for t in receipt.transitions: + at = datetime.fromisoformat(t.timestamp) + rows.append((str(t.state), 0.0 if previous is None else (at - previous).total_seconds())) + previous = at + return rows + + class ContributionSet: """Several contributions that must land together: prepare all, land in order, report.""" @@ -526,4 +543,5 @@ def records(self) -> list[AppendRecord]: "SetReceipt", "Subspace", "WriteMode", + "state_latencies", ] diff --git a/gr2/tests/test_contribution_protocol.py b/gr2/tests/test_contribution_protocol.py index ef201de..5322f91 100644 --- a/gr2/tests/test_contribution_protocol.py +++ b/gr2/tests/test_contribution_protocol.py @@ -42,6 +42,7 @@ SetMember, Subspace, WriteMode, + state_latencies, ) from gr2.prototypes.propagation_state_machine import ( Coordinate, @@ -464,3 +465,57 @@ def write_many(writer: str) -> None: for i in range(writers): mine = [r.payload["n"] for r in records if r.writer == f"w{i}"] assert mine == list(range(each)) # each writer's own order is preserved + + +# --------------------------------------------------------------------------- measurement + + +def test_per_state_latency_is_read_from_the_receipts_own_timestamps_and_printed( + parent: Parent, tmp_path: Path +) -> None: + """Not a witness: the measurement the prototype must print (per-state seconds, from + the receipt's timestamps, for one landing and for a two-member set). Run with ``-s`` + or read the captured table; the assertions only pin that the timestamps are ordered + and that every state of the landing path has a row. For SET members the ``applied`` + row spans prepare -> land by construction (the planned transition is written at + prepare time), so it reads as the time the prepared base sat, not the push alone.""" + child_a = parent.child("child-a") + author(child_a, "canon v2 (a)\n") + single = contributor(parent, child_a, "a").run(contribution("child-a"), canonical(parent)) + assert single is not None and single.state is State.ACKNOWLEDGED + + owner2 = make_owner(tmp_path, "owner2") + b1, b2 = parent.child("b1"), owner2.child("b2") + author(b1, "canon v3 (set)\n") + author(b2, "owner2 canon v2 (set)\n") + contribution_set = ContributionSet( + "timed", [member(parent, b1, "m1"), member(owner2, b2, "m2")] + ) + assert contribution_set.prepare().phase == "prepared" + landed = contribution_set.land() + assert landed.phase == "landed" + + lines = ["per-state latency (seconds since previous transition), from receipt timestamps"] + for label, receipt in [("single", single)] + [ + (o.member_id, o.receipt) for o in landed.outcomes if o.receipt is not None + ]: + rows = state_latencies(receipt) + assert [state for state, _ in rows] == [ + str(State.OBSERVED), + str(State.FETCHED), + str(State.PLANNED), + str(State.APPLIED), + str(State.VERIFIED), + str(State.ACKNOWLEDGED), + ] + assert all(seconds >= 0.0 for _, seconds in rows) + total = sum(seconds for _, seconds in rows) + lines.append( + f" {label:>6}: " + + " ".join(f"{s}={sec:.3f}" for s, sec in rows) + + f" total={total:.3f}" + ) + report = tmp_path / "latency.txt" + report.write_text("\n".join(lines) + "\n") + print("\n" + report.read_text(), end="") # visible with -s; the file is the artifact + assert report.read_text().count("total=") == 3 From 37eced357e817634d0eeac9454be2cf7ce9132cf Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Thu, 20 Aug 2026 05:35:21 -0500 Subject: [PATCH 19/29] fix(gr2): a torn journal line stops being fatal, and stops eating its neighbour The journal writer appends and fsyncs, so a kill between the two leaves a partial line. Three consequences, each with its own witness. A tear used to cost TWO rows, not one. Appending onto an unterminated line glues the new row to the remnant, so a row that was written correctly and fsynced becomes unreadable. _write now terminates a remnant before appending, confining the damage to the line actually interrupted. _rows raised a bare JSONDecodeError on any unparseable line. The daemon lets anything outside its named failure list propagate, so one bad moment exited the loop on its first tick and on every restart after it. Lines that cannot be parsed are now dropped and counted, and MalformedLine keeps the drop from being silent. Position does not discriminate cause, which is why the drop is unconditional rather than trailing-only: a tear is trailing only until the daemon restarts and appends again, after which the same orphan sits mid-file with intact rows on both sides. The question that does discriminate is whether the cursor can still be accounted for, and replay already asks it and already raises JournalInconsistent by name. Dropping here lets that check decide instead of being pre-empted by a parse error. Prototype 1 asks the machine to account for an observation on every tick, including idle ones, which moved this parse from once-per-change to once-per-tick. Rows are now cached against (size, mtime_ns); the Propagator is built once per loop and holds one Journal, so the cache spans ticks. The one state it cannot see - an in-place mutation changing neither size nor mtime_ns - is named in the code rather than assumed away. 86 passed, 1 xfailed, up from 83 by exactly the three witnesses added. Each guard is mutation-proven: removing the newline repair reddens the neighbour witness, restoring the bare parse reddens that one and the skip witness both, and disabling the cache reddens the idle-tick witness. Ref #893 - closes at promotion Co-Authored-By: Claude --- gr2/prototypes/propagation_state_machine.py | 92 ++++++++++++++++- gr2/tests/test_propagation_state_machine.py | 105 ++++++++++++++++++++ 2 files changed, 193 insertions(+), 4 deletions(-) diff --git a/gr2/prototypes/propagation_state_machine.py b/gr2/prototypes/propagation_state_machine.py index 7b89cba..c9e2a98 100644 --- a/gr2/prototypes/propagation_state_machine.py +++ b/gr2/prototypes/propagation_state_machine.py @@ -102,6 +102,19 @@ class DestinationUnreadable(RuntimeError): """A read against the destination returned an error instead of an answer.""" +@dataclass(frozen=True) +class MalformedLine: + """A journal line that could not be parsed, kept so a drop is never silent. + + ``index`` counts lines in the file, blank ones included, so it names the line a + reader would count to by hand. + """ + + index: int + excerpt: str + reason: str + + class JournalInconsistent(RuntimeError): """The cursor names a revision the journal cannot account for. @@ -371,23 +384,93 @@ def __init__(self, state_dir: Path) -> None: self.state_dir = state_dir self.path = state_dir / "journal.jsonl" self.cursors_dir = state_dir / "cursors" + # Rows are cached against the file's (size, mtime_ns) because Prototype 1 asks + # the machine to account for an observation on EVERY tick, including idle ones, + # which moved this parse from once-per-change to once-per-tick. The Propagator + # is built once per loop and holds one Journal, so the cache spans ticks. + # + # LIMITATION, stated rather than assumed away: an in-place mutation that changes + # neither size nor mtime_ns is not detected. The append-only writer never + # produces that state, and the next real append invalidates the entry — but a + # cache sitting on a corruption-detection path should name what it cannot see. + self._cache_key: tuple[int, int] | None = None + self._cached_rows: list[dict[str, object]] = [] + self._malformed: tuple[MalformedLine, ...] = () + + @property + def malformed_lines(self) -> tuple[MalformedLine, ...]: + """Lines the last read could not parse. Empty is the ordinary answer.""" + return self._malformed # -- rows def _rows(self) -> list[dict[str, object]]: - if not self.path.exists(): + """Every readable row, oldest first. Unreadable lines are dropped and counted. + + A kill between ``_write``'s write and its fsync leaves a partial line, so an + unparseable line is an ordinary consequence of the writer being killed — not + evidence of corruption, and not a reason to refuse. Raising here would exit the + daemon on its first tick and on every restart after it, because the loop lets + anything outside its named failure list propagate. + + Position does NOT discriminate cause. A tear is trailing only until the daemon + restarts and appends again, after which the same orphan sits mid-file with intact + rows on both sides. The question that does discriminate is whether the CURSOR can + still be accounted for, and ``replay`` already asks it and already raises + ``JournalInconsistent`` by name. Dropping a line here lets that check be the one + that decides, instead of a bare ``ValueError`` from this parse pre-empting it. + + Returned rows are shared with the cache and must be treated as read-only; every + consumer in this module filters or copies rather than mutating. + """ + try: + stat = self.path.stat() + except (FileNotFoundError, NotADirectoryError): + self._cache_key, self._cached_rows, self._malformed = None, [], () return [] + + key = (stat.st_size, stat.st_mtime_ns) + if key == self._cache_key: + return self._cached_rows + rows: list[dict[str, object]] = [] - for line in self.path.read_text().splitlines(): - line = line.strip() + malformed: list[MalformedLine] = [] + for index, raw in enumerate(self.path.read_text().splitlines()): + line = raw.strip() if not line: continue - rows.append(json.loads(line)) + try: + rows.append(json.loads(line)) + except json.JSONDecodeError as exc: + malformed.append( + MalformedLine(index=index, excerpt=line[:120], reason=str(exc)) + ) + self._cache_key, self._cached_rows, self._malformed = key, rows, tuple(malformed) return rows def _write(self, row: dict[str, object]) -> None: + """Append one row, terminating a torn remnant first. + + Without the repair, appending onto a partial line GLUES this row to it, and the + tear costs two rows instead of one: the interrupted write, and the next write, + which was itself correct and fsynced. Terminating first confines the damage to + the line that was actually interrupted. + + Two writers racing can at worst both terminate, leaving a blank line, which + ``_rows`` already skips. + """ self.path.parent.mkdir(parents=True, exist_ok=True) + unterminated = False + try: + with self.path.open("rb") as probe: + if probe.seek(0, os.SEEK_END): + probe.seek(-1, os.SEEK_END) + unterminated = probe.read(1) != b"\n" + except FileNotFoundError: + pass with self.path.open("a") as handle: + if unterminated: + handle.write("\n") handle.write(json.dumps(row, separators=(",", ":")) + "\n") handle.flush() os.fsync(handle.fileno()) @@ -1216,6 +1299,7 @@ def _receipt( "DestinationKind", "DestinationUnreadable", "JournalInconsistent", + "MalformedLine", "Direction", "GateResult", "Journal", diff --git a/gr2/tests/test_propagation_state_machine.py b/gr2/tests/test_propagation_state_machine.py index 37ed862..7a875d4 100644 --- a/gr2/tests/test_propagation_state_machine.py +++ b/gr2/tests/test_propagation_state_machine.py @@ -883,3 +883,108 @@ def test_outbox_event_survives_a_consumer_that_fails_after_reading(tmp_path: Pat again = _offered_types(workspace) if again != ["sync.completed"]: raise EventLost(f"offered once, then lost: second read returned {again!r}") + + +# -- grip#893: a torn journal line must not be fatal, and must not eat its neighbour +# +# The writer appends and fsyncs, so a kill between the write and the fsync leaves +# a partial trailing line. Three witnesses, each for a distinct failure: +# +# W1 the tear must not destroy the NEXT row (measured on the filed issue: with +# no newline repair the following append glues onto the remnant, so a row +# that was written correctly and fsynced becomes unreadable) +# W2 reading a torn journal must not raise; the intact prefix still answers +# W3 an idle tick must not re-parse a journal that has not changed +# +# W1 is the one the issue does not contain and is the reason the position rule in +# it does not hold: after a restart append the malformed line is no longer last. + + +def _tear(journal: Journal, partial: str = '{"kind": "not-final') -> None: + """Leave a partial trailing line, exactly as a kill between write and fsync does.""" + with journal.path.open("a") as handle: + handle.write(partial) + handle.flush() + os.fsync(handle.fileno()) + + +def test_a_torn_line_does_not_swallow_the_row_written_after_it(tmp_path: Path) -> None: + """W1. The append after a tear must not glue onto the remnant. + + Without a newline repair the file becomes ``…{"n": 3, "no{"n": 4}`` — row 4 was + written correctly and fsynced, and is unreadable because row 3 died mid-line. + A tear may cost the interrupted write; it must never cost the next one. + """ + journal = Journal(tmp_path / "state") + journal._write({"n": 1}) + journal._write({"n": 2}) + _tear(journal) + + journal._write({"n": 4}) + + rows = journal._rows() + assert {r.get("n") for r in rows} == {1, 2, 4}, ( + "the row written after the tear must survive it; " + f"file was {journal.path.read_text()!r}" + ) + + +def test_a_torn_trailing_line_is_skipped_and_reported_not_raised(tmp_path: Path) -> None: + """W2. Reading a torn journal answers from the intact prefix and says what it dropped. + + The daemon lets anything outside its named failure list propagate, so a bare + ``JSONDecodeError`` out of the parse exits the loop on the first tick and on + every restart after it. The intact prefix is still a truthful answer. + """ + journal = Journal(tmp_path / "state") + journal._write({"n": 1}) + journal._write({"n": 2}) + _tear(journal) + + rows = journal._rows() + + assert [r.get("n") for r in rows] == [1, 2] + assert len(journal.malformed_lines) == 1, "the drop must be counted, not silent" + assert journal.malformed_lines[0].index == 2 + + +def test_an_unchanged_journal_is_not_reparsed_on_the_next_tick(tmp_path: Path) -> None: + """W3. Prototype 1 consults the machine on every tick, including idle ones. + + That is correct — a corrupted sink behind an unchanged source must not read as + healthy — but it moved the whole-file parse from once-per-change to once-per- + tick. An unchanged file must cost nothing to re-read; a changed one must not + be served from a stale cache. + """ + journal = Journal(tmp_path / "state") + journal._write({"n": 1}) + journal._write({"n": 2}) + + parsed: list[str] = [] + real_loads = json.loads + + def counting_loads(payload, *args, **kwargs): # type: ignore[no-untyped-def] + parsed.append(payload) + return real_loads(payload, *args, **kwargs) + + import gr2.prototypes.propagation_state_machine as module + + original = module.json.loads + module.json.loads = counting_loads + try: + first = journal._rows() + after_first = len(parsed) + second = journal._rows() + after_second = len(parsed) + + journal._write({"n": 3}) + third = journal._rows() + after_third = len(parsed) + finally: + module.json.loads = original + + assert after_first == 2, "the first read parses the file" + assert after_second == after_first, "an unchanged journal must not be re-parsed" + assert first == second + assert after_third > after_second, "a changed journal must not be served stale" + assert [r.get("n") for r in third] == [1, 2, 3] From f051ab8374622ca41ac6d557616a0f16495f761b Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Thu, 20 Aug 2026 06:31:14 -0500 Subject: [PATCH 20/29] fix: AppendSurface survives a torn write, and counts what it skips A writer killed between its record and the terminator leaves a remnant line. Three consequences, all measured before this change: 1. append() raised on the remnant and kept raising: the guarded append point was BRICKED permanently, because every later writer re-reads the whole file to compute the next sequence number and dies on the same line. 2. The next record GLUED onto the remnant, producing one unparseable line and losing a record that had itself completed and fsynced. 3. records() raised, so every reader of the surface died too. Both scans now skip a line they cannot read AND count it (malformed_lines, reflecting the most recent full scan). Skipping alone is silent, and silence is the defect: the remnant sits in the file while a caller sees a healthy-looking surface with a contiguous sequence. append() also repairs a missing terminator before writing, so a completed record is never swallowed by an incomplete one. MalformedLine is reused from the state machine rather than re-declared, so the two append-only surfaces in this tree report a torn line in one vocabulary. Four witnesses, five mutations, each mutation killing witnesses whose failure TYPE matches it: removing the terminator repair kills by AssertionError (glue is wrong content, not an exception); removing either scan's guard kills by JSONDecodeError and KeyError, exactly the types the guard catches; silencing either count kills only that path's count assertions. Tear-fixture count for this surface: 4, previously 0. Ref #897 - closes at promotion Co-Authored-By: Claude --- gr2/prototypes/contribution_protocol.py | 45 +++++++++++++-- gr2/tests/test_contribution_protocol.py | 76 +++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 6 deletions(-) diff --git a/gr2/prototypes/contribution_protocol.py b/gr2/prototypes/contribution_protocol.py index b383a2e..499e5e1 100644 --- a/gr2/prototypes/contribution_protocol.py +++ b/gr2/prototypes/contribution_protocol.py @@ -69,6 +69,7 @@ from gr2.prototypes.propagation_state_machine import ( Coordinate, Destination, + MalformedLine, Propagator, Receipt, State, @@ -482,12 +483,25 @@ class AppendSurface: writes ``seq + 1`` with the record, flushes, fsyncs, and releases. Two writers interleave in ARRIVAL order and both land; there is no expected base to refuse on because appends commute. Nothing here ever rewrites an earlier record. + + A writer killed between ``write`` and its terminator leaves a remnant line. Both + scans SKIP such a line and COUNT it (``malformed_lines``, reflecting the most + recent full scan): an uncounted drop is silent, and this surface is the one place + a caller looks. ``append`` also repairs a missing terminator before writing, so a + remnant never GLUES the next record onto itself — without that repair the record + that follows a torn write is swallowed into an unparseable line and lost. """ def __init__(self, path: Path) -> None: self.path = path self.path.parent.mkdir(parents=True, exist_ok=True) self.path.touch(exist_ok=True) + self._malformed: tuple[MalformedLine, ...] = () + + @property + def malformed_lines(self) -> tuple[MalformedLine, ...]: + """Lines the most recent full scan could not read. Empty is the healthy case.""" + return self._malformed def append(self, writer: str, payload: dict[str, object]) -> AppendRecord: with open(self.path, "a+", encoding="utf-8") as handle: @@ -495,14 +509,26 @@ def append(self, writer: str, payload: dict[str, object]) -> AppendRecord: try: handle.seek(0) last = 0 - for line in handle: - line = line.strip() - if line: - last = int(json.loads(line)["seq"]) + malformed: list[MalformedLine] = [] + raw_last = "" + for index, raw in enumerate(handle): + raw_last = raw + line = raw.strip() + if not line: + continue + try: + last = max(last, int(json.loads(line)["seq"])) + except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + malformed.append( + MalformedLine(index=index, excerpt=line[:120], reason=str(exc)) + ) + self._malformed = tuple(malformed) record = AppendRecord( seq=last + 1, writer=writer, payload=payload, timestamp=_now() ) handle.seek(0, os.SEEK_END) + if raw_last and not raw_last.endswith("\n"): + handle.write("\n") handle.write(json.dumps(asdict(record), sort_keys=True) + "\n") handle.flush() os.fsync(handle.fileno()) @@ -512,8 +538,12 @@ def append(self, writer: str, payload: dict[str, object]) -> AppendRecord: def records(self) -> list[AppendRecord]: out: list[AppendRecord] = [] - for line in self.path.read_text(encoding="utf-8").splitlines(): - if line.strip(): + malformed: list[MalformedLine] = [] + for index, raw in enumerate(self.path.read_text(encoding="utf-8").splitlines()): + line = raw.strip() + if not line: + continue + try: data = json.loads(line) out.append( AppendRecord( @@ -523,6 +553,9 @@ def records(self) -> list[AppendRecord]: timestamp=str(data["timestamp"]), ) ) + except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + malformed.append(MalformedLine(index=index, excerpt=line[:120], reason=str(exc))) + self._malformed = tuple(malformed) return out diff --git a/gr2/tests/test_contribution_protocol.py b/gr2/tests/test_contribution_protocol.py index 5322f91..7097833 100644 --- a/gr2/tests/test_contribution_protocol.py +++ b/gr2/tests/test_contribution_protocol.py @@ -21,10 +21,15 @@ * W6 a declared append-only surface: two writers both land, in arrival order, with no expected base to refuse on; earlier records are never rewritten; a second handle on the same file sees the first handle's records +* W7 a writer killed mid-write leaves a remnant: the append point survives it + (unguarded, it raises FOREVER and the surface is dead), the next record is + not GLUED onto the remnant and lost, and both scans skip the remnant AND + COUNT it, because an uncounted drop is a loss nobody can see """ from __future__ import annotations +import json import os import subprocess from concurrent.futures import ThreadPoolExecutor @@ -467,6 +472,77 @@ def write_many(writer: str) -> None: assert mine == list(range(each)) # each writer's own order is preserved +# --------------------------------------------------------------------------- W7 + + +def _tear(path: Path, remnant: str) -> None: + """A writer killed between its write and its terminator: bytes, no newline.""" + with path.open("a", encoding="utf-8") as handle: + handle.write(remnant) + handle.flush() + os.fsync(handle.fileno()) + + +def test_w7_a_torn_remnant_does_not_brick_the_append_point(tmp_path: Path) -> None: + """Unguarded, append() raises on the remnant on EVERY later call: the surface is dead.""" + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + surface.append("a", {"n": 1}) + surface.append("a", {"n": 2}) + _tear(path, '{"seq": 3, "writer": "a", "pay') + + record = surface.append("b", {"n": 3}) + + assert record.seq == 3 # the torn write never completed, so its number is free + assert [line.index for line in surface.malformed_lines] == [2] + # and the NEXT append works too: the remnant is not a once-survivable event + assert surface.append("b", {"n": 4}).seq == 4 + + +def test_w7_the_next_record_is_not_glued_onto_the_remnant(tmp_path: Path) -> None: + """Terminator repair. Without it the record following a torn write is swallowed.""" + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + surface.append("a", {"n": 1}) + _tear(path, '{"seq": 2, "writer": "a", "pay') + + surface.append("b", {"n": 2}) + + lines = path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 3, lines # remnant and new record are SEPARATE lines + assert json.loads(lines[2])["payload"] == {"n": 2} # the new record survived whole + assert [r.payload["n"] for r in surface.records()] == [1, 2] + + +def test_w7_records_skips_the_remnant_and_counts_it(tmp_path: Path) -> None: + """Skipping alone is silent; the COUNT is what makes an invisible loss detectable.""" + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + surface.append("a", {"n": 1}) + _tear(path, '{"seq": 2, "writer": "a", "pay') + surface.append("b", {"n": 2}) + + assert [(r.seq, r.payload["n"]) for r in surface.records()] == [(1, 1), (2, 2)] + (bad,) = surface.malformed_lines + assert bad.index == 1 and bad.excerpt.startswith('{"seq": 2') and bad.reason + # the finding is a property of the FILE, not of one handle + fresh = AppendSurface(path) + assert len(fresh.records()) == 2 and len(fresh.malformed_lines) == 1 + + +def test_w7_a_parseable_line_of_the_wrong_shape_is_counted_not_raised(tmp_path: Path) -> None: + """JSON that parses but carries none of the record's fields is malformed HERE.""" + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + surface.append("a", {"n": 1}) + with path.open("a", encoding="utf-8") as handle: + handle.write('{"unrelated": true}\n') + + assert surface.append("b", {"n": 2}).seq == 2 # the append point survives it + assert [r.payload["n"] for r in surface.records()] == [1, 2] + assert len(surface.malformed_lines) == 1 + + # --------------------------------------------------------------------------- measurement From dd3529627ac757c643592fedb486b31b4c735f07 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Thu, 20 Aug 2026 07:07:29 -0500 Subject: [PATCH 21/29] fix: validate a line's shape and guard its decode, not just its parse Two review blocks on this PR, both correct, and both about the SHAPE of the guard rather than a case it was missing. FIRST: an except tuple over untrusted file content is a DENYLIST, and a denylist leaks by construction. Valid JSON seq 1e999 parses to inf, int(inf) raises OverflowError, and no list written from the parse side would have predicted it, because the COERCION AFTER the parse is what invents the new failure. Asking what else escaped the same guard found a sibling: deeply nested JSON raises RecursionError from json.loads itself and bricks identically. So the coercion is gone. Everything after the parse validates types rather than converting them, and inf is refused because a float is not a sequence number, with no exception existing to catch. SECOND: the decode is a raise site BELOW the parse. The exception type was never the problem, since UnicodeDecodeError subclasses ValueError which was already caught; the OPERATION that raises it sat outside the guarded region. In text mode the decode happens while the iterator MANUFACTURES the line, so bytes are converted before any try block can see them. The fix guarded the transformation and left the ACQUISITION unguarded. Operationally that was worse than a brick: a whole-file decode means ONE bad byte anywhere destroys every record in the surface, including the thousands written correctly around it. The file is now read as bytes and decoded one line at a time, which is what a JSONL file actually is, so a bad byte costs its own line. A line passes through four layers - read, decode, parse, shape-check - and the last three are each guarded where they happen. Naming the enumeration rather than claiming completeness, because two earlier claims that the raise-site set was closed were each broken by the next reviewer: unbounded line length is a resource limit on the READ layer and is deliberately not defended, stated in the class docstring and the PR body rather than left to be discovered. Also pinned, found when a new witness exposed a wrong expectation in a fixture of mine rather than a defect in the code: a line can be unreadable as a RECORD while its sequence number is perfectly readable, and numbering honours it anyway. No writer is ever issued a number a reader can already see on disk. Seventeen hostile-content and undecodable-byte rows plus a confinement witness proving ten good records survive a bad byte written between them. Ten mutation rows, measured with an extractor that reads only E-prefixed traceback lines and is proven against a green control - an earlier extractor read exception names out of parametrized TEST IDS and reported types that were never raised. One row was discarded as contaminated, its kills coming from a sloppy edit rather than the defect, and re-run clean. Ref #897 Co-Authored-By: Claude --- gr2/prototypes/contribution_protocol.py | 141 +++++++++++++++++++----- gr2/tests/test_contribution_protocol.py | 126 +++++++++++++++++++++ 2 files changed, 239 insertions(+), 28 deletions(-) diff --git a/gr2/prototypes/contribution_protocol.py b/gr2/prototypes/contribution_protocol.py index 499e5e1..118c936 100644 --- a/gr2/prototypes/contribution_protocol.py +++ b/gr2/prototypes/contribution_protocol.py @@ -476,6 +476,81 @@ class AppendRecord: timestamp: str +def _decode_line(raw: bytes) -> tuple[str | None, str]: + """Bytes become text HERE, so the failure that can happen HERE is guarded here. + + A JSONL file is a BYTE format. Read in text mode, the decode happens while the + iterator produces the line — outside every ``try`` in this module — so one + invalid byte anywhere raises before a guard can see it and the whole scan dies. + The exception type was never the problem: ``UnicodeDecodeError`` subclasses + ``ValueError``, which was already caught. The OPERATION that raises it simply sat + outside the guarded region. A line does not ARRIVE as text, it is MANUFACTURED as + text, and manufacturing it can fail on content; decoding per line confines a bad + byte to that line. + """ + + try: + return raw.decode("utf-8"), "" + except UnicodeDecodeError as exc: + return None, str(exc) + + +def _excerpt(raw: bytes) -> str: + """A malformed line must be reportable even when it is not valid text.""" + + return raw[:120].decode("utf-8", "replace") + + +def _read_json_object(line: str) -> tuple[dict[str, object] | None, str]: + """Parse ONE line into a JSON object, or say why it cannot be read. + + ``json.loads`` is the ONLY operation in this file that can raise on file + content. It raises ``ValueError`` (``JSONDecodeError`` subclasses it, as does + the integer-literal length limit) or ``RecursionError`` (deeply nested input). + Everything after it VALIDATES rather than coerces, and that is the whole point: + a coercion over untrusted bytes forces the caller to enumerate the exceptions it + might raise, which is a denylist, and a denylist leaks. ``int()`` on a value + ``json`` can legitimately produce (``1e999`` parses to ``inf``) raises + ``OverflowError`` — a type no list written from the parse side would predict. + Validating instead removes the raise rather than cataloguing it. + """ + + try: + obj = json.loads(line) + except (ValueError, RecursionError) as exc: + return None, str(exc) + if not isinstance(obj, dict): + return None, f"line is a {type(obj).__name__}, not an object" + return obj, "" + + +def _read_seq(obj: dict[str, object]) -> tuple[int | None, str]: + """A sequence number must be a real integer. ``bool`` is an ``int`` and is not one.""" + + seq = obj.get("seq") + if isinstance(seq, bool) or not isinstance(seq, int): + return None, f"seq is {type(seq).__name__}, not an integer" + return seq, "" + + +def _read_record(obj: dict[str, object]) -> tuple[AppendRecord | None, str]: + """Every field checked by TYPE, so nothing here can raise on hostile content.""" + + seq, reason = _read_seq(obj) + if seq is None: + return None, reason + writer = obj.get("writer") + payload = obj.get("payload") + timestamp = obj.get("timestamp") + if not isinstance(writer, str): + return None, f"writer is {type(writer).__name__}, not a string" + if not isinstance(payload, dict): + return None, f"payload is {type(payload).__name__}, not an object" + if not isinstance(timestamp, str): + return None, f"timestamp is {type(timestamp).__name__}, not a string" + return AppendRecord(seq=seq, writer=writer, payload=payload, timestamp=timestamp), "" + + class AppendSurface: """A declared append-only file with one guarded append point. @@ -490,6 +565,14 @@ class AppendSurface: a caller looks. ``append`` also repairs a missing terminator before writing, so a remnant never GLUES the next record onto itself — without that repair the record that follows a torn write is swallowed into an unparseable line and lost. + + The file is read as BYTES and decoded one line at a time. A JSONL file is a byte + format, and in text mode the decode happens while the iterator produces the line, + outside every guard here — so a single invalid byte anywhere destroyed the whole + scan, including every record written correctly around it. A line passes through + four layers: read, decode, parse, and shape-check. The last three are each guarded + where they happen. Unbounded line length (a file with no terminator for a very + long span) is a resource limit on the first layer and is NOT defended here. """ def __init__(self, path: Path) -> None: @@ -504,32 +587,37 @@ def malformed_lines(self) -> tuple[MalformedLine, ...]: return self._malformed def append(self, writer: str, payload: dict[str, object]) -> AppendRecord: - with open(self.path, "a+", encoding="utf-8") as handle: + with open(self.path, "a+b") as handle: fcntl.flock(handle.fileno(), fcntl.LOCK_EX) try: handle.seek(0) last = 0 malformed: list[MalformedLine] = [] - raw_last = "" + raw_last = b"" for index, raw in enumerate(handle): raw_last = raw - line = raw.strip() - if not line: + stripped = raw.strip() + if not stripped: continue - try: - last = max(last, int(json.loads(line)["seq"])) - except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: - malformed.append( - MalformedLine(index=index, excerpt=line[:120], reason=str(exc)) - ) + line, reason = _decode_line(stripped) + if line is not None: + obj, reason = _read_json_object(line) + if obj is not None: + seq, reason = _read_seq(obj) + if seq is not None: + last = max(last, seq) + continue + malformed.append( + MalformedLine(index=index, excerpt=_excerpt(stripped), reason=reason) + ) self._malformed = tuple(malformed) record = AppendRecord( seq=last + 1, writer=writer, payload=payload, timestamp=_now() ) handle.seek(0, os.SEEK_END) - if raw_last and not raw_last.endswith("\n"): - handle.write("\n") - handle.write(json.dumps(asdict(record), sort_keys=True) + "\n") + if raw_last and not raw_last.endswith(b"\n"): + handle.write(b"\n") + handle.write((json.dumps(asdict(record), sort_keys=True) + "\n").encode("utf-8")) handle.flush() os.fsync(handle.fileno()) return record @@ -539,22 +627,19 @@ def append(self, writer: str, payload: dict[str, object]) -> AppendRecord: def records(self) -> list[AppendRecord]: out: list[AppendRecord] = [] malformed: list[MalformedLine] = [] - for index, raw in enumerate(self.path.read_text(encoding="utf-8").splitlines()): - line = raw.strip() - if not line: + for index, raw in enumerate(self.path.read_bytes().split(b"\n")): + stripped = raw.strip() + if not stripped: continue - try: - data = json.loads(line) - out.append( - AppendRecord( - seq=int(data["seq"]), - writer=str(data["writer"]), - payload=dict(data["payload"]), - timestamp=str(data["timestamp"]), - ) - ) - except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: - malformed.append(MalformedLine(index=index, excerpt=line[:120], reason=str(exc))) + line, reason = _decode_line(stripped) + if line is not None: + obj, reason = _read_json_object(line) + if obj is not None: + record, reason = _read_record(obj) + if record is not None: + out.append(record) + continue + malformed.append(MalformedLine(index=index, excerpt=_excerpt(stripped), reason=reason)) self._malformed = tuple(malformed) return out diff --git a/gr2/tests/test_contribution_protocol.py b/gr2/tests/test_contribution_protocol.py index 7097833..d3d2437 100644 --- a/gr2/tests/test_contribution_protocol.py +++ b/gr2/tests/test_contribution_protocol.py @@ -543,6 +543,132 @@ def test_w7_a_parseable_line_of_the_wrong_shape_is_counted_not_raised(tmp_path: assert len(surface.malformed_lines) == 1 +# A line's CONTENT is untrusted. Each of these is valid JSON (or valid enough to +# reach a coercion) and each one bricked the surface under a version of this fix +# that listed exception types instead of validating shapes. The 1e999 case came from +# review on this PR; the deep-nesting case is its sibling, found by asking what ELSE +# escapes a denylist rather than patching the one instance the review cited. +def _hostile(seq: str, writer: str = '"x"', payload: str = "{}", ts: str = '"t"') -> str: + return f'{{"seq": {seq}, "writer": {writer}, "payload": {payload}, "timestamp": {ts}}}' + + +HOSTILE_LINES = { + "float infinity (1e999 -> inf; int() raises OverflowError)": _hostile("1e999"), + "not-a-number (NaN)": _hostile("NaN"), + "deeply nested (json.loads raises RecursionError)": _hostile("[" * 100000 + "]" * 100000), + "integer literal past the conversion limit": _hostile("9" * 6000), + "seq is an object": _hostile('{"a": 1}'), + "seq is a bool (bool IS an int in Python)": _hostile("true"), + "writer is an object": _hostile("1", writer='{"a": 1}'), + "payload is a string": _hostile("1", payload='"not-an-object"'), + "timestamp is a number": _hostile("1", ts="5"), + "line is an array, not an object": "[1, 2, 3]", + "line is a bare string": '"just a string"', +} + + +@pytest.mark.parametrize("label", sorted(HOSTILE_LINES)) +def test_w7_hostile_line_content_neither_bricks_nor_vanishes(label: str, tmp_path: Path) -> None: + """Both scans must survive ANY line content and count what they could not read. + + A class witness, not an instance one. A guard that LISTS exception types is a + denylist over untrusted input, and a denylist leaks: it passes the case you + thought of and bricks on the next one. Every row here must survive both paths. + """ + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + surface.append("a", {"n": 1}) + with path.open("a", encoding="utf-8") as handle: + handle.write(HOSTILE_LINES[label] + "\n") + + record = surface.append("b", {"n": 2}) # the WRITE path must not brick + assert record.seq == 2 + + assert [r.payload["n"] for r in surface.records()] == [1, 2] # READ path survives + assert len(surface.malformed_lines) == 1 # the bad line is COUNTED, not dropped + assert surface.malformed_lines[0].reason # with a reason a human can act on + + +def test_w7_a_readable_seq_on_an_unreadable_record_is_still_never_reused( + tmp_path: Path, +) -> None: + """Deliberate: a number that APPEARS in the file is not handed out again. + + A line can be unreadable as a RECORD while its sequence number is perfectly + readable. Numbering honours it anyway, so no writer is ever issued a number a + reader can already see on disk — the safe direction when the two scans disagree + about whether a line exists. + """ + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + surface.append("a", {"n": 1}) + with path.open("a", encoding="utf-8") as handle: + handle.write(_hostile("9", payload='"not-an-object"') + "\n") + + assert surface.append("b", {"n": 2}).seq == 10 # 9 is taken, even unreadably + assert [r.payload["n"] for r in surface.records()] == [1, 2] + assert len(surface.malformed_lines) == 1 + + +# A line's BYTES are untrusted too, and the decode happens BEFORE any JSON handling. +# Reading in text mode raises while the iterator manufactures the line — outside every +# guard — so one bad byte anywhere killed the whole scan. The 0xff case came from the +# second review block on this PR; the rest are its class, found by asking what else the +# bytes-to-text layer can refuse. +def _hostile_bytes(writer: bytes) -> bytes: + return b'{"seq": 1, "writer": "' + writer + b'", "payload": {}, "timestamp": "t"}' + + +HOSTILE_BYTES = { + "invalid byte 0xff": _hostile_bytes(b"\xff"), + "lone surrogate": _hostile_bytes(b"\xed\xa0\x80"), + "truncated multi-byte sequence": _hostile_bytes(b"\xe2\x82"), + "overlong encoding": _hostile_bytes(b"\xc0\xaf"), + "continuation byte with no lead": _hostile_bytes(b"\x80\x80"), + "line is pure binary": b"\x00\x01\xff\xfe\xfd", +} + + +@pytest.mark.parametrize("label", sorted(HOSTILE_BYTES)) +def test_w7_undecodable_bytes_neither_brick_nor_vanish(label: str, tmp_path: Path) -> None: + """The bytes-to-text boundary is content-dependent, so it is guarded like any other.""" + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + surface.append("a", {"n": 1}) + with path.open("ab") as handle: + handle.write(HOSTILE_BYTES[label] + b"\n") + + record = surface.append("b", {"n": 2}) # the WRITE path must not brick + assert record.seq == 2 + + assert [r.payload["n"] for r in surface.records()] == [1, 2] # READ path survives + assert len(surface.malformed_lines) == 1 + assert surface.malformed_lines[0].reason + assert surface.malformed_lines[0].excerpt # reportable even when it is not text + + +def test_w7_an_undecodable_line_is_confined_to_itself(tmp_path: Path) -> None: + """The operational property: one bad byte costs ONE line, not the file. + + Under a whole-file decode a single invalid byte anywhere destroys every record in + the surface, including the thousands written correctly before and after it. + """ + path = tmp_path / "ledger" / "events.jsonl" + surface = AppendSurface(path) + for n in range(1, 6): + surface.append("a", {"n": n}) + with path.open("ab") as handle: + handle.write(b'{"seq": 99, "writer": "\xff", "payload": {}, "timestamp": "t"}\n') + for n in range(6, 11): + surface.append("b", {"n": n}) + + records = surface.records() + + assert [r.payload["n"] for r in records] == list(range(1, 11)) # every good line kept + assert len(surface.malformed_lines) == 1 # exactly the one bad line + assert "utf-8" in surface.malformed_lines[0].reason + + # --------------------------------------------------------------------------- measurement From 6e5320c18bc78bc4faa47c063d2774c1fc790aca Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Thu, 20 Aug 2026 07:59:41 -0500 Subject: [PATCH 22/29] fix: one torn-line-safe JSONL home for both lane prototypes Both lane prototypes carried a BYTE-IDENTICAL append_jsonl and a reader with the same body under two names. That is one defect with two addresses, not a class with two instances, so it gets one fix rather than two: a shared jsonl_store module both consume. Fixing them separately would have written the same fix and the same witnesses twice and left the copies free to diverge again, which is how the situation arose. The defect, in both copies: the writer appended with no terminator repair and no fsync, so a writer killed mid-write left a remnant and the NEXT record glued onto it - a record that had itself completed became part of one unparseable line and was lost. The reader parsed with a bare json.loads over read_text().splitlines(), so the remnant raised, and the whole-file decode meant one invalid byte anywhere destroyed every record in the file rather than its own line. WHERE THE COUNT LIVES, which was the open design question: these are module-level functions with no instance to hang health on, and a module-level accumulator would be hidden state and wrong under concurrent readers. So the count is a RETURN VALUE - read_jsonl returns rows AND the lines it could not read. The question dissolves rather than getting answered. And the count reaches a CONSUMER: both CLI callers report it on stderr, so a --json caller's stdout stays machine-readable. A count nobody surfaces is the same silence as no count at all, which is the defect this fix exists to close. The reporter lives in the shared module too. The first version of this change put a helper in one consumer and an inline copy of the same three lines in the other - byte-adjacent duplication of a reporting rule across two files, which is the exact shape being eliminated, committed inside the fix. Caught before the gate. Structure is validated; schema is not. A generic reader has no fields to check, so a line that is well-formed JSON and a well-formed object IS a row even when its values are odd. That boundary is pinned by its own witness, added after a witness failed against a wrong expectation of mine rather than against the code - the hostile-content table had been copied from a surface that does have a schema. 21 witnesses, 7 mutation rows, each mutation killing witnesses whose failure TYPE matches it: removing the decode guard kills seven by real UnicodeDecodeError; removing the parse guard kills six by real JSONDecodeError, RecursionError and the integer-literal ValueError; silencing the count kills fifteen; silencing the reporter kills exactly one, which is what proves the count reaches a consumer. The fsync witness is deliberately weaker than it sounds and says so: it pins that fsync is CALLED, not that durability holds, because a real crash is unwitnessable in-process. It exists because without it, deleting the fsync killed nothing at all - and a guard nothing checks is not a guard. Unbounded line length is a resource limit on the READ layer and is deliberately not defended, named in the module docstring rather than left to be discovered. Ref #897 Co-Authored-By: Claude --- gr2/prototypes/jsonl_store.py | 158 ++++++++++++++ gr2/prototypes/lane_workspace_prototype.py | 156 ++++++++------ gr2/prototypes/recall_lane_history.py | 40 ++-- gr2/tests/test_jsonl_store.py | 239 +++++++++++++++++++++ 4 files changed, 498 insertions(+), 95 deletions(-) create mode 100644 gr2/prototypes/jsonl_store.py create mode 100644 gr2/tests/test_jsonl_store.py diff --git a/gr2/prototypes/jsonl_store.py b/gr2/prototypes/jsonl_store.py new file mode 100644 index 0000000..6b280b0 --- /dev/null +++ b/gr2/prototypes/jsonl_store.py @@ -0,0 +1,158 @@ +"""Torn-line-safe JSONL primitives, shared by the lane prototypes. + +Both lane prototypes carried a BYTE-IDENTICAL ``append_jsonl`` and a reader with +the same body under two names. They were not two instances of a defect class; +they were one function, copied. Fixing them separately would have written the +same fix and the same witnesses twice and left the copies free to diverge again, +which is how the situation arose. One home, two consumers. + +The defect, measured on the sibling surface this shape came from: + +* the writer appended with no terminator repair and no ``fsync``, so a writer + killed mid-write left a remnant and the NEXT record glued onto it — a record + that had itself completed became part of one unparseable line and was lost; +* the reader parsed with a bare ``json.loads`` over ``read_text().splitlines()``, + so the remnant raised, and the whole-file decode meant one invalid byte + anywhere destroyed every record in the file rather than its own line. + +A line passes through four layers — read, decode, parse, shape-check — and the +last three are each guarded where they happen. Types are VALIDATED rather than +coerced, because a coercion over untrusted bytes forces the caller to enumerate +the exceptions it might raise, which is a denylist, and a denylist leaks. +Unbounded line length is a resource limit on the READ layer and is deliberately +NOT defended here; it is named rather than left to be discovered. +""" + +from __future__ import annotations + +import json +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path + +from gr2.prototypes.propagation_state_machine import MalformedLine + + +@dataclass(frozen=True) +class JsonlRead: + """Rows that could be read, and the lines that could not. + + The count travels WITH the data. These are module-level functions with no + object to hang health on, and a module-level accumulator would be hidden + state and wrong under concurrent readers — so the health of the scan is a + return value instead. A caller that ignores ``malformed`` still gets correct + rows; a caller that wants to report health already holds it. Skipping alone + is silent, and silence is the defect: the remnant sits in the file while the + caller sees a healthy-looking result. + """ + + rows: list[dict] + malformed: tuple[MalformedLine, ...] = field(default=()) + + +def _decode_line(raw: bytes) -> tuple[str | None, str]: + """Bytes become text HERE, so the failure that can happen HERE is guarded here.""" + + try: + return raw.decode("utf-8"), "" + except UnicodeDecodeError as exc: + return None, str(exc) + + +def _read_object(line: str) -> tuple[dict | None, str]: + """``json.loads`` is the only operation over a decoded line that can raise.""" + + try: + obj = json.loads(line) + except (ValueError, RecursionError) as exc: + return None, str(exc) + if not isinstance(obj, dict): + return None, f"line is a {type(obj).__name__}, not an object" + return obj, "" + + +def append_jsonl(path: Path, payload: dict) -> None: + """Append one record, repairing a missing terminator first, then fsync. + + Without the repair a remnant GLUES the next record onto itself and the next + record is lost. Without the fsync the record is not durable against the very + crash that produces remnants. + """ + + path.parent.mkdir(parents=True, exist_ok=True) + unterminated = False + try: + with path.open("rb") as probe: + if probe.seek(0, os.SEEK_END): + probe.seek(-1, os.SEEK_END) + unterminated = probe.read(1) != b"\n" + except FileNotFoundError: + pass + with path.open("ab") as handle: + if unterminated: + handle.write(b"\n") + handle.write((json.dumps(payload) + "\n").encode("utf-8")) + handle.flush() + os.fsync(handle.fileno()) + + +def read_jsonl(path: Path) -> JsonlRead: + """Read every line that can be read, and report every line that cannot.""" + + if not path.exists(): + return JsonlRead(rows=[]) + rows: list[dict] = [] + malformed: list[MalformedLine] = [] + for index, raw in enumerate(path.read_bytes().split(b"\n")): + stripped = raw.strip() + if not stripped: + continue + line, reason = _decode_line(stripped) + if line is not None: + obj, reason = _read_object(line) + if obj is not None: + rows.append(obj) + continue + malformed.append( + MalformedLine( + index=index, + excerpt=stripped[:120].decode("utf-8", "replace"), + reason=reason, + ) + ) + return JsonlRead(rows=rows, malformed=tuple(malformed)) + + +def warn_unreadable(read: JsonlRead, what: str, stream: object = None) -> bool: + """Report unreadable lines on STDERR. Returns whether anything was reported. + + Lives HERE, beside the primitives it reports on, because the first version of + this fix put a helper in one consumer and an inline copy of the same three + lines in the other — which is byte-adjacent duplication of a reporting rule + across two files, the same shape as the copied ``append_jsonl`` this module + exists to eliminate. One home for the rule, or the message formats drift. + + STDERR specifically: a ``--json`` caller's stdout must stay machine-readable. + And it is reported at ALL because a count nobody surfaces is the same silence + as no count — the CLI is the layer that already looks. + """ + + if not read.malformed: + return False + first = read.malformed[0] + print( + f"warning: {len(read.malformed)} unreadable line(s) in {what}; " + f"first at line {first.index}: {first.reason}", + file=stream if stream is not None else sys.stderr, + ) + return True + + +__all__ = [ + "JsonlRead", + "MalformedLine", + "append_jsonl", + "read_jsonl", + "warn_unreadable", +] diff --git a/gr2/prototypes/lane_workspace_prototype.py b/gr2/prototypes/lane_workspace_prototype.py index 21749ef..44582bd 100644 --- a/gr2/prototypes/lane_workspace_prototype.py +++ b/gr2/prototypes/lane_workspace_prototype.py @@ -25,6 +25,12 @@ from pathlib import Path import tomli_w +from gr2.prototypes.jsonl_store import ( + JsonlRead, + append_jsonl, + read_jsonl, + warn_unreadable, +) LANE_SCHEMA_VERSION = 1 SCRATCHPAD_SCHEMA_VERSION = 1 @@ -140,9 +146,7 @@ def as_toml(self) -> str: def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Prototype gr2 lanes + shared scratchpads" - ) + parser = argparse.ArgumentParser(description="Prototype gr2 lanes + shared scratchpads") sub = parser.add_subparsers(dest="command", required=True) create = sub.add_parser("create-lane") @@ -221,7 +225,9 @@ def parse_args() -> argparse.Namespace: enter.add_argument("workspace_root", type=Path) enter.add_argument("owner_unit") enter.add_argument("lane_name") - enter.add_argument("--actor", required=True, help="actor label, e.g. human:layne or agent:atlas") + enter.add_argument( + "--actor", required=True, help="actor label, e.g. human:layne or agent:atlas" + ) enter.add_argument("--notify-channel", action="store_true") enter.add_argument("--recall", action="store_true") @@ -355,12 +361,7 @@ def workspace_edit_leases_lock_file(workspace_root: Path) -> Path: def shared_lane_access_file(workspace_root: Path, owner_unit: str, lane_name: str) -> Path: return ( - workspace_root - / ".grip" - / "state" - / "shared_lane_access" - / owner_unit - / f"{lane_name}.json" + workspace_root / ".grip" / "state" / "shared_lane_access" / owner_unit / f"{lane_name}.json" ) @@ -410,12 +411,6 @@ def load_current_lane_doc(workspace_root: Path, owner_unit: str) -> dict: return json.loads(path.read_text()) -def append_jsonl(path: Path, payload: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as fh: - fh.write(json.dumps(payload) + "\n") - - def emit_lane_event(workspace_root: Path, payload: dict) -> None: append_jsonl(lane_events_file(workspace_root), payload) @@ -424,17 +419,10 @@ def emit_recall_lane_event(workspace_root: Path, payload: dict) -> None: append_jsonl(recall_lane_events_file(workspace_root), payload) -def iter_lane_events(workspace_root: Path) -> list[dict]: - path = lane_events_file(workspace_root) - if not path.exists(): - return [] - items: list[dict] = [] - for line in path.read_text().splitlines(): - line = line.strip() - if not line: - continue - items.append(json.loads(line)) - return items +def iter_lane_events(workspace_root: Path) -> JsonlRead: + """Rows AND the lines that could not be read. See ``jsonl_store.JsonlRead``.""" + + return read_jsonl(lane_events_file(workspace_root)) def write_json(path: Path, payload: dict) -> None: @@ -654,7 +642,9 @@ def lease_conflicts(existing_mode: str, requested_mode: str) -> bool: return requested_mode in matrix.get(existing_mode, set()) -def conflicting_leases(leases: list[dict], actor: str, requested_mode: str) -> tuple[list[dict], list[dict]]: +def conflicting_leases( + leases: list[dict], actor: str, requested_mode: str +) -> tuple[list[dict], list[dict]]: active: list[dict] = [] stale: list[dict] = [] for lease in leases: @@ -801,8 +791,8 @@ def enter_lane(args: argparse.Namespace) -> int: emit_lane_event(workspace_root, event) if args.notify_channel: event["channel_message"] = ( - f'{args.actor} entered {args.owner_unit}/{args.lane_name} ' - f'[{lane_doc["lane_type"]}] repos={",".join(lane_doc.get("repos", []))}' + f"{args.actor} entered {args.owner_unit}/{args.lane_name} " + f"[{lane_doc['lane_type']}] repos={','.join(lane_doc.get('repos', []))}" ) if args.recall: emit_recall_lane_event( @@ -842,8 +832,8 @@ def exit_lane(args: argparse.Namespace) -> int: emit_lane_event(workspace_root, event) if args.notify_channel: event["channel_message"] = ( - f'{args.actor} exited {args.owner_unit}/{current_doc["lane_name"]} ' - f'[{current_doc["lane_type"]}]' + f"{args.actor} exited {args.owner_unit}/{current_doc['lane_name']} " + f"[{current_doc['lane_type']}]" ) if args.recall: emit_recall_lane_event( @@ -867,7 +857,9 @@ def exit_lane(args: argparse.Namespace) -> int: "current": next_current, "recent": recent[1:] if next_current else [], } - current_lane_file(workspace_root, args.owner_unit).write_text(json.dumps(updated, indent=2) + "\n") + current_lane_file(workspace_root, args.owner_unit).write_text( + json.dumps(updated, indent=2) + "\n" + ) print(current_lane_file(workspace_root, args.owner_unit)) return 0 @@ -879,28 +871,29 @@ def current_lane(args: argparse.Namespace) -> int: return 0 current_doc = doc["current"] print("gr2 prototype current-lane") - print(f'owner={current_doc["owner_unit"]} lane={current_doc["lane_name"]} type={current_doc["lane_type"]} actor={current_doc["actor"]}') - print(f'entered_at={current_doc["entered_at"]}') + print( + f"owner={current_doc['owner_unit']} lane={current_doc['lane_name']} type={current_doc['lane_type']} actor={current_doc['actor']}" + ) + print(f"entered_at={current_doc['entered_at']}") recent = doc.get("recent", []) if recent: print("recent:") for item in recent: - print(f' - {item["owner_unit"]}/{item["lane_name"]} ({item["lane_type"]})') + print(f" - {item['owner_unit']}/{item['lane_name']} ({item['lane_type']})") return 0 def lane_history(args: argparse.Namespace) -> int: - rows = [ - event for event in iter_lane_events(args.workspace_root.resolve()) - if event.get("owner_unit") == args.owner_unit - ] + read = iter_lane_events(args.workspace_root.resolve()) + warn_unreadable(read, "the lane event log") + rows = [event for event in read.rows if event.get("owner_unit") == args.owner_unit] if args.json: print(json.dumps(rows, indent=2)) return 0 print("TIMESTAMP\tTYPE\tACTOR\tAGENT_ID\tLANE\tREPOS") for row in rows: print( - f'{row.get("timestamp","-")}\t{row.get("type","-")}\t{row.get("agent","-")}\t{row.get("agent_id","-")}\t{row.get("lane","-")}\t{",".join(row.get("repos", []))}' + f"{row.get('timestamp', '-')}\t{row.get('type', '-')}\t{row.get('agent', '-')}\t{row.get('agent_id', '-')}\t{row.get('lane', '-')}\t{','.join(row.get('repos', []))}" ) return 0 @@ -949,6 +942,7 @@ def acquire_lane_lease(args: argparse.Namespace) -> int: print(lane_leases_file(workspace_root, args.owner_unit, args.lane_name)) return 0 + def _acquire_lane_lease_mutation(leases: list[dict], args: argparse.Namespace) -> dict: retained = [lease for lease in leases if lease["actor"] != args.actor] active_conflicts, stale_conflicts = conflicting_leases(retained, args.actor, args.mode) @@ -1041,7 +1035,7 @@ def show_lane_leases(args: argparse.Namespace) -> int: for lease in leases: state = "stale" if is_stale_lease(lease) else "active" print( - f'{lease["actor"]}\t{lease["mode"]}\t{lease.get("ttl_seconds", "-")}\t{lease["acquired_at"]}\t{lease.get("expires_at", "-")}\t{state}' + f"{lease['actor']}\t{lease['mode']}\t{lease.get('ttl_seconds', '-')}\t{lease['acquired_at']}\t{lease.get('expires_at', '-')}\t{state}" ) return 0 @@ -1088,9 +1082,7 @@ def check_review_requirements(args: argparse.Namespace) -> int: workspace_root = args.workspace_root.resolve() ref = f"{args.repo}#{args.pr_number}" required = int( - workspace_constraints(workspace_root) - .get("required_reviewers", {}) - .get(args.repo, 0) + workspace_constraints(workspace_root).get("required_reviewers", {}).get(args.repo, 0) ) matching: list[dict] = [] for path in iter_lane_files(workspace_root): @@ -1190,21 +1182,33 @@ def plan_handoff(args: argparse.Namespace) -> int: source = load_lane_doc(workspace_root, args.source_owner_unit, args.source_lane_name) find_unit_spec(workspace_root, args.target_unit) if args.mode == "shared": - access_path = shared_lane_access_file(workspace_root, args.source_owner_unit, args.source_lane_name) + access_path = shared_lane_access_file( + workspace_root, args.source_owner_unit, args.source_lane_name + ) access = json.loads(access_path.read_text()) if access_path.exists() else None payload = { "mode": "shared", "source_owner_unit": args.source_owner_unit, "source_lane_name": args.source_lane_name, "target_unit": args.target_unit, - "shared_access_present": bool(access and args.target_unit in access.get("shared_with", [])), + "shared_access_present": bool( + access and args.target_unit in access.get("shared_with", []) + ), "exec_rows": [ { "acting_unit": args.target_unit, "owner_unit": args.source_owner_unit, "lane_name": args.source_lane_name, "repo": repo, - "cwd": str(workspace_root / "agents" / args.source_owner_unit / "lanes" / args.source_lane_name / "repos" / repo), + "cwd": str( + workspace_root + / "agents" + / args.source_owner_unit + / "lanes" + / args.source_lane_name + / "repos" + / repo + ), "lease_scope": f"{args.source_owner_unit}/{args.source_lane_name}", } for repo in source.get("repos", []) @@ -1228,7 +1232,15 @@ def plan_handoff(args: argparse.Namespace) -> int: "owner_unit": args.target_unit, "lane_name": target_lane_name, "repo": repo, - "cwd": str(workspace_root / "agents" / args.target_unit / "lanes" / target_lane_name / "repos" / repo), + "cwd": str( + workspace_root + / "agents" + / args.target_unit + / "lanes" + / target_lane_name + / "repos" + / repo + ), "lease_scope": f"{args.target_unit}/{target_lane_name}", } for repo in source.get("repos", []) @@ -1293,7 +1305,7 @@ def list_lanes(args: argparse.Namespace) -> int: doc = tomllib.loads(path.read_text()) refs = ",".join(item["ref"] for item in doc.get("pr_associations", [])) or "-" print( - f'{doc["owner_unit"]}\t{doc["lane_name"]}\t{doc["lane_type"]}\t{len(doc.get("repos", []))}\t{refs}' + f"{doc['owner_unit']}\t{doc['lane_name']}\t{doc['lane_type']}\t{len(doc.get('repos', []))}\t{refs}" ) return 0 @@ -1315,7 +1327,7 @@ def list_shared_scratchpads(args: argparse.Namespace) -> int: doc = tomllib.loads(path.read_text()) participants = ",".join(doc.get("participants", [])) or "-" print( - f'{doc["name"]}\t{doc["kind"]}\t{doc["lifecycle"]}\t{age_days(path)}\t{participants}\t{doc["purpose"]}' + f"{doc['name']}\t{doc['kind']}\t{doc['lifecycle']}\t{age_days(path)}\t{participants}\t{doc['purpose']}" ) return 0 @@ -1348,7 +1360,7 @@ def audit_shared_scratchpads(args: argparse.Namespace) -> int: issues.append("empty-docs") status = "ok" if not issues else "needs-attention" - print(f'{doc["name"]}\t{status}\t{days}\t{",".join(issues) or "-"}') + print(f"{doc['name']}\t{status}\t{days}\t{','.join(issues) or '-'}") return 0 @@ -1357,24 +1369,24 @@ def plan_promote_scratchpad(args: argparse.Namespace) -> int: doc = load_shared_scratchpad_doc(workspace_root, args.name) lane_name = args.lane or f"promote-{args.name}" print("gr2 prototype scratchpad-promotion plan") - print(f'scratchpad: {doc["name"]}') - print(f'kind: {doc["kind"]}') - print(f'lifecycle: {doc["lifecycle"]}') - print(f'target repo: {args.target_repo}') - print(f'target path: {args.target_path}') - print(f'owner unit: {args.owner_unit}') - print(f'suggested lane: {lane_name}') + print(f"scratchpad: {doc['name']}") + print(f"kind: {doc['kind']}") + print(f"lifecycle: {doc['lifecycle']}") + print(f"target repo: {args.target_repo}") + print(f"target path: {args.target_path}") + print(f"owner unit: {args.owner_unit}") + print(f"suggested lane: {lane_name}") print("recommended:") - print( - f" 1. create or reuse a feature lane for {args.target_repo} under {args.owner_unit}" - ) + print(f" 1. create or reuse a feature lane for {args.target_repo} under {args.owner_unit}") print( f" 2. copy content from shared/scratchpads/{doc['name']}/docs into {args.target_repo}:{args.target_path}" ) print(f" 3. branch and commit in lane {lane_name}") print(" 4. open a PR once the artifact is ready for formal review") if not doc.get("linked_refs"): - print("warning: scratchpad has no linked refs; traceability should be added before promotion") + print( + "warning: scratchpad has no linked refs; traceability should be added before promotion" + ) return 0 @@ -1415,9 +1427,9 @@ def next_step(args: argparse.Namespace) -> int: workspace_root = args.workspace_root.resolve() lane_doc = load_lane_doc(workspace_root, args.owner_unit, args.lane_name) print("gr2 prototype next-step") - print(f'lane: {args.owner_unit}/{lane_doc["lane_name"]}') - print(f'type: {lane_doc["lane_type"]}') - print(f'repos: {", ".join(lane_doc["repos"])}') + print(f"lane: {args.owner_unit}/{lane_doc['lane_name']}") + print(f"type: {lane_doc['lane_type']}") + print(f"repos: {', '.join(lane_doc['repos'])}") if lane_doc.get("pr_associations"): print("mode: review") print("recommended:") @@ -1463,7 +1475,9 @@ def plan_exec(args: argparse.Namespace) -> int: print("gr2 lane-exec prototype") print("status=blocked reason=conflicting-active-lease") for lease in active_conflicts: - print(f'conflict: actor={lease["actor"]} mode={lease["mode"]} acquired_at={lease["acquired_at"]}') + print( + f"conflict: actor={lease['actor']} mode={lease['mode']} acquired_at={lease['acquired_at']}" + ) return 0 if stale_conflicts: payload = { @@ -1481,7 +1495,9 @@ def plan_exec(args: argparse.Namespace) -> int: print("gr2 lane-exec prototype") print("status=blocked reason=stale-conflicting-lease") for lease in stale_conflicts: - print(f'stale-conflict: actor={lease["actor"]} mode={lease["mode"]} expires_at={lease.get("expires_at", "-")}') + print( + f"stale-conflict: actor={lease['actor']} mode={lease['mode']} expires_at={lease.get('expires_at', '-')}" + ) return 0 selected_repos = lane_doc["repos"] @@ -1519,12 +1535,12 @@ def plan_exec(args: argparse.Namespace) -> int: else: print("gr2 lane-exec prototype") print( - f'owner={lane_doc["owner_unit"]} lane={lane_doc["lane_name"]} type={lane_doc["lane_type"]} fail_fast={lane_doc["exec_defaults"]["fail_fast"]}' + f"owner={lane_doc['owner_unit']} lane={lane_doc['lane_name']} type={lane_doc['lane_type']} fail_fast={lane_doc['exec_defaults']['fail_fast']}" ) print("LANE\tREPO\tBRANCH\tCWD\tCOMMAND") for row in rows: print( - f'{row["lane"]}\t{row["repo"]}\t{row["branch"]}\t{row["cwd"]}\t{" ".join(row["command"])}' + f"{row['lane']}\t{row['repo']}\t{row['branch']}\t{row['cwd']}\t{' '.join(row['command'])}" ) return 0 diff --git a/gr2/prototypes/recall_lane_history.py b/gr2/prototypes/recall_lane_history.py index 323bf65..f54b5ce 100644 --- a/gr2/prototypes/recall_lane_history.py +++ b/gr2/prototypes/recall_lane_history.py @@ -10,11 +10,16 @@ from pathlib import Path from typing import Any +from gr2.prototypes.jsonl_store import ( + JsonlRead, + append_jsonl, + read_jsonl, + warn_unreadable, +) + def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Prototype recall lane history surface" - ) + parser = argparse.ArgumentParser(description="Prototype recall lane history surface") sub = parser.add_subparsers(dest="command", required=True) demo = sub.add_parser("demo-data") @@ -40,22 +45,10 @@ def lane_events_file(workspace_root: Path) -> Path: return events_dir(workspace_root) / "lane_events.jsonl" -def append_jsonl(path: Path, payload: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as fh: - fh.write(json.dumps(payload) + "\n") +def load_jsonl(path: Path) -> JsonlRead: + """Rows AND the lines that could not be read. See ``jsonl_store.JsonlRead``.""" - -def load_jsonl(path: Path) -> list[dict]: - if not path.exists(): - return [] - rows: list[dict] = [] - for line in path.read_text().splitlines(): - line = line.strip() - if not line: - continue - rows.append(json.loads(line)) - return rows + return read_jsonl(path) def parse_ts(raw: str) -> datetime: @@ -133,11 +126,7 @@ def repo_activity(index: dict[str, Any], repo: str) -> dict[str, Any]: def time_range(index: dict[str, Any], start: str, end: str) -> dict[str, Any]: start_dt = parse_ts(start) end_dt = parse_ts(end) - rows = [ - event - for event in index["all"] - if start_dt <= parse_ts(event["timestamp"]) <= end_dt - ] + rows = [event for event in index["all"] if start_dt <= parse_ts(event["timestamp"]) <= end_dt] return { "query": {"start": start, "end": end}, "count": len(rows), @@ -263,8 +252,9 @@ def main() -> int: print(json.dumps(result, indent=2)) return 0 - events = load_jsonl(lane_events_file(args.workspace_root.resolve())) - index = build_index(events) + read = load_jsonl(lane_events_file(args.workspace_root.resolve())) + warn_unreadable(read, "the lane event log") + index = build_index(read.rows) if args.command == "query": if args.lane: diff --git a/gr2/tests/test_jsonl_store.py b/gr2/tests/test_jsonl_store.py new file mode 100644 index 0000000..6124c18 --- /dev/null +++ b/gr2/tests/test_jsonl_store.py @@ -0,0 +1,239 @@ +"""Witnesses for the shared torn-line-safe JSONL primitives. + +Both lane prototypes carried a BYTE-IDENTICAL ``append_jsonl`` and a reader with +the same body under two names. This is one defect with two addresses, not a class +with two instances, so it has one fix, one home, and one witness set. + +* W1 a writer killed mid-write leaves a remnant: the append point survives it, + the NEXT record is not GLUED onto it, and the remnant is skipped AND COUNTED +* W2 hostile line CONTENT neither bricks nor vanishes (validate, never coerce) +* W3 undecodable BYTES neither brick nor vanish, and a bad byte is CONFINED to + its own line rather than destroying every record around it +* W4 the count has a REAL CONSUMER: warn_unreadable reports it, because a count + nobody surfaces is the same silence as no count at all +""" + +from __future__ import annotations + +import io +import json +import os +from pathlib import Path + +import pytest +from gr2.prototypes.jsonl_store import ( + JsonlRead, + append_jsonl, + read_jsonl, + warn_unreadable, +) + + +def _tear(path: Path, remnant: bytes) -> None: + """A writer killed between its record and its terminator: bytes, no newline.""" + with path.open("ab") as handle: + handle.write(remnant) + handle.flush() + os.fsync(handle.fileno()) + + +# --------------------------------------------------------------------------- W1 + + +def test_w1_a_torn_remnant_does_not_brick_the_append_point(tmp_path: Path) -> None: + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + _tear(path, b'{"n": 2, "part') + + append_jsonl(path, {"n": 3}) # must not raise + append_jsonl(path, {"n": 4}) # and must not raise AGAIN: not a once-survivable event + + read = read_jsonl(path) + assert [row["n"] for row in read.rows] == [1, 3, 4] + assert len(read.malformed) == 1 + + +def test_w1_the_next_record_is_not_glued_onto_the_remnant(tmp_path: Path) -> None: + """Terminator repair. Without it the record after a torn write is swallowed.""" + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + _tear(path, b'{"n": 2, "part') + + append_jsonl(path, {"n": 3}) + + lines = path.read_bytes().split(b"\n") + assert lines[1] == b'{"n": 2, "part' # the remnant, still on its own line + assert json.loads(lines[2])["n"] == 3 # the new record survived WHOLE + assert len(read_jsonl(path).rows) == 2 + + +def test_w1_a_missing_file_reads_empty_and_reports_nothing(tmp_path: Path) -> None: + read = read_jsonl(tmp_path / "never" / "written.jsonl") + assert read == JsonlRead(rows=[], malformed=()) + + +def test_w1_the_write_is_fsynced(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Pins that fsync is CALLED on the appended handle. Deliberately weaker than + it sounds, and labelled so nobody reads more into it than it proves. + + It does NOT prove durability — that needs a real crash, which is unwitnessable + in-process. What it does prove is that the call is present and reached, which + is the difference between a guard and a guard-shaped comment: without it, + deleting the fsync kills no test at all, and a guard nothing checks is not a + guard. Durability against power loss remains ASSERTED, not demonstrated. + """ + synced: list[int] = [] + real = os.fsync + monkeypatch.setattr(os, "fsync", lambda fd: (synced.append(fd), real(fd))[1]) + + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + append_jsonl(path, {"n": 2}) + + assert len(synced) == 2, "each append must fsync its own write" + + +# --------------------------------------------------------------------------- W2 + +HOSTILE_LINES = { + "deeply nested (json.loads raises RecursionError)": '{"n": ' + + "[" * 100000 + + "]" * 100000 + + "}", + "integer literal past the conversion limit": '{"n": ' + "9" * 6000 + "}", + "truncated object": '{"n": 1', + "line is an array, not an object": "[1, 2, 3]", + "line is a bare string": '"just a string"', + "line is a bare number": "42", +} + + +# Deliberately NOT hostile here: this module validates STRUCTURE (is the line a +# JSON object?) and never SCHEMA (does it carry my fields?), because its rows are +# generic and the consumers own interpretation. Nothing coerces, so nothing raises. +# The sibling AppendSurface DOES have a record schema and rejects these — the +# difference is real and is pinned below rather than assumed. +STRUCTURALLY_VALID_BUT_ODD = { + "float infinity": ('{"n": 1e999}', float("inf")), + "not-a-number": ('{"n": NaN}', None), # NaN != NaN, checked by isnan +} + + +@pytest.mark.parametrize("label", sorted(STRUCTURALLY_VALID_BUT_ODD)) +def test_w2_structure_is_validated_but_schema_is_not(label: str, tmp_path: Path) -> None: + """These ARE accepted, deliberately. A generic reader has no schema to check. + + Found by a witness failing against my own expectation rather than against the + code: I copied a hostile-content table from a surface that HAS a record schema. + The row is well-formed JSON and a well-formed object, so it is a row. + """ + import math + + line, expected = STRUCTURALLY_VALID_BUT_ODD[label] + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + with path.open("ab") as handle: + handle.write(line.encode("utf-8") + b"\n") + append_jsonl(path, {"n": 2}) + + read = read_jsonl(path) + assert len(read.rows) == 3 + assert read.malformed == () # NOT malformed: structurally fine + value = read.rows[1]["n"] + if expected is None: + assert math.isnan(value) + else: + assert value == expected + + +@pytest.mark.parametrize("label", sorted(HOSTILE_LINES)) +def test_w2_hostile_line_content_neither_bricks_nor_vanishes(label: str, tmp_path: Path) -> None: + """A guard that LISTS exception types is a denylist over untrusted input, and a + denylist leaks: it passes the case you thought of and bricks on the next one.""" + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + with path.open("ab") as handle: + handle.write(HOSTILE_LINES[label].encode("utf-8") + b"\n") + + append_jsonl(path, {"n": 2}) # the WRITE path must not brick + + read = read_jsonl(path) + assert [row["n"] for row in read.rows] == [1, 2] + assert len(read.malformed) == 1 + assert read.malformed[0].reason + + +# --------------------------------------------------------------------------- W3 + +HOSTILE_BYTES = { + "invalid byte 0xff": b'{"n": "\xff"}', + "lone surrogate": b'{"n": "\xed\xa0\x80"}', + "truncated multi-byte sequence": b'{"n": "\xe2\x82"}', + "overlong encoding": b'{"n": "\xc0\xaf"}', + "continuation byte with no lead": b'{"n": "\x80\x80"}', + "line is pure binary": b"\x00\x01\xff\xfe\xfd", +} + + +@pytest.mark.parametrize("label", sorted(HOSTILE_BYTES)) +def test_w3_undecodable_bytes_neither_brick_nor_vanish(label: str, tmp_path: Path) -> None: + """The bytes-to-text boundary is content-dependent, so it is guarded there.""" + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + with path.open("ab") as handle: + handle.write(HOSTILE_BYTES[label] + b"\n") + + append_jsonl(path, {"n": 2}) + + read = read_jsonl(path) + assert [row["n"] for row in read.rows] == [1, 2] + assert len(read.malformed) == 1 + assert read.malformed[0].excerpt # reportable even when it is not valid text + assert "utf-8" in read.malformed[0].reason + + +def test_w3_an_undecodable_line_is_confined_to_itself(tmp_path: Path) -> None: + """One bad byte costs ONE line. A whole-file decode costs every record.""" + path = tmp_path / "log" / "events.jsonl" + for n in range(1, 6): + append_jsonl(path, {"n": n}) + with path.open("ab") as handle: + handle.write(b'{"n": "\xff"}\n') + for n in range(6, 11): + append_jsonl(path, {"n": n}) + + read = read_jsonl(path) + assert [row["n"] for row in read.rows] == list(range(1, 11)) + assert len(read.malformed) == 1 + + +# --------------------------------------------------------------------------- W4 + + +def test_w4_the_count_reaches_a_consumer(tmp_path: Path) -> None: + """A count nobody surfaces is the same silence as no count. Verify by fruit.""" + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + _tear(path, b'{"n": 2, "part') + append_jsonl(path, {"n": 3}) + + out = io.StringIO() + reported = warn_unreadable(read_jsonl(path), "the lane event log", stream=out) + + assert reported is True + text = out.getvalue() + assert "1 unreadable line(s)" in text + assert "the lane event log" in text + assert "first at line 1" in text # names WHERE, so it can be acted on + + +def test_w4_a_healthy_log_reports_nothing(tmp_path: Path) -> None: + """The negative case: silence when there is nothing to say, or the signal is noise.""" + path = tmp_path / "log" / "events.jsonl" + append_jsonl(path, {"n": 1}) + + out = io.StringIO() + reported = warn_unreadable(read_jsonl(path), "the lane event log", stream=out) + + assert reported is False + assert out.getvalue() == "" From 69e92679c749cd6e80b6827b59c5cce2995a14a6 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Thu, 20 Aug 2026 11:00:56 -0500 Subject: [PATCH 23/29] fix(events): repair torn outbox lines and report unreadable ones Fix 4 of the torn-line sweep, and the two-part contract it was ruled as. TERMINATOR REPAIR. emit() appended with "a" and wrote json + "\n". A previous write that died between write() and fsync() leaves a last line with no terminator, so the next append GLUES two records into one. The damage runs FORWARD from the tear: the torn record and THE NEXT HEALTHY APPEND fuse into one unparseable line, while the record before the tear is untouched. A torn write therefore costs that record and the next one written after it, permanently, because every later append builds on the glued line. emit() now probes the last byte under the existing write lock and heals the seam first. THE COUNT. Both readers skipped unusable lines in silence. For the channel bridge an unreadable line is a message that never reaches a channel. It is reported on EVERY read, not once: the cursor filter applies only to lines that parse, so a line with no usable seq can never be advanced past, wherever it sits -- position is irrelevant and mid-file lines repeat exactly as trailing ones do. Deliberate, not incidental, and its cost is named in the residuals. read_events_detailed() now returns the events AND the lines it could not read, from the SAME read. read_events() stays list-shaped for the ELEVEN call sites that index and len() it -- all tests, and ZERO production callers remain once the bridge moves to read_events_detailed(), so the wrapper is a test-compatibility surface rather than a load-bearing API. An earlier draft of this message said seventeen; that was a substring artifact counting a def, two prose mentions inside strings, and four hits on an unrelated _read_events helper in another test file. The bridge reports on stderr so stdout stays parseable. Reported from the read path ONLY. _current_seq() runs once per emit inside the write lock, where a count would be per-APPEND and aimed at whoever happened to be writing. Its docstring says so, because an omission and a decision look identical in code. Also guarded, each with witnesses: the decode moved to bytes-per-line so a single invalid byte cannot escape from outside every guard; the parse guard's exception tuple is now derived from what json.loads can raise rather than from what had been seen -- the old (JSONDecodeError, TypeError) caught syntax errors but missed RecursionError, which is not a ValueError, so deep nesting escaped; structure is checked separately, since a valid JSON array parses and is still not an event; seq values are type-validated, since bool is an int subclass and a float becomes inf and serializes as Infinity; and a corrupt cursor no longer bricks reads. THREE THINGS THIS FIX GOT WRONG FIRST, none found by its own witnesses: - An early version swallowed OSError in the line iterator. That is exactly the OSError-to-zero fallback an earlier fix removed on purpose: swallow it and _current_seq returns 0, then emit allocates sequence numbers that duplicate live ones. A pre-existing test stood guard and caught it. Content errors are data and get skipped-and-counted; an I/O error is not knowing what the file holds and must fail closed. The reader tolerates FileNotFoundError only, because rotation renames the file, and propagates every other OSError. - MalformedLine and EventRead were dataclasses. This module is loaded out-of-tree by spawned workers via spec_from_file_location + exec_module, which does not register it in sys.modules, and dataclass resolves field types through exactly that. Every worker died at import. Now plain classes, with a witness that fails in 0.06s naming the cause instead of after a 10s timeout that reads like flakiness. - The default report echoed raw line content to stderr, verbatim, including an API-key-shaped string in a measured probe. stderr is copied into CI logs and transcripts. Redacting was rejected: matching secret-shaped patterns in arbitrary bytes is a denylist over untrusted input, the same defect the parse guard exists to avoid. The excerpt stays on the data object; the default report prints ordinal and reason, both structural, with show_content opt-in. One existing test's monkeypatch target moved read_text -> read_bytes because the read verb changed; its contract and every assertion are unchanged, and it would otherwise have gone green by missing its target. Both it and the new witness now undo the patch before reading the file back, since a verification must not travel the path the test deliberately sabotaged. A reviewer measured both of those statements against an earlier version of this message and of the description, where the glue direction was backwards and the count was claimed to be reported once. The suite was green while the prose said the opposite of the code, because nothing pinned that behavior. Three witnesses now do, including the mid-file case, which shows the rule is broader than the trailing-line framing that surfaced it. A third review then caught a further claim, in the description only: that a torn last line self-heals at the next emit. That is true of one tear and false of the other, and the bare word "torn" hides the difference. An UNTERMINATED record -- complete, only its newline lost -- DOES recover fully once the seam is healed. A TRUNCATED record -- the write stopped mid-record -- never does: its bytes were never written, so it stays unreadable and every read reports it until the file is repaired. A FOURTH review then caught the opposite overcorrection, which a later draft of this message had made -- claiming a torn record never becomes readable -- and the unterminated-tear control disproves it. Both cases are now named separately and the bare word is not used to claim anything about recovery. That prose was wrong only because every tear fixture until then dropped the trailing newline and left the record COMPLETE -- the lucky case. Three witnesses now cover the realistic tear, with the UNTERMINATED tear kept as a discriminating control so the finding cannot be mistaken for the norm. What the repair buys, stated exactly: it cannot recover a record that was never fully written, and it saves the NEXT one. Correcting the message and the description was not enough, and a second review caught that: the same backwards claim was still in the production repair comment and in the test module docstring. Fixing the two cited surfaces and stopping there left the code asserting one direction while the description asserted the other -- an artifact contradicting itself, which is worse than being uniformly wrong. A sweep for the whole class rather than the cited instances found a third live instance neither review named: the reported-once claim, still standing in that same docstring. Those particular corrections changed comments only and left the executable syntax tree identical, verified by parsing both revisions and comparing with docstrings stripped. A FIFTH review then caught the same overclaim surviving in the test module comment -- "A TORN RECORD IS NEVER REPAIRED BY A LATER EMIT" -- which the unterminated control disproves. The class query that was supposed to have swept it missed it because the query was CASE-SENSITIVE and the comment is upper case: a false negative from my own instrument, of the kind noted one revision earlier and then committed in the next query. That round also renamed the control from "benign" to "unterminated" so the tests and the prose use one vocabulary, so unlike the previous round the syntax tree DID change here and the suite was re-run rather than reasoned about. A SIXTH review, from the other reviewer, found three defects and all three were PROSE -- the code it gated was clean. The first is the one worth the round: the reported-once claim was still live in the channel-bridge comment, in the bridge hunk of ALL FIVE versions, and every class sweep I ran missed it because it PARAPHRASES the sentence the earlier reviews cited rather than repeating it. A query built from a cited instance finds COPIES, not paraphrases; the class is the CLAIM, and the only instrument that finds a paraphrase is reading the hunk. The second: the parse-guard sentence read as if the new tuple were open-ended. It is enumerated -- (ValueError, RecursionError). What changed is where the enumeration comes FROM: what json.loads can raise, rather than what had been seen. The third: "seventeen call sites" was an artifact of a substring query that counted a def, two prose mentions inside strings, and four hits on an unrelated _read_events helper in another test file. The real count is eleven, all tests, zero production callers. BOTH reviewers co-signed seventeen at v1 from that same defective query, which is why it needed re-deriving rather than re-reading -- and the class sweep for it then found three more copies of the count in docstrings that no review had cited. A SEVENTH review found two more, and one of them was wrong in the CODE rather than in the prose about it. The bridge comment said an unterminated record is reported and then heals at the next emit. Measured false: an unterminated record whose bytes are COMPLETE is never reported at all, because _iter_outbox() splits on b"\n" and a complete final chunk parses on the spot -- before any emit, repair or no repair. The sentence implied a window of unreadability that does not exist. The gap that let it survive is now pinned: every tear fixture in this file emitted AFTER tearing, so nothing had ever asked what the READER alone does with a torn file. W11 is that pair -- the unterminated case reporting nothing across two reads, the truncated case reporting one as its discriminating control. The reviewer's own probe became the witness. Its two siblings, the W10 header and the description's table, are scoped to tear -> emit -> read and are true there, so they are deliberately unchanged. The second finding was stale suite totals, and the mechanism differs from the one proposed: the +3 is not the earlier merge, which added 21 tests that are inside both baselines, but a test directory the narrow invocation does not collect. Both figures were correct measurements of different scopes and the defect was publishing one without naming which -- which is also how the three mutation rows above went stale, unasked about by any review. RESIDUALS, named rather than presented as clean: this is a third copy of these primitives, and consolidating them is outside this fix's scope; the same content-echo property exists in the prototype reporter merged earlier; unbounded line length remains a read-layer resource limit and is undefended; and a TRUNCATED record -- an UNTERMINATED one is readable throughout and never reaches the report at all -- is permanently unreadable and reported on every read, so a consumer polling in a loop warns every cycle until the file is repaired -- suppressing that would need persisted already-reported state and would hide a re-occurring fault, so it is the lesser cost but it is a cost. Evidence: 37 witnesses, 0 before. Nine mutation rows, RE-MEASURED for this version over the two events spec files (71 tests), each killing witnesses whose failure TYPE matches the mutation, restores hash-verified with the unmutated pair as a control: decode 11, exception tuple 1, terminator repair 5, seq validation 4, count silenced 19, reporter silenced 1, OSError swallowed 3, FileNotFoundError widened 1, cursor guard 1. Three of those rows were STALE -- published as 8, 3 and 12 where the true figures are 11, 5 and 19 -- because they were measured early and never re-derived while five review rounds added witnesses to the files they count. Full suite at BOTH scopes, base pinned by SHA, since omitting the scope is what made the last version's totals unverifiable. On origin/dev@93675677 in an isolated worktree: pytest gr2/tests 1032 passed / 5 failed, pytest gr2 1035 / 5. On this head: 1069 / 5 and 1072 / 5. Delta exactly +37 in both, matching the witness count, failure sets identical in both directions. The scopes differ by gr2/gr2_overlay/tests/test_overlay_refs_namespace.py -- 3 tests, measured passing on both sides, which the narrow invocation does not collect. Lint unchanged: production 6 = 6, edited test file 23 = 23, new file 0. Premium boundary: grip is OSS; this is local file mechanics over opaque paths and carries no identity, org, or policy content. Co-Authored-By: Claude --- gr2/python_cli/channel_bridge.py | 26 +- gr2/python_cli/events.py | 290 ++++++++++++-- gr2/tests/test_events.py | 20 +- gr2/tests/test_events_torn_line.py | 592 +++++++++++++++++++++++++++++ 4 files changed, 896 insertions(+), 32 deletions(-) create mode 100644 gr2/tests/test_events_torn_line.py diff --git a/gr2/python_cli/channel_bridge.py b/gr2/python_cli/channel_bridge.py index a4c08e0..505f4b4 100644 --- a/gr2/python_cli/channel_bridge.py +++ b/gr2/python_cli/channel_bridge.py @@ -2,7 +2,7 @@ Translates outbox events into channel messages per the mapping table in HOOK-EVENT-CONTRACT.md section 8. Uses cursor-based consumption from -events.read_events(). +events.read_events_detailed(), which also reports lines it could not read. The bridge is a pure function layer: format_event() maps an event dict to a message string (or None), and run_bridge() orchestrates cursor reads and @@ -14,7 +14,7 @@ from pathlib import Path from typing import Callable -from .events import read_events +from .events import read_events_detailed, warn_unreadable _CONSUMER_NAME = "channel_bridge" @@ -103,9 +103,27 @@ def run_bridge( The post_fn receives formatted message strings; the caller decides how to deliver them (recall_channel, print, log, etc.). """ - events = read_events(workspace_root, _CONSUMER_NAME) + read = read_events_detailed(workspace_root, _CONSUMER_NAME) + # An unreadable outbox line is a message that never reaches a channel, and + # the bridge is the only place that says so. + # + # It is reported on EVERY read, not once: the cursor filter compares seq + # against the cursor and only lines that PARSE have a usable seq, so an + # unreadable line can never be advanced past, wherever it sits in the file + # (witnessed terminal AND mid-file). An UNTERMINATED record never reaches + # this report at all: the reader splits on b"\n", so a COMPLETE record whose + # only loss was its terminator is the final chunk and parses on the spot -- + # before any emit, repair or no repair (W11, and its truncated control). + # What seam repair protects is the NEXT append, which would otherwise be + # glued onto it. So everything reported here is a TRUNCATED record, and for + # those the report is permanent: a bridge polling in a loop warns every + # cycle until someone repairs the file. That cost is real and is recorded + # as a residual. + # + # stderr keeps stdout parseable for callers that consume the posted count. + warn_unreadable(read) posted = 0 - for event in events: + for event in read.events: msg = format_event(event) if msg is not None: post_fn(msg) diff --git a/gr2/python_cli/events.py b/gr2/python_cli/events.py index 0793108..826ba96 100644 --- a/gr2/python_cli/events.py +++ b/gr2/python_cli/events.py @@ -13,6 +13,7 @@ import os import sys import time +from collections.abc import Iterator from contextlib import contextmanager from datetime import datetime, timezone from enum import Enum @@ -120,21 +121,170 @@ def _event_write_lock(outbox: Path): fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) +# DELIBERATELY PLAIN CLASSES, NOT @dataclass, AND THE REASON IS LOAD-BEARING. +# +# This module is loaded out-of-tree by spawned workers via +# importlib.util.spec_from_file_location() + exec_module(), which does NOT +# register the module in sys.modules. @dataclass resolves field types through +# sys.modules[cls.__module__].__dict__, so under that loader it raises +# AttributeError: 'NoneType' object has no attribute '__dict__' AT IMPORT -- +# every worker dies before running a line of its own. +# +# Measured: adding @dataclass here killed both writers in the concurrent-emit +# integrity test before either reached sequence allocation. The failure was +# visible an hour earlier in an ad-hoc probe and was dismissed as a loader +# artifact; it was a portability constraint on this file. Anything importable by +# a spawned worker must import without sys.modules registration. +# test_events_torn_line.py carries a witness for exactly this. + + +class MalformedLine: + """One line the reader could not turn into an event, and why.""" + + __slots__ = ("ordinal", "reason", "excerpt") + + def __init__(self, ordinal: int, reason: str, excerpt: str) -> None: + self.ordinal = ordinal + self.reason = reason + self.excerpt = excerpt + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"MalformedLine(ordinal={self.ordinal!r}, reason={self.reason!r})" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MalformedLine): + return NotImplemented + return (self.ordinal, self.reason, self.excerpt) == ( + other.ordinal, + other.reason, + other.excerpt, + ) + + +class EventRead: + """Events read, AND the lines that could not be read. + + Both halves come from the SAME read. A second pass to "check health" would + describe a different moment, and the outbox is appended to concurrently. + """ + + __slots__ = ("events", "malformed") + + def __init__( + self, + events: list[dict[str, object]], + malformed: tuple[MalformedLine, ...], + ) -> None: + self.events = events + self.malformed = malformed + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"EventRead(events={len(self.events)}, malformed={len(self.malformed)})" + + +def _decode_line(raw: bytes) -> tuple[str | None, str]: + """Decode one line, or say why it could not be decoded. + + The DECODE is the acquisition, not a transformation of an existing string: + reading the outbox in text mode manufactures the line as text outside every + guard, so a single invalid byte escapes as UnicodeDecodeError from a place + no parse guard can reach. Reading bytes and decoding per line puts the one + operation that can fail on file CONTENT inside the funnel. + """ + try: + return raw.decode("utf-8"), "" + except UnicodeDecodeError as exc: + return None, str(exc) + + +def _read_object(line: str) -> tuple[dict[str, object] | None, str]: + """Parse one line into a JSON object, or say why it is not one. + + This IS an enumerated tuple -- what changed is how the entries were chosen. + The previous guard, (JSONDecodeError, TypeError), was a list of what had been + SEEN: it caught syntax errors, and missed RecursionError, which json.loads + raises on deeply nested input and which is NOT a ValueError, so deep nesting + escaped a reader whose contract is to not raise on file content. + + This tuple is derived from what the OPERATION can raise: ValueError as the + base class covering JSONDecodeError and any other value error, plus + RecursionError, the one thing json.loads raises that ValueError does not + cover. Structure -- is the result an object? -- is then checked separately + below, because a valid JSON array parses cleanly and is still not an event. + """ + try: + obj = json.loads(line) + except (ValueError, RecursionError) as exc: + return None, str(exc) + if not isinstance(obj, dict): + return None, f"line is a {type(obj).__name__}, not an object" + return obj, "" + + +def _read_seq(obj: dict[str, object]) -> tuple[int | None, str]: + """The event's sequence number, or why it cannot be used as one. + + bool is an int subclass, so `isinstance(True, int)` is True and `max(0, True)` + quietly yields 1. A float slips through comparison and arithmetic and then + fails at serialization: 1e999 is valid JSON input, becomes inf, and + json.dumps writes `Infinity`, which is not valid JSON output. Validate the + type here rather than discovering it downstream. + """ + seq = obj.get("seq") + if isinstance(seq, bool) or not isinstance(seq, int): + return None, f"seq is {type(seq).__name__}, not an integer" + return seq, "" + + +def _iter_outbox(outbox: Path) -> Iterator[tuple[dict[str, object] | None, str, bytes]]: + """Yield (object, reason, raw) per line, never raising on file CONTENT. + + Splitting bytes on b"\n" rather than iterating text: see _decode_line. + """ + # DELIBERATELY CATCHES NOTHING. An I/O failure is not a malformed line: it + # is not knowing what the file contains, and the caller's correct response + # differs. _current_seq() is on the WRITE path, where swallowing this + # returns 0 and emit() then allocates a sequence number that duplicates + # existing ones -- silent event-log corruption, which is why an earlier fix + # removed exactly this OSError-to-zero fallback and left a test standing + # guard over it. Reintroducing it here was caught by that test and not by + # any witness of mine. + blob = outbox.read_bytes() + for raw in blob.split(b"\n"): + if not raw.strip(): + continue + line, reason = _decode_line(raw) + if line is None: + yield None, reason, raw + continue + obj, reason = _read_object(line) + yield obj, reason, raw + + +def _excerpt(raw: bytes) -> str: + return raw[:120].decode("utf-8", "replace") + + def _current_seq(outbox: Path) -> int: + """Highest sequence number in the outbox. + + DELIBERATELY REPORTS NO COUNT, and that is a decision rather than an + omission. This runs once per emit, inside the write lock, so a count here + would be a per-APPEND number reported to whoever happened to be writing -- + the wrong altitude and the wrong audience. Unreadable lines are surfaced + once, from read_events_detailed(), where the number is per-READ and reaches + a consumer that can act on it. + """ if not outbox.exists(): return 0 - text = outbox.read_text() last_seq = 0 - for line in text.strip().split("\n"): - line = line.strip() - if not line: + for obj, _reason, _raw in _iter_outbox(outbox): + if obj is None: continue - try: - obj = json.loads(line) - if isinstance(obj, dict) and "seq" in obj: - last_seq = max(last_seq, obj["seq"]) - except (json.JSONDecodeError, TypeError): + seq, _seq_reason = _read_seq(obj) + if seq is None: continue + last_seq = max(last_seq, seq) return last_seq @@ -189,8 +339,25 @@ def emit( event["agent_id"] = agent_id event.update(payload) - with outbox.open("a") as event_file: - event_file.write(json.dumps(event, separators=(",", ":")) + "\n") + # TERMINATOR REPAIR. A previous write that died between write() and + # fsync() leaves a last line with no "\n". Appending onto that GLUES + # two records into one line, and the damage runs FORWARD from the + # tear: the torn record and THE NEXT HEALTHY APPEND fuse into one + # unparseable line, while the record before the tear is untouched. + # So a torn write costs that record and the next one written after + # it, permanently, because every later append builds on the glued + # line. Probe the last byte and heal the seam before writing. + # (Direction measured, not reasoned: see the glue witness in + # test_events_torn_line.py. An earlier version of this comment + # stated it backwards.) + with outbox.open("a+b") as event_file: + event_file.seek(0, os.SEEK_END) + if event_file.tell() > 0: + event_file.seek(-1, os.SEEK_END) + if event_file.read(1) != b"\n": + event_file.write(b"\n") + payload_bytes = json.dumps(event, separators=(",", ":")).encode("utf-8") + event_file.write(payload_bytes + b"\n") event_file.flush() os.fsync(event_file.fileno()) except Exception as exc: @@ -228,27 +395,52 @@ def emit_after_outcome( ) -def read_events(workspace_root: Path, consumer: str) -> list[dict[str, object]]: +def read_events_detailed(workspace_root: Path, consumer: str) -> EventRead: + """New events for `consumer`, AND the lines that could not be read. + + This is the primitive; read_events() is the list-shaped wrapper kept for the + eleven existing call sites, all of them tests. The count is a RETURN VALUE rather than + hidden state: these are module-level functions with no instance to hang + health on, and a module-level accumulator would be wrong under concurrent + readers, which the outbox explicitly has. + + An unreadable line here is a LOST EVENT -- for the channel bridge it is a + message that never reaches a channel -- so silence is the failure, not the + safe default. + """ outbox = _outbox_path(workspace_root) if not outbox.exists(): - return [] + return EventRead([], ()) cursor = _load_cursor(workspace_root, consumer) last_seq = cursor.get("last_seq", 0) + if isinstance(last_seq, bool) or not isinstance(last_seq, int): + # A hand-edited or truncated cursor must not brick every future read. + last_seq = 0 events: list[dict[str, object]] = [] - text = outbox.read_text() - for line in text.strip().split("\n"): - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: + malformed: list[MalformedLine] = [] + try: + lines = list(_iter_outbox(outbox)) + except FileNotFoundError: + # ONLY this one, and only on the read path: _maybe_rotate() renames the + # outbox, so a reader can legitimately lose the file between exists() + # and the read. The events are not gone, they are in an archive. Any + # OTHER OSError is a real failure and propagates -- a reader that + # swallows EIO reports "no new events" forever. + return EventRead([], ()) + for ordinal, (obj, reason, raw) in enumerate(lines, start=1): + if obj is None: + malformed.append(MalformedLine(ordinal, reason, _excerpt(raw))) continue - if not isinstance(obj, dict): + seq, seq_reason = _read_seq(obj) + if seq is None: + # An event whose seq is unusable cannot be ordered against the + # cursor. Comparing it anyway is how "x" <= 0 raises TypeError from + # inside a reader whose job is to not raise on file content. + malformed.append(MalformedLine(ordinal, seq_reason, _excerpt(raw))) continue - if obj.get("seq", 0) <= last_seq: + if seq <= last_seq: continue events.append(obj) @@ -265,7 +457,57 @@ def read_events(workspace_root: Path, consumer: str) -> list[dict[str, object]]: }, ) - return events + return EventRead(events, tuple(malformed)) + + +def read_events(workspace_root: Path, consumer: str) -> list[dict[str, object]]: + """New events for `consumer`, DISCARDING the unreadable-line report. + + Kept list-shaped because eleven call sites index and len() the result, all + of them tests -- no production caller remains once the bridge moves to + read_events_detailed(), so this is a test-compatibility surface. + It discards information, which is exactly the silence this sweep exists to + remove -- so if you are writing a NEW consumer, call read_events_detailed() + and say something about `malformed`. The one production consumer, the + channel bridge, does. + """ + return read_events_detailed(workspace_root, consumer).events + + +def warn_unreadable(read: EventRead, stream=None, *, show_content: bool = False) -> bool: + """Report unreadable outbox lines on stderr. True when any were reported. + + stderr, never stdout: a --json consumer's stdout must stay parseable, and a + warning written there turns a health report into a parse failure. + + THE EXCERPT IS NOT PRINTED BY DEFAULT, and that is a deliberate call rather + than lost fidelity. `ordinal` and `reason` are STRUCTURAL -- a line number + and a parser complaint -- and carry no payload. The excerpt is CONTENT, and + printing it moves bytes out of a file the operator already owns into places + that get copied: CI logs, terminal scrollback, transcripts pasted into a + chat. Measured: a malformed line containing an API-key-shaped string echoed + that string verbatim. + + Redacting it instead was rejected. Matching "secret-looking" patterns in + arbitrary bytes is a denylist over untrusted input, which leaks by + construction -- the same defect shape this module's parse guard exists to + avoid. So the excerpt stays on MalformedLine, where a caller that needs it + can ask, and stays out of the default report. Pass show_content=True to + include it when you are debugging a specific file and know what is in it. + """ + if not read.malformed: + return False + out = stream if stream is not None else sys.stderr + count = len(read.malformed) + plural = "" if count == 1 else "s" + print( + f"warning: skipped {count} unreadable line{plural} in the event outbox", + file=out, + ) + for bad in read.malformed: + detail = f": {bad.excerpt}" if show_content else "" + print(f" line {bad.ordinal}: {bad.reason}{detail}", file=out) + return True def _load_cursor(workspace_root: Path, consumer: str) -> dict[str, object]: diff --git a/gr2/tests/test_events.py b/gr2/tests/test_events.py index eb78ede..3eab675 100644 --- a/gr2/tests/test_events.py +++ b/gr2/tests/test_events.py @@ -487,19 +487,27 @@ def test_emit_fails_closed_on_write_failure(self, workspace: Path): def test_emit_fails_closed_when_existing_sequence_cannot_be_read( self, workspace: Path, monkeypatch: pytest.MonkeyPatch ): - """Restoring _current_seq's OSError-to-zero fallback must turn this RED.""" + """Restoring _current_seq's OSError-to-zero fallback must turn this RED. + + MONKEYPATCH TARGET UPDATED read_text -> read_bytes (torn-line sweep fix 4). + The outbox is now read as BYTES so a single invalid byte cannot escape as + UnicodeDecodeError from outside every guard. This test's CONTRACT and every + assertion below are unchanged; only the read verb it intercepts moved, because + patching read_text no longer intercepts anything and the test would have gone + green by missing its target rather than by the behavior holding. + """ from gr2.python_cli.events import EventEmitError, EventType, _outbox_path, emit outbox = _outbox_path(workspace) outbox.write_text('{"seq":41}\n') - original_read_text = Path.read_text + original_read_bytes = Path.read_bytes def fail_for_outbox(path: Path, *args, **kwargs): if path == outbox: raise OSError("forced sequence-read failure") - return original_read_text(path, *args, **kwargs) + return original_read_bytes(path, *args, **kwargs) - monkeypatch.setattr(Path, "read_text", fail_for_outbox) + monkeypatch.setattr(Path, "read_bytes", fail_for_outbox) with pytest.raises(EventEmitError) as exc_info: emit( event_type=EventType.LANE_ENTERED, @@ -510,6 +518,10 @@ def fail_for_outbox(path: Path, *args, **kwargs): ) assert isinstance(exc_info.value.__cause__, OSError) + # Undo before the read-back: the patch now targets read_bytes, which is + # also how this assertion reads the file. A verification must not travel + # the path the test deliberately sabotaged. + monkeypatch.undo() assert outbox.read_bytes() == b'{"seq":41}\n' def test_emit_creates_events_dir_if_missing(self, workspace: Path): diff --git a/gr2/tests/test_events_torn_line.py b/gr2/tests/test_events_torn_line.py new file mode 100644 index 0000000..cc61307 --- /dev/null +++ b/gr2/tests/test_events_torn_line.py @@ -0,0 +1,592 @@ +"""Witnesses for the event outbox's torn-line contract and its unreadable-line count. + +Fix 4 of the torn-line sweep. Two parts, and they are separate claims: + + 1. TERMINATOR REPAIR -- an append onto a file whose last line lost its "\n" + must not GLUE two records together. The damage runs FORWARD from the tear: + the torn record and THE NEXT HEALTHY APPEND fuse into one unparseable + line, while the record before the tear is untouched. So a torn write costs + that record and the next one written after it, permanently, because every + later append builds on the glued line. + + 2. THE COUNT -- a line the reader cannot use is a LOST EVENT, and for the + channel bridge it is a message that never reaches a channel. It is + reported on EVERY read, not once: the cursor filter applies only to lines + that PARSE, so a line with no usable seq can never be advanced past, + wherever it sits -- position is irrelevant, and mid-file lines repeat + exactly as trailing ones do. It is reported from read_events_detailed() + only: _current_seq() runs once per emit inside the write lock, where a + count would be per-APPEND and aimed at whoever happened to be writing. + +Both statements above were WRONG in an earlier version of this file, in the +same words, while every test here passed. Prose is not covered by the suite +unless something pins the behavior it describes, so the witnesses at the end of +this file exist to pin exactly these two claims: which record the glue destroys, +and how long an unreadable line keeps being reported. +""" +from __future__ import annotations + +import io +import json +from pathlib import Path + +import pytest +from gr2.python_cli.channel_bridge import run_bridge +from gr2.python_cli.events import ( + EventType, + _current_seq, + _outbox_path, + emit, + read_events, + read_events_detailed, + warn_unreadable, +) + + +@pytest.fixture +def workspace(tmp_path: Path) -> Path: + (tmp_path / ".grip" / "events").mkdir(parents=True) + return tmp_path + + +def _emit(workspace: Path, **payload) -> None: + emit( + EventType.PROPAGATION_RECEIPT, + workspace, + actor="witness", + owner_unit="unit", + payload=payload or {"note": "n"}, + ) + + +def _lines(workspace: Path) -> list[bytes]: + raw = _outbox_path(workspace).read_bytes() + return [ln for ln in raw.split(b"\n") if ln.strip()] + + +# --- W1: terminator repair -------------------------------------------------- + +def test_append_onto_torn_line_does_not_glue_records(workspace: Path): + _emit(workspace, note="first") + outbox = _outbox_path(workspace) + # Tear the file exactly as a process killed mid-write leaves it. + blob = outbox.read_bytes() + assert blob.endswith(b"\n") + outbox.write_bytes(blob[:-1]) + + _emit(workspace, note="second") + + lines = _lines(workspace) + assert len(lines) == 2, "the append glued two records into one line" + for ln in lines: + json.loads(ln) # both must still parse + + +def test_torn_line_repair_survives_a_second_tear(workspace: Path): + """One repair is not a fix if the next tear re-breaks it.""" + _emit(workspace, note="a") + outbox = _outbox_path(workspace) + outbox.write_bytes(outbox.read_bytes()[:-1]) + _emit(workspace, note="b") + outbox.write_bytes(outbox.read_bytes()[:-1]) + _emit(workspace, note="c") + + lines = _lines(workspace) + assert len(lines) == 3 + notes = [json.loads(ln)["note"] for ln in lines] + assert notes == ["a", "b", "c"] + + +def test_no_spurious_terminator_on_a_fresh_outbox(workspace: Path): + """The repair must not write a leading blank line into an empty file.""" + _emit(workspace, note="only") + assert _outbox_path(workspace).read_bytes().count(b"\n") == 1 + + +def test_torn_line_does_not_lose_the_earlier_event_to_a_reader(workspace: Path): + """The point of the repair, stated as the consumer sees it.""" + _emit(workspace, note="earlier") + outbox = _outbox_path(workspace) + outbox.write_bytes(outbox.read_bytes()[:-1]) + _emit(workspace, note="later") + + read = read_events_detailed(workspace, "c") + notes = [e.get("note") for e in read.events] + assert notes == ["earlier", "later"] + assert read.malformed == () + + +# --- W2: undecodable bytes -------------------------------------------------- + +def test_invalid_utf8_does_not_brick_emit(workspace: Path): + """A single bad byte must not make the outbox permanently unwritable. + + _current_seq() runs inside emit(); if it raises, EVERY future emit fails. + That is worse than a crash -- it is a permanent denial with no recovery + path short of deleting the file. + """ + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":1,"note":"ok"}\n\xff\xfe not utf-8\n') + _emit(workspace, note="after") # must not raise + assert any(b"after" in ln for ln in _lines(workspace)) + + +def test_invalid_utf8_is_counted_not_raised(workspace: Path): + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":1,"note":"ok"}\n\xff\xfe\n{"seq":2,"note":"two"}\n') + read = read_events_detailed(workspace, "c") + assert [e["note"] for e in read.events] == ["ok", "two"] + assert len(read.malformed) == 1 + assert "utf-8" in read.malformed[0].reason.lower() + + +def test_seq_allocation_survives_an_undecodable_line(workspace: Path): + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":7,"note":"ok"}\n\xff\n') + assert _current_seq(outbox) == 7 + + +# --- W3: hostile but syntactically valid content ---------------------------- + +@pytest.mark.parametrize( + "raw,why", + [ + (b'{"seq":1e999,"note":"inf"}', "float seq becomes inf and serializes as Infinity"), + (b'{"seq":"1","note":"str"}', "string seq cannot be ordered against the cursor"), + (b'{"seq":true,"note":"bool"}', "bool is an int subclass and max() accepts it silently"), + (b'{"seq":null,"note":"none"}', "null seq"), + (b'[1,2,3]', "a JSON array is not an event"), + (b'"just a string"', "a JSON string is not an event"), + (b'{"seq":1,', "truncated object"), + (b'not json at all', "not JSON"), + ], +) +def test_hostile_line_is_counted_and_never_raises(workspace: Path, raw: bytes, why: str): + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":1,"note":"good"}\n' + raw + b"\n") + read = read_events_detailed(workspace, "c") + assert [e["note"] for e in read.events] == ["good"], why + assert len(read.malformed) == 1, why + assert _current_seq(outbox) == 1, why + + +def test_deeply_nested_json_is_counted_not_raised(workspace: Path): + """RecursionError is raised by json.loads and is NOT a ValueError. + + The previous guard caught (JSONDecodeError, TypeError) and would have let + this escape -- a denylist over untrusted input leaks by construction. + """ + outbox = _outbox_path(workspace) + bomb = b"[" * 20000 + b"]" * 20000 + outbox.write_bytes(b'{"seq":1,"note":"good"}\n' + bomb + b"\n") + read = read_events_detailed(workspace, "c") + assert [e["note"] for e in read.events] == ["good"] + assert len(read.malformed) == 1 + + +def test_emit_still_allocates_a_usable_seq_beside_hostile_lines(workspace: Path): + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":3,"note":"good"}\n{"seq":1e999,"note":"inf"}\n') + _emit(workspace, note="next") + last = json.loads(_lines(workspace)[-1]) + assert last["seq"] == 4, "an unusable seq must not poison allocation" + json.dumps(last) # must remain serializable -- inf would emit `Infinity` + + +# --- W4: the count reaches a consumer --------------------------------------- + +def test_warn_unreadable_writes_to_the_given_stream(workspace: Path): + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":1,"note":"ok"}\n\xff\n') + read = read_events_detailed(workspace, "c") + buf = io.StringIO() + assert warn_unreadable(read, stream=buf) is True + assert "skipped 1 unreadable line" in buf.getvalue() + + +def test_warn_unreadable_is_silent_when_clean(workspace: Path): + """The negative case. Without it, a reporter that always warns passes.""" + _emit(workspace, note="fine") + read = read_events_detailed(workspace, "c") + buf = io.StringIO() + assert warn_unreadable(read, stream=buf) is False + assert buf.getvalue() == "" + + +def test_bridge_reports_on_stderr_and_still_posts_good_events(workspace: Path, capsys): + outbox = _outbox_path(workspace) + outbox.write_bytes( + b'{"seq":1,"type":"propagation.receipt","note":"ok"}\n\xff\n' + ) + posted: list[str] = [] + count = run_bridge(workspace, post_fn=posted.append) + captured = capsys.readouterr() + assert "unreadable" in captured.err, "the count never reached a consumer" + assert captured.out == "", "a health warning on stdout breaks --json callers" + assert count == len(posted) + + +def test_bridge_is_silent_on_a_clean_outbox(workspace: Path, capsys): + _emit(workspace, note="clean") + run_bridge(workspace, post_fn=lambda _m: None) + assert "unreadable" not in capsys.readouterr().err + + +# --- W5: the cursor is untrusted input too ---------------------------------- + +def test_a_corrupt_cursor_does_not_brick_reads(workspace: Path): + """The cursor is a file on disk; a truncated write makes last_seq a non-int. + + Comparing seq against it would raise from inside a reader whose contract is + to not raise on file content. + """ + _emit(workspace, note="one") + cursors = workspace / ".grip" / "events" / "cursors" + cursors.mkdir(parents=True, exist_ok=True) + (cursors / "c.json").write_text(json.dumps({"consumer": "c", "last_seq": "not-an-int"})) + read = read_events_detailed(workspace, "c") + assert [e["note"] for e in read.events] == ["one"] + + +# --- the wrapper keeps its shape ------------------------------------------- + +def test_read_events_wrapper_still_returns_a_plain_list(workspace: Path): + """Eleven call sites index and len() this. The shape is the contract. + + All of them are tests: no production caller of read_events() remains once the + bridge moves to read_events_detailed(). The wrapper is a test-compatibility + surface, not a load-bearing API. + """ + _emit(workspace, note="a") + events = read_events(workspace, "c") + assert isinstance(events, list) + assert len(events) == 1 + assert events[0]["note"] == "a" + + +# --- W6: an I/O failure is NOT a malformed line ----------------------------- +# +# This boundary was NOT found by any witness above. It was found by an existing +# test in test_events.py whose docstring stands guard over it, after the first +# version of this fix swallowed OSError inside the line iterator. Content errors +# are data and get skipped-and-counted; an I/O error is not knowing what the file +# holds, and on the WRITE path that becomes a duplicate sequence number. + +def test_emit_fails_closed_when_the_outbox_cannot_be_read(workspace: Path, monkeypatch): + """Swallowing this returns seq 0, and emit then reuses live sequence numbers.""" + from gr2.python_cli.events import EventEmitError + + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":41}\n') + original = Path.read_bytes + + def boom(path: Path, *a, **k): + if path == outbox: + raise OSError("forced sequence-read failure") + return original(path, *a, **k) + + monkeypatch.setattr(Path, "read_bytes", boom) + with pytest.raises(EventEmitError) as exc: + _emit(workspace, note="must not land") + assert isinstance(exc.value.__cause__, OSError) + # The read-back must not travel the sabotaged path -- my first version of + # this witness verified the file using the very method it had broken, and + # failed for that reason rather than for anything about emit(). + monkeypatch.undo() + assert outbox.read_bytes() == b'{"seq":41}\n', "a failed emit must not mutate the log" + + +def test_reader_tolerates_the_outbox_vanishing_mid_read(workspace: Path, monkeypatch): + """_maybe_rotate() renames the outbox, so this race is real and benign.""" + outbox = _outbox_path(workspace) + _emit(workspace, note="a") + original = Path.read_bytes + + def vanish(path: Path, *a, **k): + if path == outbox: + raise FileNotFoundError("rotated away") + return original(path, *a, **k) + + monkeypatch.setattr(Path, "read_bytes", vanish) + read = read_events_detailed(workspace, "c") + assert read.events == [] + assert read.malformed == () + + +def test_reader_propagates_a_real_io_error(workspace: Path, monkeypatch): + """The discriminating control for the case above. + + Without this, catching FileNotFoundError could widen to OSError and a reader + would report 'no new events' forever while the disk was failing. + """ + outbox = _outbox_path(workspace) + _emit(workspace, note="a") + original = Path.read_bytes + + def eio(path: Path, *a, **k): + if path == outbox: + raise OSError("EIO") + return original(path, *a, **k) + + monkeypatch.setattr(Path, "read_bytes", eio) + with pytest.raises(OSError): + read_events_detailed(workspace, "c") + + +# --- W7: the report must not amplify file content into logs ----------------- + +def test_default_report_does_not_echo_line_content(workspace: Path): + """stderr gets copied -- into CI logs, scrollback, pasted transcripts. + + Found by asking what this new output path could carry, not by a failing + test. `reason` is structural; the excerpt is content, and it is available on + the data object for callers that genuinely need it. + """ + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":1,"note":"ok"}\n{"token":"sk-NOTREAL-abcdef","broken\n') + read = read_events_detailed(workspace, "c") + buf = io.StringIO() + warn_unreadable(read, stream=buf) + assert "NOTREAL" not in buf.getvalue() + assert "line 2" in buf.getvalue(), "the line must still be identified" + assert "NOTREAL" in read.malformed[0].excerpt, "fidelity kept on the data object" + + +def test_show_content_opt_in_still_works(workspace: Path): + """The discriminating control: without it, an always-redacting reporter passes.""" + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":1,"note":"ok"}\n{"token":"sk-NOTREAL-abcdef","broken\n') + read = read_events_detailed(workspace, "c") + buf = io.StringIO() + warn_unreadable(read, stream=buf, show_content=True) + assert "NOTREAL" in buf.getvalue() + + +# --- W8: this module must import WITHOUT sys.modules registration ----------- + +def test_events_module_loads_under_an_out_of_tree_loader(tmp_path: Path): + """Spawned workers load events.py by path, without registering it. + + importlib.util.spec_from_file_location() + exec_module() leaves the module + OUT of sys.modules, and @dataclass resolves its field types through + sys.modules[cls.__module__].__dict__ -- so a dataclass here raises + AttributeError AT IMPORT and every worker dies before running its own code. + + That is not hypothetical: adding @dataclass to this module killed both + writers in the concurrent-emit integrity test before either reached + sequence allocation. This witness exists so the next person to reach for a + dataclass here finds out in one second instead of in a concurrency test + whose failure looks like flakiness. + """ + import importlib.util + + events_path = Path(__file__).resolve().parents[1] / "python_cli" / "events.py" + spec = importlib.util.spec_from_file_location("events_out_of_tree_probe", events_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + # DELIBERATELY NOT registered in sys.modules -- that is the whole point. + spec.loader.exec_module(module) + + assert hasattr(module, "read_events_detailed") + assert module.MalformedLine(1, "why", "x").ordinal == 1 + assert module.EventRead([], ()).malformed == () + + +# --- W9: how long an unreadable line keeps being reported -------------------- +# +# These exist because a REVIEWER measured this and the PR body claimed the +# opposite, with the whole suite green. Nothing pinned the real behavior, so +# prose and code were free to disagree. The reviewer found the TERMINAL case; +# measuring the mid-file case showed the same thing, so the rule is simpler and +# broader than the finding that produced it. + +def test_unreadable_line_is_reported_on_every_read_terminal(workspace: Path): + """A trailing unreadable line is counted again on the next read.""" + _emit(workspace, note="good") + outbox = _outbox_path(workspace) + outbox.write_bytes(outbox.read_bytes() + b"\xff not utf-8\n") + + first = read_events_detailed(workspace, "c") + second = read_events_detailed(workspace, "c") + assert [e["note"] for e in first.events] == ["good"] + assert len(first.malformed) == 1 + assert second.events == [], "the good event was consumed, as expected" + assert len(second.malformed) == 1, "the unreadable line is reported again" + + +def test_unreadable_line_is_reported_on_every_read_midfile(workspace: Path): + """MID-FILE too, which is the part the terminal framing would hide. + + The cursor filter `seq <= last_seq` only applies to lines that PARSE. A line + with no usable seq can never be advanced past, wherever it sits -- so + position is irrelevant and "trailing" is not the distinguishing property. + """ + outbox = _outbox_path(workspace) + outbox.write_bytes(b'{"seq":1,"note":"a"}\n\xff\n{"seq":2,"note":"b"}\n') + + first = read_events_detailed(workspace, "c") + second = read_events_detailed(workspace, "c") + assert [e["note"] for e in first.events] == ["a", "b"] + assert len(first.malformed) == 1 + assert second.events == [], "both good events were consumed" + assert len(second.malformed) == 1, "a mid-file unreadable line repeats too" + + +def test_glue_destroys_the_next_append_not_the_preceding_record(workspace: Path): + """The measured direction of the damage, which the body stated backwards. + + Tear after record B, then append C WITHOUT the repair. A is untouched; B and + C fuse into one unparseable line. The damage runs FORWARD from the tear. + """ + _emit(workspace, note="A") + _emit(workspace, note="B") + outbox = _outbox_path(workspace) + outbox.write_bytes(outbox.read_bytes()[:-1]) # tear after B + # bypass emit() to reproduce the pre-fix append + outbox.write_bytes( + outbox.read_bytes() + json.dumps({"seq": 3, "note": "C"}).encode("utf-8") + b"\n" + ) + + lines = _lines(workspace) + assert len(lines) == 2, "B and C fused into one line" + assert json.loads(lines[0])["note"] == "A", "the record BEFORE the tear survives" + with pytest.raises(ValueError): + json.loads(lines[1]) # B+C, unparseable + + +# --- W10: the TRUNCATED tear, which no fixture ABOVE this line exercises ----- +# +# "TORN" IS TWO DIFFERENT FAILURES WITH OPPOSITE OUTCOMES, AND USING THE BARE +# WORD IS WHAT MADE FOUR SEPARATE DESCRIPTIONS OF THIS CODE WRONG: +# +# UNTERMINATED -- the record is COMPLETE and only its "\n" was lost. Seam +# repair ends the line and it parses. RECOVERS FULLY, zero malformed. +# TRUNCATED -- the write stopped mid-record. Seam repair stops the next append +# being glued on, and the record itself NEVER becomes readable, because its +# bytes were never written. +# +# Every terminator fixture ABOVE this line uses read_bytes()[:-1] -- the +# UNTERMINATED case, and the lucky one. The fixtures below are the truncated +# case. That gap is why prose describing this code kept being wrong: the only +# tear ever exercised was the one that recovers. +# +# Measured: unterminated -> 3 events, 0 malformed. Truncated -> 2 events, 1 +# malformed, and still 1 on the second and third read. +# +# What the repair actually buys, stated so it holds for BOTH cases: it cannot +# recover a record that was never fully written, and it saves the NEXT one. +# Without repair you lose two records; with it you lose the one the crash +# truncated -- and if the crash truncated nothing, you lose none. + +def _tear_mid_record(outbox: Path) -> None: + """Truncate the last record mid-JSON, as a partial write leaves it.""" + lines = outbox.read_bytes().split(b"\n") + body = [ln for ln in lines if ln.strip()] + body[-1] = body[-1][:20] + outbox.write_bytes(b"\n".join(body)) # no trailing newline either + + +def test_realistic_tear_saves_the_next_append_but_not_the_torn_record(workspace: Path): + _emit(workspace, note="A") + _emit(workspace, note="B") + outbox = _outbox_path(workspace) + _tear_mid_record(outbox) + + _emit(workspace, note="C") + + read = read_events_detailed(workspace, "c") + notes = [e.get("note") for e in read.events] + assert notes == ["A", "C"], "the NEXT append survives; the truncated record cannot" + assert len(read.malformed) == 1, "the truncated record is unreadable, permanently" + + +def test_a_truncated_record_never_heals(workspace: Path): + """A TRUNCATED record is reported on every read, forever; no emit repairs it. + + THE SEAM HEALS; A TRUNCATED RECORD DOES NOT. The next append is no longer + glued on, which is the whole of what the repair achieves for this case -- + the truncated record's bytes were never written and nothing can reconstruct + them. An UNTERMINATED record is the other case entirely and does recover; + its witness is directly below and the two are each other's control. + """ + _emit(workspace, note="A") + _emit(workspace, note="B") + outbox = _outbox_path(workspace) + _tear_mid_record(outbox) + _emit(workspace, note="C") + + counts = [len(read_events_detailed(workspace, "c").malformed) for _ in range(3)] + assert counts == [1, 1, 1], f"expected a permanent malformed line, got {counts}" + + +def test_unterminated_tear_DOES_fully_recover(workspace: Path): + """The discriminating control, and it has already earned its keep twice. + + Without it the two witnesses above would also pass against an implementation + that simply never recovers anything, and the truncated case would read as the + norm. It then caught its own author: a draft of the PR body claimed a torn + record never becomes readable, and THIS witness disproves that. An + UNTERMINATED record -- complete, only its newline lost -- must come back with + zero malformed lines. + """ + _emit(workspace, note="A") + _emit(workspace, note="B") + outbox = _outbox_path(workspace) + outbox.write_bytes(outbox.read_bytes()[:-1]) # newline only + + _emit(workspace, note="C") + + read = read_events_detailed(workspace, "c") + assert [e.get("note") for e in read.events] == ["A", "B", "C"] + assert read.malformed == () + + +# --- W11: the READER against a torn file, with NO intervening emit ---------- +# +# Every tear fixture above emits after tearing, so until this pair nothing ever +# asked what the READER alone does with a torn file. That gap is exactly how a +# false sentence survived into the bridge comment: it said an unterminated +# record is reported and then heals at the next emit, implying a window in +# which it is unreadable. There is no such window. _iter_outbox() splits on +# b"\n", so a COMPLETE record whose only loss was its terminator is simply the +# final chunk, and it parses -- repair or no repair, emit or no emit. +# +# The truncated case below is the discriminating control: without it this pair +# would also pass against a reader that never reported anything at all. + + +def test_unterminated_record_is_never_reported_even_without_an_emit(workspace: Path): + """A complete record missing only its "\n" is readable immediately. + + Not "recovers at the next emit" -- never unreadable in the first place. + Read twice, because a claim of transience would show as a difference + between the reads. + """ + _emit(workspace, note="A") + _emit(workspace, note="B") + outbox = _outbox_path(workspace) + outbox.write_bytes(outbox.read_bytes()[:-1]) # newline only; no emit follows + + for i in (1, 2): + read = read_events_detailed(workspace, f"c{i}") + assert [e.get("note") for e in read.events] == ["A", "B"], f"read {i}" + assert read.malformed == (), f"read {i}: expected nothing reported, got {read.malformed}" + + +def test_truncated_record_IS_reported_without_an_emit(workspace: Path): + """The discriminating control for the witness above. + + Byte-identical setup except the tear cuts mid-record instead of at the + terminator. If this one also came back empty, the witness above would be + measuring a reader that reports nothing rather than a record that is + readable. + """ + _emit(workspace, note="A") + _emit(workspace, note="B") + outbox = _outbox_path(workspace) + _tear_mid_record(outbox) # no emit follows + + for i in (1, 2): + read = read_events_detailed(workspace, f"c{i}") + assert [e.get("note") for e in read.events] == ["A"], f"read {i}" + assert len(read.malformed) == 1, f"read {i}: expected the truncated record reported" From a9263bf1bc12532bad1c1199946255b259f42315 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Fri, 21 Aug 2026 06:35:59 -0500 Subject: [PATCH 24/29] fix(checkout): count repo failures and disjoin them into the exit code gr checkout reported every per-repo failure to the terminal and incremented no counter, so a run in which every repo failed printed "Switched 0/N repos to " and exited 0. The printed ratio is something a caller has to read and interpret; the exit code is the only failure signal a script sees, and it reported the batch as done. Adopt cli/repo_iter::for_each_repo -- which is what gives this command an error count at all -- and make a nonzero exit disjoin the per-repo failures. Two changes fall out of the adoption: - for_each_repo's cloned-check called path_exists on the repo directory while its own docstring promised to skip repos that "aren't cloned". RepoInfo::exists tests for .git, which is what cloned means; the two diverge on a directory that exists and is not a clone. Aligning the code with the docstring preserves checkout's existing behavior exactly rather than reclassifying a skip as an error. - for_each_repo_path had no callers and gains none here, so it is removed rather than left relying on pub visibility to keep its own dead-code warning quiet. The plan step prescribing that suppression is struck in the same change. An unused private item warns and an unused pub item does not, so following the step as written left the module with zero callers and nothing anywhere going red for as long as it existed. A warning is a detector; silencing one to reach a clean build is not a fix. Tests pin both sides of one property. The discriminator is an ABSENT .git versus a PRESENT BUT UNOPENABLE one: absent is a skip and still exits 0, present but unopenable is an error and exits nonzero. Without both sides, a witness asserting only the failure case could pass while the skip path had silently become an error too. These are two separate tests with their own fixtures, and those fixtures differ in more than the discriminating property -- one corrupts a single repo and the other both, one creates a branch first, and they target different branches. The claim is about which property discriminates, not about the fixtures being otherwise identical. --- docs/IMPLEMENTATION_PLAN.md | 2 +- docs/PLAN-p2-maintainability.md | 6 +- src/cli/commands/checkout.rs | 103 ++++++++++++++------------------ src/cli/repo_iter.rs | 55 +++-------------- tests/test_checkout.rs | 64 ++++++++++++++++++++ 5 files changed, 122 insertions(+), 108 deletions(-) diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 16bd1da..69c54bf 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -59,7 +59,7 @@ Pass `--hostname` to `gh` CLI in `src/platform/github.rs` `enable_auto_merge()` ### Phase 3: Repo iteration helper - [x] New `src/cli/repo_iter.rs`: `RepoVisitResult`, `RepoOpSummary`, `for_each_repo()`, `for_each_repo_path()` -- [ ] Wire into commands — Deferred: most commands accumulate custom state that doesn't fit the simple Success/Skipped/Error enum +- [ ] Wire into commands — Partially done: `checkout` adopted it (2026-08-21). The remaining commands are blocked on `?` propagation out of the loop, three disagreeing skip taxonomies, and per-arm payload types — not on the enum alone ### Phase 4-6: Migrate all commands to WorkspaceContext - [x] 28/30 commands in main.rs use `load_workspace_context()` (Init, Completions, Bench don't need workspace) diff --git a/docs/PLAN-p2-maintainability.md b/docs/PLAN-p2-maintainability.md index 9af36a1..64ca255 100644 --- a/docs/PLAN-p2-maintainability.md +++ b/docs/PLAN-p2-maintainability.md @@ -24,7 +24,9 @@ No conflicts expected — P0/P1 didn't touch `main.rs` or `cli/mod.rs`. ``` cargo build && cargo test && cargo clippy && cargo fmt --check ``` -Watch for clippy warnings on unused `repo_iter` imports. If flagged, the `pub` visibility from `cli/mod.rs` → `lib.rs` should suppress it. +~~Watch for clippy warnings on unused `repo_iter` imports. If flagged, the `pub` visibility from `cli/mod.rs` → `lib.rs` should suppress it.~~ + +**Struck.** This prescribed silencing the one instrument that would have reported the problem. `pub` does not resolve a dead-code warning, it disables the analysis — an unused private item warns, an unused `pub` item does not — so following this step left `repo_iter` with zero callers and nothing anywhere going red for as long as it existed. A warning is a detector; suppressing it to reach a clean build is not a fix, and writing the suppression down as a step made it the default for whoever came next. If an item has no consumer, give it one or delete it. ### 3. Commit the refactor Stage all 4 files and commit: @@ -59,7 +61,7 @@ gr pr create -t "refactor: P2 maintainability — WorkspaceContext and load_grip | Command signature migration to `&WorkspaceContext` | Would touch every command file + every test; current ctx field extraction in dispatch works fine | | Compact dispatch function (Phase 7) | 612-line match for 30 commands is standard; only one dispatch site | | sync.rs / release.rs decomposition (Phase 8) | Already have helpers (`sync_single_repo`, `execute_post_sync_hooks`, etc.) | -| Wiring `for_each_repo()` into commands | Most commands accumulate custom state that doesn't fit the simple Success/Skipped/Error enum | +| Wiring `for_each_repo()` into commands | Largely accurate, and now measured rather than assumed. Of the 9 command files that hand-roll these counters, `sync`/`pull` do not iterate repos at all, `pr/merge` iterates PRs and awaits, and `push`/`commit`/`forall` propagate `?` out of the loop — which the closure's return type cannot express. `checkout` was the one clean fit and has adopted it. | ## Files touched diff --git a/src/cli/commands/checkout.rs b/src/cli/commands/checkout.rs index 87888eb..b3e90dd 100644 --- a/src/cli/commands/checkout.rs +++ b/src/cli/commands/checkout.rs @@ -1,15 +1,13 @@ //! Checkout command implementation use crate::cli::output::Output; +use crate::cli::repo_iter::{for_each_repo, RepoVisitResult}; use crate::core::manifest::Manifest; use crate::core::repo::{ filter_repos, get_manifest_repo_info, validate_repo_filters_known, RepoInfo, }; use crate::core::workspace_checkout; -use crate::git::{ - branch::{branch_exists, checkout_branch, create_and_checkout_branch}, - open_repo, -}; +use crate::git::branch::{branch_exists, checkout_branch, create_and_checkout_branch}; use std::path::Path; /// Run the checkout command @@ -51,71 +49,62 @@ pub fn run_checkout( )); println!(); - let mut success_count = 0; - let mut _skip_count = 0; - - for repo in &repos { - if !repo.exists() { - Output::warning(&format!("{}: not cloned", repo.name)); - _skip_count += 1; - continue; - } - - match open_repo(&repo.absolute_path) { - Ok(git_repo) => { - let exists = branch_exists(&git_repo, branch_name); - - if create { - // -b flag: create if doesn't exist, checkout if it does - if exists { - match checkout_branch(&git_repo, branch_name) { - Ok(()) => { - Output::success(&format!( - "{}: checked out (already exists)", - repo.name - )); - success_count += 1; - } - Err(e) => Output::error(&format!("{}: {}", repo.name, e)), - } - } else { - match create_and_checkout_branch(&git_repo, branch_name) { - Ok(()) => { - Output::success(&format!("{}: created and checked out", repo.name)); - success_count += 1; - } - Err(e) => Output::error(&format!("{}: {}", repo.name, e)), - } - } - } else { - // Normal checkout: skip if branch doesn't exist - if !exists { - Output::info(&format!("{}: branch doesn't exist, skipping", repo.name)); - _skip_count += 1; - continue; - } - - match checkout_branch(&git_repo, branch_name) { - Ok(()) => { - Output::success(&repo.name); - success_count += 1; - } - Err(e) => Output::error(&format!("{}: {}", repo.name, e)), + // Iterating through for_each_repo rather than by hand is what gives this + // command an error count at all. The previous loop reported every failure + // to the terminal and incremented nothing, so the summary below counted + // successes against a total and stayed silent about the difference. + let summary = for_each_repo(&repos, false, |repo, git_repo| { + let exists = branch_exists(git_repo, branch_name); + + if create { + // -b flag: create if doesn't exist, checkout if it does + if exists { + match checkout_branch(git_repo, branch_name) { + Ok(()) => RepoVisitResult::Success(format!( + "{}: checked out (already exists)", + repo.name + )), + Err(e) => RepoVisitResult::Error(format!("{}: {}", repo.name, e)), + } + } else { + match create_and_checkout_branch(git_repo, branch_name) { + Ok(()) => { + RepoVisitResult::Success(format!("{}: created and checked out", repo.name)) } + Err(e) => RepoVisitResult::Error(format!("{}: {}", repo.name, e)), } } - Err(e) => Output::error(&format!("{}: {}", repo.name, e)), + } else if !exists { + // Normal checkout: a missing branch is a skip, not a failure. + RepoVisitResult::Skipped(format!("{}: branch doesn't exist, skipping", repo.name)) + } else { + match checkout_branch(git_repo, branch_name) { + Ok(()) => RepoVisitResult::Success(repo.name.clone()), + Err(e) => RepoVisitResult::Error(format!("{}: {}", repo.name, e)), + } } - } + }); println!(); println!( "Switched {}/{} repos to {}", - success_count, + summary.success_count, repos.len(), Output::branch_name(branch_name) ); + // The count above is a ratio a caller has to read and interpret. The exit + // code is the only failure signal a script sees, so it has to disjoin the + // per-repo failures rather than report the batch as done. + if summary.error_count > 0 { + anyhow::bail!( + "{} of {} repos failed to switch to {}", + summary.error_count, + repos.len(), + branch_name + ); + } + Ok(()) } diff --git a/src/cli/repo_iter.rs b/src/cli/repo_iter.rs index 24f3851..f99f1c5 100644 --- a/src/cli/repo_iter.rs +++ b/src/cli/repo_iter.rs @@ -5,7 +5,7 @@ use crate::cli::output::Output; use crate::core::repo::RepoInfo; -use crate::git::{open_repo, path_exists}; +use crate::git::open_repo; use git2::Repository; /// Result of visiting a single repo @@ -45,7 +45,12 @@ where }; for repo in repos { - if !path_exists(&repo.absolute_path) { + // `RepoInfo::exists` tests for `.git`, which is what "cloned" means. + // A bare `path_exists` on the directory answers a different question: + // a checkout whose `.git` is gone still has its files, so it would + // pass that check and then fail to open, turning a not-cloned skip + // into an error. This is what the docstring above has always claimed. + if !repo.exists() { if !quiet { Output::warning(&format!("{}: not cloned", repo.name)); } @@ -81,49 +86,3 @@ where summary } - -/// Iterate over repos by path (without opening git2::Repository). -/// -/// Useful for operations that shell out to `git` directly rather than -/// using libgit2 (e.g., cherry-pick, gc). -pub fn for_each_repo_path(repos: &[RepoInfo], quiet: bool, mut op: F) -> RepoOpSummary -where - F: FnMut(&RepoInfo) -> RepoVisitResult, -{ - let mut summary = RepoOpSummary { - success_count: 0, - skip_count: 0, - error_count: 0, - }; - - for repo in repos { - if !path_exists(&repo.absolute_path) { - if !quiet { - Output::warning(&format!("{}: not cloned", repo.name)); - } - summary.skip_count += 1; - continue; - } - - match op(repo) { - RepoVisitResult::Success(msg) => { - if !quiet { - Output::success(&msg); - } - summary.success_count += 1; - } - RepoVisitResult::Skipped(msg) => { - if !quiet { - Output::info(&msg); - } - summary.skip_count += 1; - } - RepoVisitResult::Error(msg) => { - Output::error(&msg); - summary.error_count += 1; - } - } - } - - summary -} diff --git a/tests/test_checkout.rs b/tests/test_checkout.rs index 847a0ce..f75cc27 100644 --- a/tests/test_checkout.rs +++ b/tests/test_checkout.rs @@ -635,3 +635,67 @@ fn test_absolute_repo_path_in_metadata_cannot_escape_the_checkout() { "a rejected reconstruction should explain why, not fail silently" ); } + +// ── Every repo fails to open ──────────────────────────────────── +// Witness for the error arms, which previously reported each failure to the +// terminal and incremented no counter, so the command printed "Switched 0/N" +// and exited 0. Each .git is replaced by an empty directory: the repo still +// counts as cloned under either cloned-check, so this exercises the error arm +// rather than the not-cloned skip arm. + +#[test] +fn test_checkout_all_repos_failing_is_not_success() { + let ws = WorkspaceBuilder::new() + .add_repo("frontend") + .add_repo("backend") + .build(); + + let manifest = ws.load_manifest(); + + for name in ["frontend", "backend"] { + let git_dir = ws.repo_path(name).join(".git"); + std::fs::remove_dir_all(&git_dir).unwrap(); + std::fs::create_dir(&git_dir).unwrap(); + assert!( + git_dir.exists(), + "{name}: .git must still exist for this witness" + ); + } + + let result = gitgrip::cli::commands::checkout::run_checkout( + &ws.workspace_root, + &manifest, + "main", + false, + None, + None, + ); + + assert!( + result.is_err(), + "every repo failed to open; checkout must not report success" + ); +} + +// The companion that keeps the fix from over-correcting: a branch that is +// simply absent is a skip, not a failure, and must still exit zero. +#[test] +fn test_checkout_absent_branch_is_still_success() { + let ws = WorkspaceBuilder::new().add_repo("frontend").build(); + let manifest = ws.load_manifest(); + + let result = gitgrip::cli::commands::checkout::run_checkout( + &ws.workspace_root, + &manifest, + "no-such-branch", + false, + None, + None, + ); + + assert!( + result.is_ok(), + "an absent branch is a skip, not a failure: {:?}", + result.err() + ); +} From af17ae9d0e67599a354fccdac976209408d53e52 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Fri, 21 Aug 2026 05:05:30 -0500 Subject: [PATCH 25/29] feat(workspace): initialize spec from topology --- gr2/python_cli/app.py | 133 +++++++++++++- .../test_workspace_init_from_topology.py | 165 ++++++++++++++++++ 2 files changed, 290 insertions(+), 8 deletions(-) create mode 100644 gr2/tests/test_workspace_init_from_topology.py diff --git a/gr2/python_cli/app.py b/gr2/python_cli/app.py index 095fa30..d691734 100644 --- a/gr2/python_cli/app.py +++ b/gr2/python_cli/app.py @@ -4,6 +4,7 @@ import io import json import os +import tomllib from pathlib import Path from types import SimpleNamespace from typing import Optional @@ -291,32 +292,67 @@ def _configured_merge_method(workspace_root: Path) -> str | None: return value +def _toml_basic_string(value: str) -> str: + """Render one string through the TOML basic-string grammar. + + WorkspaceSpec values are operator-controlled at several call sites. Keeping + the escaping here makes every value written by ``_write_workspace_spec`` + parseable, rather than relying on each caller to reject a partial set of + characters. + """ + if not isinstance(value, str): + raise TypeError(f"TOML basic string requires str, got {type(value).__name__}") + + escapes = { + '"': '\\"', + "\\": "\\\\", + "\b": "\\b", + "\t": "\\t", + "\n": "\\n", + "\f": "\\f", + "\r": "\\r", + } + rendered: list[str] = ['"'] + for character in value: + code_point = ord(character) + if character in escapes: + rendered.append(escapes[character]) + elif code_point < 0x20 or code_point == 0x7F: + rendered.append(f"\\u{code_point:04X}") + elif 0xD800 <= code_point <= 0xDFFF: + raise ValueError("TOML basic strings cannot contain surrogate code points") + else: + rendered.append(character) + rendered.append('"') + return "".join(rendered) + + def _write_workspace_spec(workspace_root: Path, repos: list[dict[str, str]], default_unit: str) -> Path: spec_path = _workspace_spec_path(workspace_root) - spec_path.parent.mkdir(parents=True, exist_ok=True) lines = [ - f'workspace_name = "{workspace_root.name}"', + f"workspace_name = {_toml_basic_string(workspace_root.name)}", "", ] for repo in repos: lines.extend( [ "[[repos]]", - f'name = "{repo["name"]}"', - f'path = "{repo["path"]}"', - f'url = "{repo["url"]}"', + f"name = {_toml_basic_string(repo['name'])}", + f"path = {_toml_basic_string(repo['path'])}", + f"url = {_toml_basic_string(repo['url'])}", "", ] ) lines.extend( [ "[[units]]", - f'name = "{default_unit}"', - f'path = "agents/{default_unit}/home"', - "repos = [" + ", ".join(f'"{repo["name"]}"' for repo in repos) + "]", + f"name = {_toml_basic_string(default_unit)}", + f"path = {_toml_basic_string(f'agents/{default_unit}/home')}", + "repos = [" + ", ".join(_toml_basic_string(repo["name"]) for repo in repos) + "]", "", ] ) + spec_path.parent.mkdir(parents=True, exist_ok=True) spec_path.write_text("\n".join(lines)) return spec_path @@ -343,6 +379,53 @@ def _scan_existing_repos(workspace_root: Path) -> list[dict[str, str]]: return repos +def _declared_repos_from_workspace_topology(workspace_root: Path) -> list[dict[str, str]]: + """Lower neutral ``workspace.toml`` repository declarations for gr2. + + This deliberately reads only the fields the existing WorkspaceSpec writer + accepts. Declarations can carry fields such as ``default_ref`` as well, + but those do not belong in the WorkspaceSpec emission path. + """ + topology_path = workspace_root / "workspace.toml" + if not topology_path.is_file(): + raise SystemExit(f"workspace topology not found: {topology_path}") + try: + with topology_path.open("rb") as topology_file: + document = tomllib.load(topology_file) + except tomllib.TOMLDecodeError as exc: + raise SystemExit(f"workspace.toml at {topology_path} is not valid TOML: {exc}") from exc + + raw_repos = document.get("repos", []) + if not isinstance(raw_repos, list): + raise SystemExit("workspace.toml repos must be an array of tables") + if not raw_repos: + raise SystemExit("workspace.toml declares no [[repos]] entries") + + repos: list[dict[str, str]] = [] + for index, raw_repo in enumerate(raw_repos): + if not isinstance(raw_repo, dict): + raise SystemExit(f"workspace.toml repos[{index}] must be a table") + key = str(raw_repo.get("key", "")) + for field in ("key", "path", "url"): + value = raw_repo.get(field) + if not value: + raise SystemExit( + f"workspace.toml repos[{index}] ({key!r}) is missing {field!r}" + ) + if not isinstance(value, str): + raise SystemExit( + f"workspace.toml repos[{index}] ({key!r}) must declare {field!r} as a string" + ) + repos.append( + { + "name": str(raw_repo["key"]), + "path": str(raw_repo["path"]), + "url": str(raw_repo["url"]), + } + ) + return repos + + def _exit(code: int) -> None: if code != 0: raise typer.Exit(code=code) @@ -414,6 +497,40 @@ def workspace_init( typer.echo("\n".join(lines)) +@workspace_app.command("init-from-topology") +def workspace_init_from_topology( + workspace_root: Path, + default_unit: str = typer.Option("default", help="Default owner unit for declared repos"), + json_output: bool = typer.Option(False, "--json", help="Emit machine-readable JSON"), +) -> None: + """Create WorkspaceSpec from neutral ``workspace.toml`` repo declarations.""" + workspace_root = workspace_root.resolve() + repos = _declared_repos_from_workspace_topology(workspace_root) + spec_path = _write_workspace_spec(workspace_root, repos, default_unit) + payload = { + "workspace_root": str(workspace_root), + "spec_path": str(spec_path), + "repo_count": len(repos), + "repos": repos, + "default_unit": default_unit, + "source": "workspace.toml", + } + if json_output: + typer.echo(json.dumps(payload, indent=2)) + else: + lines = [ + "WorkspaceInitFromTopology", + f"workspace_root = {workspace_root}", + f"spec_path = {spec_path}", + f"default_unit = {default_unit}", + f"repo_count = {len(repos)}", + "source = workspace.toml", + "REPOS", + ] + lines.extend(f"- {repo['name']}\t{repo['path']}\t{repo['url']}" for repo in repos) + typer.echo("\n".join(lines)) + + @workspace_app.command("materialize") def workspace_materialize( workspace_root: Path, diff --git a/gr2/tests/test_workspace_init_from_topology.py b/gr2/tests/test_workspace_init_from_topology.py new file mode 100644 index 0000000..2149c54 --- /dev/null +++ b/gr2/tests/test_workspace_init_from_topology.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +import tomllib +from pathlib import Path + +from gr2.python_cli.app import app +from typer.testing import CliRunner + +runner = CliRunner() + + +def _write_topology(workspace_root: Path) -> None: + (workspace_root / "workspace.toml").write_text( + """\ +schema_version = 2 +workspace_name = "declarative-team" + +[[repos]] +key = "product" +url = "https://example.invalid/product.git" +path = "repos/product" +default_ref = "main" + +[[repos]] +key = "config" +url = "https://example.invalid/config.git" +path = "config" +default_ref = "dev" +""" + ) + + +def test_workspace_init_from_topology_writes_declared_repos_in_an_empty_directory( + tmp_path: Path, +) -> None: + """The zero-to-team wire must not fall back to scanning local Git repos. + + Mutation: replace the declared-topology reader with `_scan_existing_repos`. + This root has no Git repositories, so the old adoption path refuses and this + witness goes red before a WorkspaceSpec can be written. + """ + workspace_root = tmp_path / "empty-team" + workspace_root.mkdir() + _write_topology(workspace_root) + + result = runner.invoke( + app, + [ + "workspace", + "init-from-topology", + str(workspace_root), + "--default-unit", + "team", + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["workspace_root"] == str(workspace_root) + assert payload["spec_path"] == str(workspace_root / ".grip" / "workspace_spec.toml") + assert payload["repo_count"] == 2 + assert [repo["name"] for repo in payload["repos"]] == ["product", "config"] + assert [repo["path"] for repo in payload["repos"]] == ["repos/product", "config"] + assert [repo["url"] for repo in payload["repos"]] == [ + "https://example.invalid/product.git", + "https://example.invalid/config.git", + ] + assert payload["default_unit"] == "team" + assert payload["source"] == "workspace.toml" + assert not list(workspace_root.glob("*/.git")) + assert not (workspace_root / "repos" / "product").exists() + assert not (workspace_root / "config").exists() + + with (workspace_root / ".grip" / "workspace_spec.toml").open("rb") as spec_file: + spec = tomllib.load(spec_file) + assert spec["repos"] == payload["repos"] + assert spec["units"] == [ + { + "name": "team", + "path": "agents/team/home", + "repos": ["product", "config"], + } + ] + + text_result = runner.invoke( + app, + ["workspace", "init-from-topology", str(workspace_root), "--default-unit", "team"], + ) + assert text_result.exit_code == 0, text_result.output + assert "source = workspace.toml" in text_result.output + + +def test_workspace_init_from_topology_refuses_an_incomplete_declared_repo_before_writing( + tmp_path: Path, +) -> None: + workspace_root = tmp_path / "incomplete-team" + workspace_root.mkdir() + (workspace_root / "workspace.toml").write_text( + """\ +[[repos]] +key = "product" +path = "repos/product" +default_ref = "main" +""" + ) + + result = runner.invoke(app, ["workspace", "init-from-topology", str(workspace_root)]) + + assert result.exit_code != 0 + assert "workspace.toml repos[0] ('product') is missing 'url'" in result.output + assert not (workspace_root / ".grip" / "workspace_spec.toml").exists() + + +def test_workspace_init_from_topology_serializes_every_writer_value( + tmp_path: Path, +) -> None: + """Hostile strings must round-trip through every WorkspaceSpec value slot. + + Mutation: replace the writer serializer with raw interpolation. The command + still exits 0, but this direct parse of its bytes fails. The witness thus + catches the WRONG-BUT-GREEN failure mode rather than only a refusal path. + """ + workspace_root = tmp_path / 'unsafe"team' + workspace_root.mkdir() + (workspace_root / "workspace.toml").write_text( + r''' +[[repos]] +key = 'product"\\name' +url = 'https://example.invalid/a"\\b.git' +path = 'repos/product"\\path' +''' + ) + + default_unit = 'team"\\unit' + result = runner.invoke( + app, + [ + "workspace", + "init-from-topology", + str(workspace_root), + "--default-unit", + default_unit, + ], + ) + + assert result.exit_code == 0, result.output + with (workspace_root / ".grip" / "workspace_spec.toml").open("rb") as spec_file: + spec = tomllib.load(spec_file) + assert spec["workspace_name"] == workspace_root.name + assert spec["repos"] == [ + { + "name": r'product"\\name', + "path": r'repos/product"\\path', + "url": r'https://example.invalid/a"\\b.git', + } + ] + assert spec["units"] == [ + { + "name": default_unit, + "path": f"agents/{default_unit}/home", + "repos": [r'product"\\name'], + } + ] From 01ef91181bbc6038b6c59ed3b6b9cc7675bd1d82 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Fri, 21 Aug 2026 07:54:14 -0500 Subject: [PATCH 26/29] test(workspace): cover encoding refusal before directory creation --- .../test_workspace_init_from_topology.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/gr2/tests/test_workspace_init_from_topology.py b/gr2/tests/test_workspace_init_from_topology.py index 2149c54..be1b8a0 100644 --- a/gr2/tests/test_workspace_init_from_topology.py +++ b/gr2/tests/test_workspace_init_from_topology.py @@ -113,6 +113,41 @@ def test_workspace_init_from_topology_refuses_an_incomplete_declared_repo_before assert not (workspace_root / ".grip" / "workspace_spec.toml").exists() +def test_workspace_init_from_topology_refuses_an_encoding_error_before_creating_grip_directory( + tmp_path: Path, +) -> None: + """A writer encoding refusal must leave neither the spec nor its directory. + + TOML itself rejects a surrogate escape in ``workspace.toml`` before the + command reaches the writer, so this uses the CLI-owned default-unit value, + one of the writer's seven string positions, to exercise the serializer. + + Mutation: move ``spec_path.parent.mkdir`` above ``lines`` construction. + The command still refuses, but this witness reaches its directory assertion + and fails while the three pre-existing witnesses remain green. + """ + workspace_root = tmp_path / "encoding-refusal-team" + workspace_root.mkdir() + _write_topology(workspace_root) + + result = runner.invoke( + app, + [ + "workspace", + "init-from-topology", + str(workspace_root), + "--default-unit", + "team-\ud800", + ], + ) + + assert result.exit_code != 0 + assert isinstance(result.exception, ValueError) + assert "surrogate" in str(result.exception) + assert not (workspace_root / ".grip" / "workspace_spec.toml").exists() + assert not (workspace_root / ".grip").exists() + + def test_workspace_init_from_topology_serializes_every_writer_value( tmp_path: Path, ) -> None: From 3bce8c25433608733a11dac95b71048802173186 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Fri, 21 Aug 2026 08:49:48 -0500 Subject: [PATCH 27/29] fix(workspace): honor declared topology name --- gr2/python_cli/app.py | 30 ++++++++++++---- .../test_workspace_init_from_topology.py | 34 ++++++++++++++++--- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/gr2/python_cli/app.py b/gr2/python_cli/app.py index d691734..2a7aad0 100644 --- a/gr2/python_cli/app.py +++ b/gr2/python_cli/app.py @@ -327,10 +327,17 @@ def _toml_basic_string(value: str) -> str: return "".join(rendered) -def _write_workspace_spec(workspace_root: Path, repos: list[dict[str, str]], default_unit: str) -> Path: +def _write_workspace_spec( + workspace_root: Path, + repos: list[dict[str, str]], + default_unit: str, + *, + workspace_name: str | None = None, +) -> Path: spec_path = _workspace_spec_path(workspace_root) + emitted_workspace_name = workspace_root.name if workspace_name is None else workspace_name lines = [ - f"workspace_name = {_toml_basic_string(workspace_root.name)}", + f"workspace_name = {_toml_basic_string(emitted_workspace_name)}", "", ] for repo in repos: @@ -379,7 +386,9 @@ def _scan_existing_repos(workspace_root: Path) -> list[dict[str, str]]: return repos -def _declared_repos_from_workspace_topology(workspace_root: Path) -> list[dict[str, str]]: +def _declared_workspace_topology( + workspace_root: Path, +) -> tuple[str | None, list[dict[str, str]]]: """Lower neutral ``workspace.toml`` repository declarations for gr2. This deliberately reads only the fields the existing WorkspaceSpec writer @@ -395,6 +404,10 @@ def _declared_repos_from_workspace_topology(workspace_root: Path) -> list[dict[s except tomllib.TOMLDecodeError as exc: raise SystemExit(f"workspace.toml at {topology_path} is not valid TOML: {exc}") from exc + workspace_name = document.get("workspace_name") + if workspace_name is not None and not isinstance(workspace_name, str): + raise SystemExit("workspace.toml workspace_name must be a string when declared") + raw_repos = document.get("repos", []) if not isinstance(raw_repos, list): raise SystemExit("workspace.toml repos must be an array of tables") @@ -423,7 +436,7 @@ def _declared_repos_from_workspace_topology(workspace_root: Path) -> list[dict[s "url": str(raw_repo["url"]), } ) - return repos + return workspace_name, repos def _exit(code: int) -> None: @@ -505,8 +518,13 @@ def workspace_init_from_topology( ) -> None: """Create WorkspaceSpec from neutral ``workspace.toml`` repo declarations.""" workspace_root = workspace_root.resolve() - repos = _declared_repos_from_workspace_topology(workspace_root) - spec_path = _write_workspace_spec(workspace_root, repos, default_unit) + workspace_name, repos = _declared_workspace_topology(workspace_root) + spec_path = _write_workspace_spec( + workspace_root, + repos, + default_unit, + workspace_name=workspace_name, + ) payload = { "workspace_root": str(workspace_root), "spec_path": str(spec_path), diff --git a/gr2/tests/test_workspace_init_from_topology.py b/gr2/tests/test_workspace_init_from_topology.py index be1b8a0..1c77ca8 100644 --- a/gr2/tests/test_workspace_init_from_topology.py +++ b/gr2/tests/test_workspace_init_from_topology.py @@ -10,11 +10,18 @@ runner = CliRunner() -def _write_topology(workspace_root: Path) -> None: +def _write_topology( + workspace_root: Path, + *, + workspace_name: str | None = "declarative-team", +) -> None: + declared_workspace_name = ( + f'workspace_name = "{workspace_name}"\n' if workspace_name is not None else "" + ) (workspace_root / "workspace.toml").write_text( - """\ + f"""\ schema_version = 2 -workspace_name = "declarative-team" +{declared_workspace_name} [[repos]] key = "product" @@ -75,6 +82,7 @@ def test_workspace_init_from_topology_writes_declared_repos_in_an_empty_director with (workspace_root / ".grip" / "workspace_spec.toml").open("rb") as spec_file: spec = tomllib.load(spec_file) + assert spec["workspace_name"] == "declarative-team" assert spec["repos"] == payload["repos"] assert spec["units"] == [ { @@ -92,6 +100,22 @@ def test_workspace_init_from_topology_writes_declared_repos_in_an_empty_director assert "source = workspace.toml" in text_result.output +def test_workspace_init_from_topology_falls_back_to_root_name_when_name_is_absent( + tmp_path: Path, +) -> None: + """The pre-existing scan-style fallback remains explicit when topology omits a name.""" + workspace_root = tmp_path / "fallback-team" + workspace_root.mkdir() + _write_topology(workspace_root, workspace_name=None) + + result = runner.invoke(app, ["workspace", "init-from-topology", str(workspace_root)]) + + assert result.exit_code == 0, result.output + with (workspace_root / ".grip" / "workspace_spec.toml").open("rb") as spec_file: + spec = tomllib.load(spec_file) + assert spec["workspace_name"] == "fallback-team" + + def test_workspace_init_from_topology_refuses_an_incomplete_declared_repo_before_writing( tmp_path: Path, ) -> None: @@ -161,6 +185,8 @@ def test_workspace_init_from_topology_serializes_every_writer_value( workspace_root.mkdir() (workspace_root / "workspace.toml").write_text( r''' +workspace_name = 'declared"\\workspace' + [[repos]] key = 'product"\\name' url = 'https://example.invalid/a"\\b.git' @@ -183,7 +209,7 @@ def test_workspace_init_from_topology_serializes_every_writer_value( assert result.exit_code == 0, result.output with (workspace_root / ".grip" / "workspace_spec.toml").open("rb") as spec_file: spec = tomllib.load(spec_file) - assert spec["workspace_name"] == workspace_root.name + assert spec["workspace_name"] == r'declared"\\workspace' assert spec["repos"] == [ { "name": r'product"\\name', From 0e320e56eaddcb3ec4ac185bc43190734bc81c70 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Fri, 21 Aug 2026 08:29:52 -0500 Subject: [PATCH 28/29] fix(pr): count failed merges and disjoin them into the exit code gr pr merge reported success when every merge in a run had failed. The per-repo diagnostics were printed and truthful; the exit code was not, so a script driving the command was told a batch had merged when none of it had. Three exits returned Ok while carrying failures: - an empty candidate list that was empty because the PR lookups FAILED. "We looked and found nothing" and "we could not look" are different answers, and only the first is an absence of PRs. They are now tracked separately rather than arriving at the summary as one state. - the --auto path, where every attempt to enable auto-merge could fail. - the final exit, after a mixed or wholly failed run. Each now fails with a count of what failed against what was attempted. The test fixtures needed correcting first. mock_get_pr mounts a single invariant response, so a read issued after a successful merge still reported merged: false, which the real API cannot produce. Three groups of tests were affected, by three different causes, and they should not be counted together. FIVE tests began failing for the fixture reason once the exit code became truthful. The new mock_pr_lifecycle couples the GET and the merge PUT through shared state so the sequence behaves as the live API does; all five then passed with NO ASSERTION CHANGED, which is what distinguishes a fixture defect from a contract change. A SIXTH test, repo_filter_excludes_non_target, carried the same fixture defect and never went red at all: its PR was mocked as state "open" and merged: true simultaneously, making the command's own post-merge verification vacuous, so it passed whether or not the merge did anything. It now starts unmerged and GAINS an assertion that the merge actually fired. ONE test, branch_behind_suggests_update, was red for an unrelated reason and is the only assertion this change edits. It required is_ok() while its own failure message said "handled without crashing". Those are different claims. The merge did not happen, so graceful handling means a useful error rather than a success. Its fixture is untouched. Every added guard is mutation-proved: neutering each of the three exits independently turns a named test red. The --auto path had no test of any kind before this change, so its guard arrived with the first one. Ref #884 -- closes at promotion Ref #886 -- closes at promotion --- src/cli/commands/pr/merge.rs | 56 ++++++++ tests/common/mock_platform.rs | 100 ++++++++++++- tests/test_pr_merge.rs | 259 +++++++++++++++++++++++++++++++--- 3 files changed, 398 insertions(+), 17 deletions(-) diff --git a/src/cli/commands/pr/merge.rs b/src/cli/commands/pr/merge.rs index 76f28c9..b6eac3c 100644 --- a/src/cli/commands/pr/merge.rs +++ b/src/cli/commands/pr/merge.rs @@ -333,6 +333,11 @@ pub async fn run_pr_merge( let mut prs_to_merge: Vec = Vec::new(); let mut json_skipped: Vec = Vec::new(); + // A repo whose PR lookup failed belongs in neither of the two collections + // above: it is not a merge candidate and it was not skipped. Without a + // third one, "we looked and found nothing" and "we could not look" arrive + // at the summary as the same state. + let mut lookup_failures: Vec = Vec::new(); for repo in &all_repos { if !path_exists(&repo.absolute_path) { @@ -426,11 +431,24 @@ pub async fn run_pr_merge( if !opts.json { Output::error(&format!("{}: {}", repo.name, e)); } + lookup_failures.push(repo.name.clone()); } } } if prs_to_merge.is_empty() { + // An empty candidate list has two causes that read identically here. + // Only one of them is an absence of PRs; the other is an absence of + // knowledge, and reporting it as the first is a false statement the + // exit code then endorses. + if !lookup_failures.is_empty() { + anyhow::bail!( + "could not determine PR state for {} of {} repositories: {}", + lookup_failures.len(), + all_repos.len(), + lookup_failures.join(", ") + ); + } println!("No open PRs found for any repository."); println!("Repositories checked: {}", all_repos.len()); return Ok(()); @@ -730,6 +748,18 @@ pub async fn run_pr_merge( )); } + // Case 3: any per-repo failure makes the run a failure, including a + // mixed run. The warning above is read by a human; the exit code is + // the only part a script sees, and it reported this as done. + if error_count > 0 || !lookup_failures.is_empty() { + anyhow::bail!( + "{} of {} auto-merge attempts failed{}", + error_count, + success_count + error_count, + describe_lookup_failures(&lookup_failures) + ); + } + return Ok(()); } @@ -1112,9 +1142,35 @@ pub async fn run_pr_merge( } } + // Case 3, on the path that matters most: a run that failed to merge some + // of the PRs it selected has already emitted its JSON document and its + // human summary by this point. Both are truthful. The exit code was not. + if error_count > 0 || !lookup_failures.is_empty() { + anyhow::bail!( + "{} of {} PR merges failed{}", + error_count, + success_count + error_count, + describe_lookup_failures(&lookup_failures) + ); + } + Ok(()) } +/// Render the lookup-failure tail of a summary, or nothing when every repo +/// was successfully inspected. Kept separate so the three case-3 exits phrase +/// the same fact identically. +fn describe_lookup_failures(failures: &[String]) -> String { + if failures.is_empty() { + String::new() + } else { + format!( + "; PR state could not be determined for {}", + failures.join(", ") + ) + } +} + /// Check if a repo has changes ahead of its default branch /// Returns Ok(true) if there are changes, Ok(false) if no changes or on default branch fn check_repo_for_changes(repo: &RepoInfo) -> anyhow::Result { diff --git a/tests/common/mock_platform.rs b/tests/common/mock_platform.rs index 9aa6b4f..5ab8ae0 100644 --- a/tests/common/mock_platform.rs +++ b/tests/common/mock_platform.rs @@ -4,8 +4,10 @@ //! testing of platform adapter methods. use serde_json::{json, Map, Value}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use wiremock::matchers::{header, method, path}; -use wiremock::{Mock, MockServer, ResponseTemplate}; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; /// Start a wiremock server and configure GITHUB_TOKEN env var. /// Returns the server and a GitHubAdapter pointed at it. @@ -376,6 +378,76 @@ pub async fn mock_merge_pr(server: &MockServer, number: u64, merged: bool) { .await; } +/// GitHub API: a PR whose GET response reflects whether its merge PUT has fired. +/// +/// `mock_get_pr` mounts one invariant response, so a GET issued *after* a +/// successful merge PUT still reports `merged: false`. That is a state the real +/// API cannot produce, and any command that verifies its own merge by reading +/// the PR back sees a contradiction that belongs to the fixture rather than to +/// the code under test. This helper couples the two endpoints through shared +/// state so the sequence GET -> PUT -> GET behaves as the live API does. +/// +/// Returns the merged flag so a test can assert the PUT actually fired. That +/// matters: without it, a command that never attempted the merge and a command +/// that merged successfully both leave the fixture reporting `merged: false` +/// for different reasons. +pub async fn mock_pr_lifecycle(server: &MockServer, number: u64) -> Arc { + let merged = Arc::new(AtomicBool::new(false)); + + struct ReadPr { + merged: Arc, + number: u64, + } + + impl Respond for ReadPr { + fn respond(&self, _request: &Request) -> ResponseTemplate { + let is_merged = self.merged.load(Ordering::SeqCst); + ResponseTemplate::new(200).set_body_json(github_pr_json( + self.number, + if is_merged { "closed" } else { "open" }, + "feat/test", + "main", + is_merged, + "PR description\n", + )) + } + } + + struct MergePr { + merged: Arc, + } + + impl Respond for MergePr { + fn respond(&self, _request: &Request) -> ResponseTemplate { + self.merged.store(true, Ordering::SeqCst); + ResponseTemplate::new(200).set_body_json(json!({ + "sha": "merge123", + "merged": true, + "message": "Pull Request successfully merged" + })) + } + } + + Mock::given(method("GET")) + .and(path(format!("/repos/owner/repo/pulls/{}", number))) + .respond_with(ReadPr { + merged: Arc::clone(&merged), + number, + }) + .mount(server) + .await; + + Mock::given(method("PUT")) + .and(path(format!("/repos/owner/repo/pulls/{}/merge", number))) + .respond_with(MergePr { + merged: Arc::clone(&merged), + }) + .mount(server) + .await; + + merged +} + /// GitHub API: merge PR returns 405 with "branch behind" message. pub async fn mock_merge_pr_behind(server: &MockServer, number: u64) { let body = json!({ @@ -666,6 +738,32 @@ pub fn point_repo_at_mock( } /// Mock a GitHub repo info response (GET /repos/:owner/:repo). +/// GitHub API: repo info with explicit merge-method permissions. +/// +/// `mock_repo_info` always reports every method as allowed, so no test could +/// reach the branch where a command refuses a method the repository forbids. +pub async fn mock_repo_info_with_methods( + server: &MockServer, + owner: &str, + repo: &str, + allow_squash: bool, + allow_merge_commit: bool, + allow_rebase: bool, +) { + let mut body = github_repo_json(owner, repo); + if let Value::Object(ref mut m) = body { + m.insert("allow_squash_merge".into(), json!(allow_squash)); + m.insert("allow_merge_commit".into(), json!(allow_merge_commit)); + m.insert("allow_rebase_merge".into(), json!(allow_rebase)); + } + + Mock::given(method("GET")) + .and(path(format!("/repos/{}/{}", owner, repo))) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .mount(server) + .await; +} + pub async fn mock_repo_info(server: &MockServer, owner: &str, repo: &str) { let body = github_repo_json(owner, repo); diff --git a/tests/test_pr_merge.rs b/tests/test_pr_merge.rs index 3cf186e..62426a9 100644 --- a/tests/test_pr_merge.rs +++ b/tests/test_pr_merge.rs @@ -10,7 +10,8 @@ use common::fixtures::WorkspaceBuilder; use common::git_helpers; use common::mock_platform::{ mock_check_runs, mock_get_pr, mock_legacy_combined_status, mock_list_prs, mock_merge_pr, - mock_merge_pr_behind, mock_pr_reviews, point_repo_at_mock, setup_github_mock, + mock_merge_pr_behind, mock_pr_lifecycle, mock_pr_reviews, mock_repo_info_with_methods, + mock_server_error, mock_server_error_put, point_repo_at_mock, setup_github_mock, }; use gitgrip::core::manifest::{PlatformConfig, PlatformType}; use wiremock::http::Method; @@ -235,10 +236,9 @@ async fn test_pr_merge_force_bypasses_checks() { }); mock_list_prs(&server, vec![(42, "feat/test")]).await; - mock_get_pr(&server, 42, "open", false).await; + mock_pr_lifecycle(&server, 42).await; mock_pr_reviews(&server, 42, vec![("COMMENTED", "alice")]).await; mock_check_runs(&server, "feat/test", vec![("CI", "in_progress", None)]).await; - mock_merge_pr(&server, 42, true).await; let result = gitgrip::cli::commands::pr::run_pr_merge( &ws.workspace_root, @@ -331,10 +331,16 @@ async fn test_pr_merge_branch_behind_suggests_update() { ) .await; + // This assertion used to require `is_ok()` while its own message said + // "handled without crashing" -- two different claims. The merge genuinely + // did not happen, so graceful handling means a useful error, not a success. + // Reporting a batch as done when every merge in it failed is the exact + // false-success this change removes. + let error = result.expect_err("a branch-behind merge did not merge, so the run failed"); assert!( - result.is_ok(), - "branch-behind merge should be handled without crashing: {:?}", - result.err() + error.to_string().contains("1 of 1"), + "the error should name how many merges failed, got: {}", + error ); let requests = server.received_requests().await.unwrap(); @@ -380,7 +386,12 @@ async fn test_pr_merge_repo_filter_excludes_non_target() { // Only mock PR for frontend (PR #10). Backend should never be queried. mock_list_prs(&server, vec![(10, "feat/shared")]).await; - mock_get_pr(&server, 10, "open", true).await; + // The fixture here used to be `mock_get_pr(&server, 10, "open", true)` -- a PR + // reported as state "open" AND already merged, which the real API cannot + // produce. It made the command's own post-merge verification vacuous: the + // read-back said "merged" whether or not the merge had done anything, so + // this test was green for a reason unrelated to what it claims to check. + let merged_flag = mock_pr_lifecycle(&server, 10).await; mock_pr_reviews(&server, 10, vec![("APPROVED", "alice")]).await; mock_check_runs( &server, @@ -388,7 +399,6 @@ async fn test_pr_merge_repo_filter_excludes_non_target() { vec![("CI", "completed", Some("success"))], ) .await; - mock_merge_pr(&server, 10, true).await; // Filter to frontend only, force to bypass readiness checks let result = gitgrip::cli::commands::pr::run_pr_merge( @@ -428,6 +438,14 @@ async fn test_pr_merge_repo_filter_excludes_non_target() { 1, "exactly one merge request should be sent (frontend only, not backend)" ); + + // The request count proves the filter. This proves the merge actually + // happened -- the claim the old fixture asserted by construction rather + // than by observing anything. + assert!( + merged_flag.load(std::sync::atomic::Ordering::SeqCst), + "the filtered repo's PR should have been merged, not merely attempted" + ); } // ── Repo Filter: No Matching Repos ──────────────────────────── @@ -498,10 +516,9 @@ async fn test_pr_merge_force_yes_merges_without_prompt() { }); mock_list_prs(&server, vec![(42, "feat/test")]).await; - mock_get_pr(&server, 42, "open", false).await; + mock_pr_lifecycle(&server, 42).await; mock_pr_reviews(&server, 42, vec![]).await; mock_check_runs(&server, "feat/test", vec![("CI", "in_progress", None)]).await; - mock_merge_pr(&server, 42, true).await; // --force --yes should merge without stdin prompt let result = gitgrip::cli::commands::pr::run_pr_merge( @@ -632,7 +649,7 @@ async fn test_pr_merge_all_flag_proceeds_and_merges_every_match() { } mock_list_prs(&server, vec![(1, "feat/shared-name")]).await; - mock_get_pr(&server, 1, "open", false).await; + mock_pr_lifecycle(&server, 1).await; mock_pr_reviews(&server, 1, vec![("APPROVED", "alice")]).await; mock_check_runs( &server, @@ -640,7 +657,6 @@ async fn test_pr_merge_all_flag_proceeds_and_merges_every_match() { vec![("CI", "completed", Some("success"))], ) .await; - mock_merge_pr(&server, 1, true).await; let result = gitgrip::cli::commands::pr::run_pr_merge( &ws.workspace_root, @@ -712,13 +728,12 @@ async fn test_pr_merge_wait_does_not_block_when_no_checks_are_configured() { }); mock_list_prs(&server, vec![(42, "feat/no-ci")]).await; - mock_get_pr(&server, 42, "open", false).await; + mock_pr_lifecycle(&server, 42).await; mock_pr_reviews(&server, 42, vec![("APPROVED", "alice")]).await; // Exact GitHub shape for a ref with no CI configured: check-runs reports // zero runs, and the legacy fallback reports "pending" with zero statuses. mock_check_runs(&server, "feat/no-ci", vec![]).await; mock_legacy_combined_status(&server, "feat/no-ci", "pending", vec![]).await; - mock_merge_pr(&server, 42, true).await; let start = std::time::Instant::now(); let result = gitgrip::cli::commands::pr::run_pr_merge( @@ -942,7 +957,7 @@ async fn test_skip_gate_approval_allows_a_comment_ratified_merge() { // PR that is open and mergeable. Passing `true` here made it UNmergeable and // the merge was correctly blocked by the `mergeable` gate, which looked like // the waiver failing. The scenario was wrong, not the waiver. - mock_get_pr(&server, 42, "open", false).await; + mock_pr_lifecycle(&server, 42).await; mock_pr_reviews(&server, 42, vec![("COMMENTED", "alice")]).await; mock_check_runs( &server, @@ -950,7 +965,6 @@ async fn test_skip_gate_approval_allows_a_comment_ratified_merge() { vec![("CI", "completed", Some("success"))], ) .await; - mock_merge_pr(&server, 42, true).await; let result = gitgrip::cli::commands::pr::run_pr_merge( &ws.workspace_root, @@ -982,3 +996,216 @@ async fn test_skip_gate_approval_allows_a_comment_ratified_merge() { "approval was the only failing gate and it was waived by name — the merge must proceed" ); } + +#[tokio::test] +async fn test_pr_merge_all_lookups_failing_is_not_success() { + let (server, _adapter) = setup_github_mock().await; + mock_server_error(&server, "/repos/owner/repo/pulls").await; + + let ws = WorkspaceBuilder::new() + .add_repo("frontend") + .add_repo("backend") + .build(); + + let mut manifest = ws.load_manifest(); + point_repo_at_mock(&mut manifest, "frontend", &server); + point_repo_at_mock(&mut manifest, "backend", &server); + + // Both off the default branch, or they are skipped before any lookup runs + // and the witness would pass for the wrong reason. + for name in ["frontend", "backend"] { + git_helpers::create_branch(&ws.repo_path(name), "feat/witness"); + git_helpers::commit_file(&ws.repo_path(name), "w.txt", "w", "witness commit"); + } + + let result = gitgrip::cli::commands::pr::run_pr_merge( + &ws.workspace_root, + &manifest, + &gitgrip::cli::commands::pr::MergeOptions { + method: None, + force: false, + skip_gates: Vec::new(), + update: false, + auto: false, + json: false, + wait: false, + timeout: 600, + delete_branch: true, + repo_filter: None, + yes: true, + allow_all: false, + }, + ) + .await; + + assert!( + result.is_err(), + "every PR lookup failed; the command must not report success" + ); +} + +#[tokio::test] +async fn test_pr_merge_all_lookups_failing_exits_nonzero() { + let (server, _adapter) = setup_github_mock().await; + + let ws = WorkspaceBuilder::new().add_repo("app").build(); + let mut manifest = ws.load_manifest(); + + git_helpers::create_branch(&ws.repo_path("app"), "feat/test"); + git_helpers::commit_file( + &ws.repo_path("app"), + "feature.txt", + "feature", + "Add feature", + ); + + point_repo_at_mock(&mut manifest, "app", &server); + let manifest_yaml = serde_yaml::to_string(&manifest).unwrap(); + std::fs::write( + ws.workspace_root.join(".gitgrip/spaces/main/gripspace.yml"), + manifest_yaml, + ) + .unwrap(); + + // Every PR lookup fails, so the candidate list is empty for a reason that + // is not "there are no PRs". + mock_server_error(&server, "/repos/owner/repo/pulls").await; + + let output = tokio::process::Command::new(assert_cmd::cargo::cargo_bin!("gr")) + .current_dir(&ws.workspace_root) + .env("GITHUB_TOKEN", "test") + .args(["pr", "merge", "--method", "merge", "--yes"]) + .output() + .await + .unwrap(); + + assert_ne!( + output.status.code(), + Some(0), + "a run that could not determine PR state must not exit 0; stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test] +async fn test_pr_merge_failed_merge_exits_nonzero() { + let (server, _adapter) = setup_github_mock().await; + + let ws = WorkspaceBuilder::new().add_repo("app").build(); + let mut manifest = ws.load_manifest(); + + git_helpers::create_branch(&ws.repo_path("app"), "feat/test"); + git_helpers::commit_file( + &ws.repo_path("app"), + "feature.txt", + "feature", + "Add feature", + ); + + point_repo_at_mock(&mut manifest, "app", &server); + let manifest_yaml = serde_yaml::to_string(&manifest).unwrap(); + std::fs::write( + ws.workspace_root.join(".gitgrip/spaces/main/gripspace.yml"), + manifest_yaml, + ) + .unwrap(); + + mock_list_prs(&server, vec![(42, "feat/test")]).await; + mock_get_pr(&server, 42, "open", false).await; + mock_pr_reviews(&server, 42, vec![("APPROVED", "alice")]).await; + mock_check_runs( + &server, + "feat/test", + vec![("CI", "completed", Some("success"))], + ) + .await; + // The lookup succeeds and the merge itself fails: a genuine per-repo error + // on the path that matters most. + mock_server_error_put(&server, "/repos/owner/repo/pulls/42/merge").await; + + let output = tokio::process::Command::new(assert_cmd::cargo::cargo_bin!("gr")) + .current_dir(&ws.workspace_root) + .env("GITHUB_TOKEN", "test") + .args(["pr", "merge", "--force", "--method", "merge", "--yes"]) + .output() + .await + .unwrap(); + + assert_ne!( + output.status.code(), + Some(0), + "a failed merge must not exit 0; stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +/// The `--auto` path had no test of any kind, so its exit code was unverified +/// in both directions. A PR whose requested merge method the repository forbids +/// is counted as an error inside the auto loop; before this change the run +/// still returned Ok, so a script enabling auto-merge across a workspace could +/// be told every PR was queued when none of them were. +#[tokio::test] +async fn test_pr_merge_auto_enable_failure_exits_nonzero() { + let (server, _adapter) = setup_github_mock().await; + + let ws = WorkspaceBuilder::new().add_repo("app").build(); + let mut manifest = ws.load_manifest(); + + git_helpers::create_branch(&ws.repo_path("app"), "feat/test"); + git_helpers::commit_file( + &ws.repo_path("app"), + "feature.txt", + "feature", + "Add feature", + ); + + let repo_config = manifest.repos.get_mut("app").unwrap(); + repo_config.url = Some("https://github.com/owner/repo.git".to_string()); + repo_config.platform = Some(PlatformConfig { + platform_type: PlatformType::GitHub, + base_url: Some(server.uri()), + }); + + mock_list_prs(&server, vec![(42, "feat/test")]).await; + mock_pr_lifecycle(&server, 42).await; + mock_pr_reviews(&server, 42, vec![("APPROVED", "alice")]).await; + mock_check_runs( + &server, + "feat/test", + vec![("CI", "completed", Some("success"))], + ) + .await; + // Readiness passes, so the run reaches the auto loop. The repository then + // forbids the requested method, which is the loop's own error branch. + mock_repo_info_with_methods(&server, "owner", "repo", true, false, true).await; + + let merge_method = gitgrip::platform::MergeMethod::Merge; + let result = gitgrip::cli::commands::pr::run_pr_merge( + &ws.workspace_root, + &manifest, + &gitgrip::cli::commands::pr::MergeOptions { + method: Some(&merge_method), + force: false, + skip_gates: Vec::new(), + update: false, + auto: true, + json: false, + wait: false, + timeout: 600, + delete_branch: true, + repo_filter: None, + yes: true, + allow_all: false, + }, + ) + .await; + + let error = result.expect_err("no auto-merge was enabled, so the run failed"); + assert!( + error.to_string().contains("auto-merge attempts failed"), + "the error should say the auto-merge attempts failed, got: {}", + error + ); +} From 2eca9045b09fd0d90c4e7ca391dc904d5e42a9f2 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Fri, 21 Aug 2026 19:26:15 -0500 Subject: [PATCH 29/29] chore: release gr 1.2.0 Bumps the crate to 1.2.0, adds the 1.2.0 changelog entry, corrects a false [Unreleased] heading, and fixes formatter drift in one test file. SCOPE. The work promoted is v1.1.0..6192e8c5 -- the 1.1.0 tag to the dev tip at freeze -- which is 42 commits / 14 first-parent units. This release-prep commit sits one beyond that; counting it makes the range 43 / 15. Units are classified by diffing each against its FIRST PARENT. A merge commit has no canonical diff and git show --name-only returns nothing for these, so the method is part of the claim. FIVE of those 14 units are in the published crate. Nine are gr2, a separate Python surface at 0.1.0 that is not distributed. An entry claiming fourteen units of work would describe a release nobody can install. Two claims are deliberately narrow. The exit-code work fixes two instances of a class with 9+ known members, and the most consequential member is untouched. The freshness work ships with two measured holes open and no user-facing documentation. Both are stated as limits in the entry. A commit-level statistic was removed twice. First as unsourced -- "roughly a third of this repository's active commit volume," no range, no metric. Then as mis-sourced: "22 of 42 commits, measured by first-parent diff" cited a method that did not produce it, since 22 comes from a path-limited rev-list whose history simplification drops eight merges that touch gr2 against their first parent. Under the named method the figure is 30 of 42. A number citing the wrong method is worse than an unsourced one, because the citation invites trust it has not earned. The unit measure, 9 of 14, needs no footnote and is what remains. The [Unreleased] heading was false: that work shipped in v1.1.0 on 2026-08-13 and is on crates.io. Verified by checking the feature's own symbols into the v1.1.0 tree with a negative control at v1.0.2, not by commit archaeology. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 64 +++++++++++++++++++++++++++++++- Cargo.lock | 2 +- Cargo.toml | 2 +- tests/manifest_input_warnings.rs | 5 +-- 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dce507..eca04ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,69 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.2.0] - 2026-08-21 + +**Scope, stated first because the range and the artifact are not the same thing.** + +The work promoted in this release is the range **`v1.1.0..6192e8c5`** — from the 1.1.0 tag to the +`dev` tip at freeze — which is **42 commits / 14 first-parent units**. The release-prep commit +that carries this entry sits one beyond that and is deliberately excluded; counting it makes the +range 43 / 15. Both numbers are true of different ranges, so the range is named rather than left +to inference. + +Units are classified by diffing each against its **first parent**. A merge commit has no +canonical diff, and `git show --name-only` returns nothing for these, so the method is +part of the claim. + +**Five of those 14 units are in the published `gitgrip` crate; nine are not.** gr2 is a separate +Python surface at version `0.1.0` that is not distributed, so its nine units — the propagation +prototypes, the append/torn-line work, the native daily verbs, and workspace-spec-from-topology — +are present in this repository and absent from anything `cargo install gitgrip` gives you. The +entries below cover the five gr1 units only. Read the git range if you want the gr2 work. + +### Fixed +- **`gr pr merge` and `gr checkout` exit nonzero when part of a batch fails** — a multi-repo + operation that failed in some repos and succeeded in others previously exited 0, so a caller + or CI step reading the exit code saw success over a partial failure. + + **This does not close the exit-code-honesty class, and should not be read as doing so.** + The class is tracked in `grip#886`, which records 9+ known instances; two are fixed here. + **`gr push` — the most consequential instance, the one that can silently lose work — is + untouched by this release.** +- **`gr link --apply` reports a stale source instead of composing it silently** and + **gripspace pins are checked for freshness**. + + **Two measured holes in this remain open** (`grip#891`), and this feature ships with **no + user-facing documentation** — `README.md` has no `gr link` section, only a one-line table + mention. Link freshness is **not** guaranteed by this release; what changed is that one class + of stale composition now reports rather than proceeding quietly. + +### Known gaps at this release +- The new exit-code semantics (`EXIT_REFUSED=2`, operational failure `1`, success `0`) are not + documented in `README.md` for either `gr checkout` or `gr pr merge`. +- `CONTRIBUTING.md` does not mention gr2 anywhere, despite **9 of the 14 first-parent units in + `v1.1.0..6192e8c5` touching only `gr2/`** — the same first-parent classification used + throughout this entry. A contributor following that document has no path to discovering gr2 + exists or how to set up its separate Python environment. + + **Two earlier drafts of this line carried a commit-level statistic; both are withdrawn, and + the second is the more instructive.** The first said gr2 was "roughly a third of this + repository's active commit volume" — no range, no metric, nothing to reproduce. The second + replaced it with "22 of 42 commits (52%), measured by first-parent diff," which named a range + and a method and was **still wrong, because the number did not come from the method it + named**: 22 is `git rev-list --count -- gr2/`, whose path-history simplification silently drops + eight merge commits that *do* touch `gr2/` against their first parent. Under the stated method + the figure is 30 of 42 (71%). A sourced number can be more misleading than an unsourced one, + because citing a method invites trust the number has not earned. The unit measure above needs + no footnote, so it is the only one kept. + +## [1.1.0] - 2026-08-13 + +These entries sat under an `[Unreleased]` heading until 2026-08-21. **That heading was false:** +this work shipped in `v1.1.0`, tagged 2026-08-13 and published to crates.io. Verified by +checking the feature's own symbols into the `v1.1.0` tree rather than by commit archaeology, +with a negative control. A reader trusting the old heading would have believed a shipped +feature was still pending. ### Fixed - **`gr pr merge` no longer defaults to squash** (#829) — with no `--method`, the command queried the host for its allowed methods and took the first of squash > merge > rebase, so on any repo permitting squash the tool actively chose it. A workspace whose policy is merge-commit-only got squashes from its own tooling, and on a private repo — where hosting rulesets are unavailable on most plans — nothing downstream could reject the result. The default is now a real merge commit, configurable via `settings.merge_method` in the manifest. Choosing is the workspace's job; the host is asked only whether the choice is permitted. diff --git a/Cargo.lock b/Cargo.lock index cbbbd67..ad0c16a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -814,7 +814,7 @@ dependencies = [ [[package]] name = "gitgrip" -version = "1.1.0" +version = "1.2.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 8dedf7b..ff97c9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gitgrip" -version = "1.1.0" +version = "1.2.0" edition = "2021" rust-version = "1.80" description = "Multi-repo workflow tool - manage multiple git repositories as one" diff --git a/tests/manifest_input_warnings.rs b/tests/manifest_input_warnings.rs index 1a9fb56..96e3140 100644 --- a/tests/manifest_input_warnings.rs +++ b/tests/manifest_input_warnings.rs @@ -281,7 +281,6 @@ fn json_stdout_should_be_parseable_json_and_currently_is_not() { let (_ws, out) = init_then_sync(&url, &["--json"]); let stdout = String::from_utf8_lossy(&out.stdout); - serde_json::from_str::(stdout.trim()).unwrap_or_else(|e| { - panic!("--json stdout is not parseable JSON ({e}).\nstdout: {stdout}") - }); + serde_json::from_str::(stdout.trim()) + .unwrap_or_else(|e| panic!("--json stdout is not parseable JSON ({e}).\nstdout: {stdout}")); }