diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index a550decf..90872c9e 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,7 +1,7 @@ { "$schema": "https://anthropic.com/claude-code/marketplace.schema.json", "name": "autoform", - "description": "Lean repository setup, Markdown-vault roadmaps, proving, and human or agent review with Lean LSP and REPL tools.", + "description": "Lean repository setup, Markdown-vault roadmaps, publication, and human or agent review with Lean LSP and REPL tools.", "owner": { "name": "Vivien Cabannes", "url": "https://github.com/facebookresearch/autoform-bot" @@ -9,7 +9,7 @@ "plugins": [ { "name": "autoform", - "description": "Set up, plan, publish, prove, and review Lean formalizations as linked Markdown nodes checked through Lean LSP and REPL.", + "description": "Set up, plan, publish, and review Lean formalizations as linked Markdown nodes checked through Lean LSP and REPL.", "source": "./", "category": "productivity" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 9178009a..ae255ec8 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "autoform", - "description": "Set up Lean repositories, build source-grounded Markdown roadmaps, and support human or agent review with Lean LSP and REPL tools.", + "description": "Set up Lean repositories, build source-grounded Markdown roadmaps, orchestrate ready work, and support human or agent review with Lean LSP and REPL tools.", "version": "0.5.0", "author": { "name": "Vivien Cabannes", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 465bbf00..38ed5051 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "autoform", "version": "0.5.0+codex.20260812000640", - "description": "Set up Lean repositories, build source-grounded Markdown roadmaps, and support human or agent review.", + "description": "Set up Lean repositories, build source-grounded Markdown roadmaps, orchestrate ready work, and support human or agent review.", "author": { "name": "Vivien Cabannes" }, @@ -21,6 +21,7 @@ "defaultPrompt": [ "Set up this Lean repository with an Autoform vault, verification CI, and GitHub Pages without planning the mathematics.", "Build or refine an Autoform roadmap from my mathematical sources.", + "Work through ready nodes using the Autoform CLI and durable claims.", "Prepare the visual blueprint surfaces so I can review this formalization.", "Judge this roadmap or Lean formalization with evidence-based review rubrics.", "Develop Autoform itself through its executable formalization example." diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0a2fbdaf..190c3560 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,15 +2,72 @@ name: tests on: push: + branches: [main] pull_request: permissions: contents: read +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + jobs: - test: + deterministic: + name: deterministic (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.13"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.12.1" + python-version: ${{ matrix.python-version }} + enable-cache: true + - name: Install locked development environment + run: timeout --signal=TERM --kill-after=30s 3m uv sync --frozen --extra dev --extra repl + - name: Run deterministic tests + run: timeout --signal=TERM --kill-after=30s 8m make test-deterministic + - name: Lint + if: matrix.python-version == '3.13' + run: timeout --signal=TERM --kill-after=30s 2m make lint + - name: Validate and build the example + if: matrix.python-version == '3.13' + run: timeout --signal=TERM --kill-after=30s 4m make check-example + + daemon: + name: daemon (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 12 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.13"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.12.1" + python-version: ${{ matrix.python-version }} + enable-cache: true + - name: Install locked development environment + run: timeout --signal=TERM --kill-after=30s 3m uv sync --frozen --extra dev --extra repl + - name: Run detached runtime tests + env: + AUTOFORM_REPL_TOTAL_WORKERS: "1" + AUTOFORM_MAX_LEAN_PROJECTS: "1" + run: timeout --signal=TERM --kill-after=30s 8m make test-daemon + + installed-wheel: + name: installed wheel (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest + timeout-minutes: 15 strategy: + fail-fast: false matrix: python-version: ["3.10", "3.13"] steps: @@ -20,7 +77,60 @@ jobs: version: "0.12.1" python-version: ${{ matrix.python-version }} enable-cache: true - - run: uv sync --extra dev --extra repl - - run: uv run ruff check autoform_cli servers tests - - run: uv run pytest -q - - run: make check-example + - name: Install locked development environment + run: timeout --signal=TERM --kill-after=30s 3m uv sync --frozen --extra dev --extra repl + - name: Build, install, and probe the wheel + run: timeout --signal=TERM --kill-after=30s 10m make test-wheel + + real-lean: + name: real Lean (pinned v4.32.2) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: "0.12.1" + python-version: "3.13" + enable-cache: true + - name: Install locked development environment + run: timeout --signal=TERM --kill-after=30s 3m uv sync --frozen --extra dev --extra repl + - name: Verify the repository Lean and Mathlib pins + run: | + set -euo pipefail + test "$(tr -d '\r\n' < skills/setup/assets/cabannes-thesis-project/lean-toolchain)" = "leanprover/lean4:v4.32.2" + grep -Fq 'rev = "v4.32.2"' skills/setup/assets/cabannes-thesis-project/lakefile.toml + uv run python - <<'PY' + import json + from pathlib import Path + + catalog = json.loads(Path("autoform_cli/project/releases.json").read_text(encoding="utf-8")) + pinned = [ + release + for release in catalog["releases"] + if release["lean"]["toolchain"] == "leanprover/lean4:v4.32.2" + and release["mathlib"]["revision"] == "v4.32.2" + ] + assert len(pinned) == 1 + PY + - name: Install pinned Elan + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=15s 2m curl -sSfL \ + https://github.com/leanprover/elan/releases/download/v4.2.3/elan-x86_64-unknown-linux-gnu.tar.gz \ + -o elan.tar.gz + echo "df0b2b3a439961ffcbb3985214365ffe40f49bc871df04dff268c7d8e21ca8b2 elan.tar.gz" \ + | sha256sum --check --strict + tar xzf elan.tar.gz + ./elan-init -y --default-toolchain none + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + - name: Install and verify pinned Lean + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=30s 10m elan toolchain install leanprover/lean4:v4.32.2 + version="$(timeout --signal=TERM --kill-after=10s 30s elan run leanprover/lean4:v4.32.2 lean --version)" + printf '%s\n' "$version" + grep -Fq "Lean (version 4.32.2" <<<"$version" + command -v lake + - name: Run mandatory real-Lean tests + run: timeout --signal=TERM --kill-after=30s 25m make test-real-lean diff --git a/.muse-plugin/plugin.json b/.muse-plugin/plugin.json index 8d7577f6..2f01b103 100644 --- a/.muse-plugin/plugin.json +++ b/.muse-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "autoform", "displayName": "AutoForm Bot", "version": "0.5.0", - "description": "Set up Lean repositories, build Markdown roadmaps, and support human or agent review.", + "description": "Set up Lean repositories, build Markdown roadmaps, orchestrate ready work, and support human or agent review.", "compat": { "source": "native", "manifestDir": ".muse-plugin" @@ -21,6 +21,11 @@ "path": "skills/roadmap/SKILL.md", "enabledDefault": true }, + { + "id": "orchestrate", + "path": "skills/orchestrate/SKILL.md", + "enabledDefault": true + }, { "id": "human-review", "path": "skills/human-review/SKILL.md", diff --git a/Makefile b/Makefile index 9e781072..72094a35 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,25 @@ -.PHONY: setup test lint check-example +.PHONY: setup test test-deterministic test-daemon test-wheel test-real-lean lint check-example THESIS_EXAMPLE := skills/setup/assets/cabannes-thesis-project +PYTEST := uv run pytest -q setup: uv sync --extra dev --extra repl test: - uv run pytest -q + $(PYTEST) + +test-deterministic: + $(PYTEST) -m "not daemon and not installed_wheel and not real_lean" + +test-daemon: + $(PYTEST) -m daemon + +test-wheel: + $(PYTEST) -m installed_wheel + +test-real-lean: + $(PYTEST) -m real_lean lint: uv run ruff check autoform_cli servers tests diff --git a/README.md b/README.md index 3f24013c..8de93bed 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,25 @@ # AutoformBot -AutoformBot is a coding-agent plugin and Python CLI for Lean 4 formalization -projects. It builds source-grounded Markdown roadmaps, validates dependencies, -publishes progress views, and prepares human or agent review. The plugin and -CLI use the identifier `autoform`; the canonical repository is -[`facebookresearch/autoform-bot`](https://github.com/facebookresearch/autoform-bot). +AutoformBot is a Claude Code and Codex plugin for turning mathematical sources +into a Lean 4 formalization and a readable companion site. It provides: -The default `main` branch provides repository setup, roadmap planning, -publication, human and agent review, and shared Lean LSP/REPL tools. It does -**not** include autonomous orchestration. +- repository setup for Lean, Mathlib, CI, and GitHub Pages; +- source-grounded Markdown roadmaps with explicit theorem dependencies; +- exhaustive source-unit coverage checks; +- shared Lean LSP and REPL tools; +- human and independent agent review workflows; and +- CLI-backed work discovery and durable claims for concurrent contributors. -Autonomous execution is an opt-in overlay on the -[`execution`](https://github.com/facebookresearch/autoform-bot/tree/execution) -branch. It adds orchestration, claim-backed workers, specialist agents, and -prover adapters on top of `main`. Use `main` unless you are explicitly -evaluating that execution stack. +The plugin and Python commands use the name `autoform`. The canonical repository +is [`facebookresearch/autoform-bot`](https://github.com/facebookresearch/autoform-bot). -## Prerequisites +## Requirements -- Python 3.10 or newer and [`uv`](https://docs.astral.sh/uv/) +- Python 3.10 or newer +- [`uv`](https://docs.astral.sh/uv/) - Git -- Lean and Lake for Lean tooling and verification -- Claude Code or Codex for the installation flows below +- Lean and Lake for proof checking +- Claude Code or Codex ## Install @@ -36,109 +34,143 @@ Codex: ```bash codex plugin marketplace add facebookresearch/autoform-bot --ref main -codex plugin add autoform@autoform +codex +/plugins ``` -Start a new agent session so the skills and MCP servers reload. A native Muse -manifest is included, but Muse installation is not covered here. +Select Autoform in the plugin browser and install it. Start a new agent session +after installation so the skills and MCP servers are loaded. A Muse manifest is +bundled, but Muse installation is separate. -## Quick start +## Use the plugin -Work from an existing Lean repository. First scaffold the blueprint and site -configuration from an Autoform checkout: +Invoke the skill that matches the current stage: + +| Task | Claude Code | Codex | +| --- | --- | --- | +| Set up or repair a repository | `/autoform:setup` | `$autoform:setup` | +| Build or refine the roadmap | `/autoform:roadmap` | `$autoform:roadmap` | +| Inspect the rendered plan yourself | `/autoform:human-review` | `$autoform:human-review` | +| Run an independent audit | `/autoform:agent-review` | `$autoform:agent-review` | +| Work through ready nodes | `/autoform:orchestrate` | `$autoform:orchestrate` | + +A typical request is: + +> Use Autoform to set up this Lean repository, build a roadmap for every result +> in `sources/book.pdf`, have an independent agent audit it, then formalize the +> ready nodes and maintain the readable companion. + +Autoform keeps authored state in an Obsidian-compatible blueprint vault. Each +roadmap article records its sources, statement dependencies, proof dependencies, +and verified Lean declarations. Status, graphs, and publication pages are +derived from those files. + +The skills invoke the bundled Python tools for you. Plugin installation does +not add their console scripts to the shell `PATH`. For manual use, first obtain +the absolute root of the loaded Autoform plugin, then run every command through +that project: ```bash -uv run autoform init /path/to/lean-project \ - --autoform-ref +export AUTOFORM_PLUGIN_ROOT="" +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform --help ``` -This creates `blueprint/`, `mkdocs.yml`, and `requirements-docs.txt`. GitHub -workflows are created only when Autoform has an immutable commit pin. The Setup -skill can inspect and repair this infrastructure, but its new-project Lean -bootstrap helper is not packaged on `main`; start from an existing Lean project -and use `autoform init` for the blueprint and publication files. +## Create a project -Next use the host skills from the Lean project: +For a new Lean repository, let the Setup skill select the bundled compatible +Lean and Mathlib release. The underlying commands are: -| Goal | Claude Code | Codex | -| --- | --- | --- | -| Build a source-grounded roadmap | `/autoform:roadmap` | `$roadmap` | -| Prepare a person-led review | `/autoform:human-review` | `$human-review` | -| Run an independent agent review | `/autoform:agent-review` | `$agent-review` | - -For example: “Build a roadmap for Sections 2–4 of `paper.pdf`; confirm the scope -and completion criteria before writing articles.” Keep the source in the -repository or provide an accessible path. Human and agent review are -alternatives; review the roadmap before treating it as an execution plan. - -## Blueprint model - -```text -blueprint/ -├── README.md -├── coverage/README.md -├── roadmap/ -│ ├── README.md -│ └── convexity/ -│ ├── README.md -│ ├── convex.md -│ └── separating-hyperplane.md -└── sources/paper.md +```bash +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform project provenance --json +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform project versions --json +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform project new MyProject --package MyProject \ + --release \ + --autoform-source https://github.com/facebookresearch/autoform-bot.git \ + --autoform-ref ``` -Every Markdown file below `blueprint/roadmap/` is an article. A nested -`README.md` represents its directory and contains the articles below it. -Optional `declaration: theorem`, `declaration: def`, and similar frontmatter -marks a formalizable article. Inline relative links under `## Depends on` and -`## Proof depends on` define dependency edges; reference-style links do not. +For an existing repository that needs several independent formalization +projects, create a workspace and register each blueprint: + +```bash +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform workspace init . \ + --blueprint-root docs/blueprints +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform blueprint new textbook \ + --path Textbook --title "Textbook" +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform workspace inspect . +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform workspace check . --lean-root . +``` -Markdown is the source of truth; Mermaid graphs and MkDocs pages are derived -views. See the [blueprint format and CLI reference](autoform_cli/README.md) for -complete frontmatter, hierarchy, status, and validation rules. +The root `.autoform.toml` is the only workspace registry. Registered blueprint +paths must not overlap. The original single-project layout remains available: -## CLI and publication +```bash +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform init . \ + --autoform-source https://github.com/facebookresearch/autoform-bot.git \ + --autoform-ref +``` -| Command | Purpose | -| --- | --- | -| `autoform init` | Scaffold the blueprint and site; add CI when immutably pinned. | -| `autoform check` | Validate Markdown structure and dependencies. | -| `autoform audit` | Audit completeness and checked facts. | -| `autoform doctor` | Diagnose the local blueprint contract. | -| `autoform claim` | Coordinate temporary ownership through Git refs. | -| `autoform render` | Generate publishable MkDocs source. | -| `autoform-visualize` | Generate the Mermaid dependency graph. | +## Validate and publish -Inside an Autoform checkout, use `uv run`: +Run these commands from a Lean project that uses the original single-project +layout: ```bash -uv run autoform check /path/to/project/blueprint --lean-root /path/to/project -uv run autoform-visualize /path/to/project/blueprint -uv run autoform render /path/to/project/blueprint \ - --output /path/to/project/site-src \ - --lean-root /path/to/project --require-declarations +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform doctor . --lean-root . +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform check blueprint --lean-root . +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform audit blueprint --lean-root . +lake build +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform render blueprint --output site-src \ + --lean-root . --require-declarations ``` -From a consumer project, resolve the installed plugin root and prefix commands -with `uv run --project ""`, or separately install the -Python package so its console scripts are on `PATH`. +In a registered multi-project workspace, use `.` instead of `blueprint` and add +`--project ` to each `autoform` command. -`check --lean-root` lexically resolves names in local Lean files; it does not -compile them or prove that they belong to a Lake target. Use `lake build` and -the verification workflow for compilation and audit, while treating the -blueprint-to-declaration match as a separate contract. +`check` validates the Markdown dependency graph. `audit` checks roadmap and +source coverage. `lake build` checks Lean. `render` produces MkDocs source for +the human-readable companion; the generated Pages workflow can publish it once +GitHub Pages is enabled. -`render` writes MkDocs source, not a deployed site. The generated Pages workflow -deploys from `main` only after GitHub Pages is enabled in repository settings. +For the complete blueprint format and command flags, see the +[CLI reference](autoform_cli/README.md). -## Documentation +## Formalize ready work -- [Cabannes thesis example](skills/setup/assets/cabannes-thesis-project/README.md) -- [Roadmap example](skills/roadmap/references/cabannes-thesis-roadmap.md) -- [Lean server architecture and operations](servers/README.md) +The Orchestrate skill uses the public CLI rather than a separate worker +runtime. First list the statement or proof phases whose roadmap prerequisites +are satisfied. This command also requires a complete source-unit coverage +contract: -## Development +```bash +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform ready . --lean-root . --json +``` -Development also requires Make: +Before editing an item, acquire its returned `article_id`. Each concurrent +contributor uses a separate Git worktree and a fail-closed Git-ref claim: + +```bash +export AUTOFORM_WORKER_ID="worker-name" +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform claim acquire +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform claim renew +uv run --project "$AUTOFORM_PLUGIN_ROOT" autoform claim release +``` + +Use the `claim list` subcommand to inspect ownership. Contributors also claim shared +resources such as `lake-build` before using a shared build cache. The current +host agent performs the Lean work with the bundled LSP and REPL, runs the +focused Lake build, obtains an independent source-faithfulness and proof review, +records the exact verified metadata, and then runs `autoform check` and +`autoform audit` against that final state. Release the +article claim only after the verified commit reaches the authorized shared +branch. If an attempt is abandoned without a candidate, release it. For an +integration failure or handoff, report the branch, commit, and claim state +instead of making unfinished work appear available. Then call `autoform ready` +again from the updated shared base. +In a registered workspace, pass the same `--project ` to the ready, claim, +check, and audit commands. + +## Development ```bash git clone https://github.com/facebookresearch/autoform-bot.git @@ -149,9 +181,6 @@ make test make check-example ``` -Claude Code uses `/autoform:develop-plugin`; Codex uses `$develop-plugin`. -`make check-example` validates, renders, and builds the example documentation. -Run `lake build` in the Cabannes fixture when changing its Lean sources or -declarations. - -AutoformBot is released under the [MIT License](LICENSE). +Plugin maintainers can use `/autoform:develop-plugin` in Claude Code or +`$autoform:develop-plugin` in Codex. AutoformBot is released under the +[MIT License](LICENSE). diff --git a/autoform_cli/README.md b/autoform_cli/README.md index 0fac0beb..caa79e7a 100644 --- a/autoform_cli/README.md +++ b/autoform_cli/README.md @@ -1,16 +1,18 @@ # Blueprint format and CLI The Autoform CLI validates, visualizes, and publishes the multilevel dependency -graph embedded in `blueprint/roadmap/`. The Markdown book is the graph: no -separate authored or generated graph file exists. +graph embedded in a blueprint vault's `roadmap/` directory. A root +`.autoform.toml` can register multiple vaults in one repository; the original +single-vault layout remains available at `blueprint/`. The Markdown book is the +graph: no separate authored or generated graph file exists. ## Articles and containment -Every Markdown file below `blueprint/roadmap/` is an article node. A +Every Markdown file below a selected vault's `roadmap/` is an article node. A `README.md` represents its directory and strictly contains the articles below it; the nearest ancestor `README.md` is the single parent. This supports any number of levels, from book to chapter to section to declaration. Ordinary -files use their path without `.md` as a stable ID; `README.md` uses its +files use their path without `.md` as the current graph ID; `README.md` uses its directory path, with the root article named `roadmap`. The H1 is the article's human title. Container @@ -60,9 +62,12 @@ relative to the current article and must point at another roadmap article. The optional `declaration` field marks a formalizable leaf and describes its intended Lean artifact, for example `def`, `theorem`, `lemma`, `structure`, or `instance`. Container and exposition articles omit it. Autoform records this -hint but does not constrain the set of Lean declaration commands. Declarations -that introduce data rather than a proposition carry no separate proof -obligation. +intent and generated CI checks it against the built declaration. The supported +intents are `abbrev`, `axiom`, `class`, `corollary`, `def`, `definition`, +`inductive`, `instance`, `lemma`, `opaque`, `proposition`, `structure`, and +`theorem`; theorem-like aliases share Lean's kernel-level theorem kind. +Declarations that introduce data rather than a proposition carry no separate +proof obligation. `origin` records provenance for formalizable work: `cited` for a direct source target, `bridged` for a result introduced between source targets, and @@ -79,7 +84,9 @@ An article asserts only facts a human or agent verified: | --- | --- | | `statement: formalized` | The Lean statement exists and compiles. | | `proof: formalized` | The Lean proof is complete. | -| `mathlib: true` | The result is upstreamed into Mathlib. | +| `mathlib: true` | The exact result exists in the pinned Mathlib dependency. Requires `mathlib_declaration` and `mathlib_file`. | +| `mathlib_declaration: Ns.decl` | Exact upstream declaration name(s). | +| `mathlib_file: Mathlib/Path/File.lean` | Exact Mathlib source file that declares the upstream name(s). | | `not_ready: true` | Needs more blueprint work before it can be attempted. | | `lean: Ns.decl` | Declaration name(s) that discharge the article. | | `discussion: 42` | Issue number or URL where the article is being discussed. | @@ -115,17 +122,190 @@ the loaded plugin and prefix each one, running from the project root: uv run --project "" autoform check blueprint --lean-root . ``` +### Multi-project workspaces + +Use a workspace manifest when a repository contains several blueprint efforts, +when the legacy `blueprint/` name conflicts with its established layout, or when +Autoform must coexist with unrelated documentation. Initialize the +repository-level registry without creating a vault, then add projects +independently: + +```bash +autoform workspace init . --blueprint-root docs/blueprints +autoform blueprint new finite-flat \ + --path FiniteFlat --title "Finite Flat Group Schemes" +autoform blueprint new another-project \ + --path AnotherProject --title "Another Project" +autoform blueprint register imported-project \ + --path ExistingVault --title "Imported Project" +autoform workspace inspect . +autoform blueprint list . +autoform workspace check . --lean-root . +``` + +The generated `.autoform.toml` is the sole ownership registry: + +```toml +schema = "autoform-workspace/v1" + +[locations.blueprints] +path = "docs/blueprints" +provides = ["blueprints"] + +[projects."finite-flat"] +title = "Finite Flat Group Schemes" +blueprint = { location = "blueprints", path = "FiniteFlat" } +``` + +Location and project identifiers are user-defined. Autoform recognizes the +generic `blueprints` capability; it never special-cases repository names such +as those shown in examples. A repository may declare additional named locations +and capabilities for other tooling or future Autoform features. Project +blueprint paths name immediate child directories of their selected collection, +and registered vault paths may not overlap, so every managed vault has a +distinct boundary even when a repository declares several locations. + +No `autoform.toml` is written inside a vault. Autoform checks only entries under +`projects`; unregistered siblings are ignored. Paths are repository-relative, +portable to common case-sensitive and case-insensitive filesystems, confined +beneath the workspace root, distinct under Unicode-normalized case-insensitive +comparison, and may not traverse symbolic links. Windows-reserved names, +forbidden characters, control characters, trailing dots, and surrounding +whitespace are rejected before any filesystem mutation. Registration uses a +TOML-aware edit, preserving comments and supporting both standard and inline +`projects` tables while validating the complete result before publishing it +atomically. +Workspace mutation currently requires POSIX-style file locking, no-follow +opens, and directory-descriptor support; unsupported platforms fail before +writing. The manifest format itself remains portable across common +case-sensitive and case-insensitive filesystems. + +From inside a registered vault, single-project commands select that project. A +workspace root also selects its sole registered project. Autoform never infers +a project from an unrelated directory; pass `--project` there and at a +workspace root containing multiple projects: + +```bash +autoform check . --project finite-flat --lean-root . +autoform audit . --project finite-flat --lean-root . +autoform doctor . --project finite-flat --lean-root . +autoform-visualize . --project finite-flat +autoform render . --project finite-flat --output site-src/finite-flat +``` + +`workspace check` is the repository-wide verification command and visits every +registered project exactly once, applying the same path and symlink checks as +single-project commands. Explicit vault paths continue to work for +ad-hoc inspection, but do not add an unregistered directory to repository-wide +checks. Workspace initialization currently creates only the manifest and its +blueprint collection; publication remains an explicit per-vault setup decision. +`blueprint register` validates an existing vault and adds only the root registry +entry, which is the migration path for pre-existing blueprint directories. +Workspace JSON responses carry operation-specific versioned schemas so callers +can distinguish initialization, blueprint changes, listing, inspection, +checking, and errors. + +### Legacy single-vault setup + Create a new project's vault, site configuration, and CI. The layout is fixed, -so it is written rather than described; existing files are left alone, which -makes the same command the repair path: +so it is written rather than described. This command is retained for dedicated +repositories already using the canonical lowercase `blueprint/` layout: ```bash autoform init . --title "Finite Flat Group Schemes" \ --repository-url https://github.com/owner/repo ``` -Pass `--autoform-ref ` to pin the generated workflows at an immutable -commit, `--force` to overwrite, and `--json` for machine-readable output. +Pass `--autoform-source ` and +`--autoform-ref ` together to pin the generated workflows at an immutable +commit. Passing only one is an error. Use `--force` to overwrite and `--json` +for machine-readable output. + +Do not run legacy `init` in a manifest-managed workspace: it would create an +unregistered `blueprint/` vault. Use `workspace init` and `blueprint new` +instead. Likewise, `project repair` deliberately refuses manifest-managed +workspaces until shared publication infrastructure has a workspace-aware repair +contract. + +Create or inspect a Lean project and list Autoform's bundled known-good release pairs: + +```bash +autoform project versions +autoform project provenance --json +autoform project new ./FiniteFlat \ + --package FiniteFlat \ + --release lean-v4.32.2-mathlib-v4.32.2 \ + --autoform-source https://github.com/facebookresearch/autoform-bot.git \ + --autoform-ref +autoform project new . \ + --package FiniteFlat \ + --release lean-v4.32.2-mathlib-v4.32.2 \ + --autoform-source https://github.com/facebookresearch/autoform-bot.git \ + --autoform-ref +autoform project inspect . +autoform project inspect path/inside/project --json +autoform project repair . --dry-run --json +autoform project repair . --title "Finite Flat Group Schemes" \ + --repository-url https://github.com/owner/repo +autoform project versions --json +``` + +`project new` requires an absent target, or the literal target `.` when the +current directory is empty, plus an explicit release ID. An absent target uses +one atomic no-replace rename. The `.` form preserves the directory inode and +mode and uses a durable, recoverable transaction to publish each top-level +entry without replacement. It never overwrites an existing path. Exactly one +cooperative concurrent creator can win; ambiguous recovery state is preserved +for inspection rather than deleted. +The command does not run Git, Lake, Lean, subprocesses, or network operations. +It accepts an already verified Autoform source and full commit together, and +omits generated workflows when neither is supplied. + +`project provenance` is the online step. It accepts only the exact plugin-root +checkout or the bounded Codex installer record, fetches the recorded commit, +and compares the installed plugin and importable packages with that commit. +It reports a credential-free HTTPS source and full SHA only after all checks +pass. A plain wheel cannot infer provenance. Run it before creating the +consumer target, then pass both returned values to `project new`. + +`project repair` operates only on an explicitly named project root that already +has a clean, supported Lake/Lean configuration. It preserves every existing +managed path byte-for-byte and adds only absent Autoform overlay files whose +content is canonical and unambiguous. Missing parameterized files require their +exact inputs: `--title`, `--repository-url` (use an explicit empty value when +that is intended), and the `--autoform-source`/`--autoform-ref` pair for +workflows. A project whose workflows were deliberately omitted remains valid; +a partial workflow pair does not. Repair never writes when preflight finds a +symlink, unsafe or missing parent, stale repair temporary, malformed +configuration, unsupported release, or missing input. `--dry-run` performs the +same plan without mutation. Calls serialize on the project root, and +publication is atomic per file without replacing a concurrent writer. After an +interrupted multi-file repair, inspect any reported retained path, then retry +with the same inputs; there is no operation-wide transaction. If the project +changes after a file is published, repair retains that file and reports it for +manual recovery rather than risk unlinking a concurrent replacement. A failed +pre-publication attempt likewise retains and reports its exact temporary path +rather than deleting by pathname after a separate identity check. Like creation +and inspection, repair runs no Git, Lake, Lean, subprocess, or network operation. + +`project inspect` is deterministic, local, and read-only. It discovers the +nearest project root; parses bounded `lakefile.toml`, `lean-toolchain`, the +optional `.autoform.toml`, and known Autoform paths; records configuration +hashes; and reports whether the +configured Lean/Mathlib pair exactly matches the bundled catalog. It does not +run Lake, Lean, Git, subprocesses, or network operations. A `lakefile.lean` is +reported as present but unevaluated because executing it would violate that +boundary. Symlinked decision-bearing configuration and malformed consumed +fields fail inspection. Reports contain only project-relative paths, never the +host's absolute project location. + +`project versions` reads the catalog packaged with the installed wheel. The +catalog is an explicit known-good allowlist, not a resolver: the command never +contacts a registry, selects a version, or mutates a project. An unlisted but +structurally valid pair is advisory; absence from this catalog does not prove a +project is incompatible. It is a snapshot refreshed when Autoform is released; +its single recommended entry is the newest stable Lean and Mathlib pair +validated at that time. Publishing a project runs four steps in order: validate, write the Mermaid graph into the vault, render the site source, then strict-build the site. @@ -139,6 +319,20 @@ uv run --with mkdocs --with mkdocs-material --with mkdocs-literate-nav \ --with pymdown-extensions mkdocs build --strict ``` +Generated CI additionally rebuilds the root Lake package, then checks every +local `lean:` target belongs to one of those built modules. Each +`mathlib_declaration` must exist in the module named by `mathlib_file`. Lake +must resolve that module from the sole `mathlib` entry in `lake-manifest.json`. +That entry must pin a full commit from the canonical upstream Mathlib URL, and +the checked-out dependency must have the matching clean Git `HEAD` and origin. +The queried artifacts must remain inside that checkout's build directory, and +must carry a valid Lake build or cache trace; a full build trace must record the +`mathlib` package id. Local path packages, forks, mirrors, dirty or mismatched +checkouts, and other dependencies exporting a `Mathlib.*` module are rejected, +as is a root-package declaration impersonating a Mathlib result. Existence and +declaration kind are read from Lean's environment, not inferred from source +text. + Drop `--require-declarations` when reviewing work in progress, where a statement may name a Lean declaration that does not exist yet. @@ -162,6 +356,52 @@ canonical rows, counts, and the exact coverage source hash, while `publication.json` records aggregate counts without duplicating the authored rows. +For exhaustive source work, opt in with exact frontmatter: + +```markdown +--- +schema: autoform-coverage/v2 +artifact: sources/book.txt +artifact_sha256: <64 lowercase hex characters> +--- + +| Unit | Area | Lines | Locator | Unit SHA-256 | Coverage | Evidence | +| --- | --- | --- | --- | --- | --- | --- | +| chapter-one | First chapter | 1-42 | Chapter 1 | | DECOMPOSED | [Result](../roadmap/result.md) | +``` + +The artifact must be a nonempty, regular, non-symlink UTF-8 file with LF line +endings and a final LF. Ordered one-based spans must partition it exactly, and +each unit hash covers the raw LF-terminated bytes in that span. A decomposed +unit may link only to formalizable roadmap leaves. Those leaves reciprocate in +their frontmatter with `source_units: [chapter-one]`. The immutable +`load_execution_input` API binds this contract to the unchanged +`autoform-runtime/v1` projection; schema-less v1 remains valid for audit and +render but is refused for ready-work discovery with `coverage-v2-required`. + +List the formalization phases that are ready for the current agent: + +```bash +autoform ready blueprint --lean-root . +autoform ready . --project finite-flat --lean-root . --json +``` + +`ready` is read-only. It loads the same immutable execution input, requires +durable `article_id` frontmatter, and returns only dispatchable formalizable +leaves whose statement or proof prerequisites are satisfied. Its deterministic +JSON includes the roadmap and source-contract revisions, structured blocked +items with unmet dependency IDs, and counts for ready, blocked, and complete +work. With local formalized progress, pass `--lean-root`; `ready` rejects +missing or unresolved Lean targets rather than treating stale metadata as +complete. It does not acquire a claim or edit files; use the claim commands +below before beginning the returned item. + +A v2 publication excludes the entire `blueprint/sources/` authority tree, so +renamed artifacts cannot survive an incremental render. With repository +coordinates its authored Markdown links become repository blob links; without +them, those links become plain text instead of dangling site links. Raw HTML +links into the excluded tree are rejected. + The contract is read as published Markdown and fails closed. A table inside an HTML comment, a fenced block, or a four-space-indented block is documentation rather than contract, and is not discovered at all. A closing fence must carry @@ -282,8 +522,9 @@ autoform migrate article-ids blueprint --check `article_id` accepts opaque values in the form `af_` plus 24 lowercase hex digits. The planner validates uniqueness, proposes deterministic IDs for missing articles, includes exact source hashes, and is strictly read-only. -Applying plans, moving runtime consumers and claims to durable IDs, and -preserving publication routes are intentionally deferred to follow-up changes. +Apply the proposed IDs to article frontmatter before dispatching collaborative +work. Claims resolve current roadmap paths to these durable IDs, so renaming an +article does not change its lock. Coordinate temporary cross-machine ownership without modifying the book: @@ -292,13 +533,22 @@ export AUTOFORM_WORKER_ID="agent-name" autoform claim acquire "chapter/main-result" autoform claim renew "chapter/main-result" autoform claim release "chapter/main-result" +autoform claim acquire --resource lake-build +autoform claim list +autoform claim cleanup ``` Claims are fail-closed compare-and-swap leases under `refs/autoform-claims/` on the Git `origin`; pass `--repo` for another claim -board. A failed acquire or renew means the caller cannot prove ownership and -must stop before committing or pushing protected work. Claims do not prove -mathematical correctness and do not replace branch-level Git CAS. +board. Article targets are resolved against the current project's `blueprint/` +and keyed by their durable `article_id`; use `--blueprint` when invoking the +command elsewhere. Raw locks require `--resource`. The CLI derives a stable +session from the worktree, with `--session-id` or +`AUTOFORM_CLAIM_SESSION_ID` as an explicit override. A failed acquire or renew +means the caller cannot prove ownership and must stop before committing or +pushing protected work. Claims do not prove mathematical correctness and do +not replace branch-level Git CAS. `list` and `cleanup` need neither a worker nor +a worktree when `--repo` and, if needed, `--scratch` are supplied. Write the Mermaid dependency graph into the vault, where Obsidian renders it: @@ -326,6 +576,22 @@ its local context. Point `mkdocs.yml` at `docs_dir: site-src` and enable `md_in_html` plus a `pymdownx.superfences` mermaid fence; see the [repository example](../skills/setup/assets/cabannes-thesis-project/mkdocs.yml). +Publication is staged, synced, validated, and atomically exchanged with the +previous generated site. This fail-closed transaction requires macOS +`renameatx_np` or Linux `renameat2`; other platforms can still use the remaining +CLI commands but cannot run `autoform render`. A legacy +`autoform-publication/v1` output is never deleted automatically. Remove it +explicitly or choose an empty output directory once, then subsequent v2 renders +can replace only the exact checksummed generation they inspected. +The renderer hashes both the blueprint snapshot and the exact Lean-file +generation used for declaration links, then rechecks both under the publication +lock immediately before the atomic rename. That check is the publication +linearization point; later source edits belong to the next render. Generated +v1/v2 publication trees and private staging directories are never indexed as +Lean source. +An existing v2 publication from before Lean-source hashing is still replaced +only after its complete inventory is verified, then upgraded in place. + ## Validation `autoform check` rejects cycles, missing targets, escaping paths, @@ -387,22 +653,43 @@ work, stamps articles, or creates another graph artifact. ## Claim contract -Claims use canonical `autoform-claim/v1` JSON in orphan commit messages and -exact observed object IDs as update preconditions. Absent and verifiably expired -leases may be acquired; live peer leases are refused. Malformed or unreadable -refs are unverifiable and may not be acquired, renewed, released, or removed by -cleanup. A heartbeat verifies ownership on entry and permanently records any -later refusal or transport uncertainty as lost ownership. - -A claim key is a slug and digest of any string, not a validated node id, so a -shared resource is locked the same way a node is. Parallel agents get one Git -worktree each and serialize `lake build` behind a `lake-build` claim, because -builds share the elan toolchain and the Mathlib cache even when the checkouts -are separate. - -Claims are temporary operational state, never article frontmatter. Future -Deicyde workers may share this protocol, but their current continue-uncoordinated -failure behavior must be removed before they use the canonical claim API. +Claims use canonical `autoform-claim/v2` JSON in orphan commit messages. A +cryptographically random lease ID and a session-local receipt for the exact +pushed object fence every ownership operation; `worker_id` is display metadata, +not authority. Live peer leases cannot be stolen. Leases are limited to 3600 +seconds and assume clocks differ by at most 300 seconds. Entries outside those +bounds fail closed, appear as `_recovery_required` in `claim list`, and are +recovered only by an explicit CAS-safe `claim cleanup`. A heartbeat captures one +lease ID, records any refusal or transport uncertainty as lost ownership, and +waits for an in-flight renewal before exiting. + +Owners must stop at `expires_at`. Other observers cannot take over or clean up +the ref until `expires_at + 300` seconds, so the intervening skew window fails +closed instead of admitting two owners. A valid unrenewed lease is bounded by +its 3600-second TTL plus this 300-second reclaim grace; a timestamp already 300 +seconds ahead of an observer can add at most one further 300-second offset. +Renewals clamp their timestamp and expiry to the prior values when a clock steps +backward. + +Moving from v1 path keys to v2 durable IDs is a one-way rollout. Stop v1 clients +before the first v2 claim. Autoform refuses live or unreadable v1 author refs, +replaces expired v1 refs with permanent compatibility fences, and installs a +fence for the current article path or raw resource name before acquiring its v2 +key. Old clients reject those fences, so they cannot acquire a path already +owned by v2. Historical renamed paths with no claim ref cannot be discovered; +retiring v1 clients is therefore part of the protocol, not an optional cleanup. +Use `claim cleanup --blueprint PROJECT` during rollout so expired v1 path refs +become fences while expired durable-ID refs remain reusable. + +Article claims require a real graph node with materialized `article_id` +frontmatter. Separate raw-resource keys cover coordination outside the roadmap. +Parallel contributors get one Git worktree each and serialize shared build state with +`autoform claim acquire --resource lake-build`. + +Leases are temporary operational state; compatibility fences are persistent +migration state. Neither belongs in article frontmatter. The Orchestrate skill +is a thin client of `ready`, `claim`, `check`, and `audit`; there is no second +worker scheduler or provider-specific execution service. ## Local runtime doctor @@ -423,10 +710,10 @@ review. The bundled example intentionally exits nonzero while its declared coverage still holds `MAPPED` rows. This command is strictly read-only and local. It does not invoke Git, GitHub, -subprocesses, network services, claims, queues, reviews, recovery state, -providers, workers, renderers, or dashboards, and it creates no cache, scratch +subprocesses, network services, claims, reviews, orchestration state, +providers, renderers, or dashboards, and it creates no cache, scratch repository, service, state directory, or `graph.json`. It is a project/runtime -doctor, separate from any future Deicyde fleet or machine-capability preflight. +doctor, separate from ready-work discovery and machine-capability preflight. ## Runtime contract @@ -447,12 +734,10 @@ and bytes, excluding timestamps, absolute paths, Git state, and operational state. Optional Lean locations come from a local lexical scan and do not by themselves establish compilation or proof correctness. -Schema v1 retains the graph's path-derived article ID. That is suitable for -an ephemeral runtime projection and temporary claims, but it is not yet an -approved durable identity. Queues, reviews, recovery records, PR markers, -dashboard routes, providers, and logs must not persist against this ID until a -path-move identity and migration policy is defined. Those records remain private -and excluded from runtime snapshots and publication. +Each node retains its path-derived `id` for links and carries optional authored +`article_id` frontmatter for identity across path moves. `autoform ready` and +article claims require that durable identity; ordinary check, audit, render, +and human review continue to support roadmaps while IDs are being migrated. ## Publication contract @@ -463,6 +748,14 @@ credentials, logs, provider state, and agent/task state inside the blueprint cause the render to fail rather than silently leak them. Source and output directories must be disjoint. -Every render writes `publication.json` with the source-content hash, Git ref, -article and dependency counts, and available views. It contains no timestamp or -absolute path, so identical inputs produce identical output files. +Every render writes `publication.json` with blueprint and Lean-source hashes, +Git ref, article and dependency counts, complete file inventory, and available +views. It contains no timestamp or absolute path, so identical inputs produce +identical output files. Autoform validates and syncs the staged tree before one +atomic filesystem commit, then verifies ownership and syncs both parent +directories. Once the commit begins, Autoform never tries to exchange a recovery +path back into the live destination. If it cannot verify the final state or +durability, it preserves the private workspace and reports its exact recovery +path instead of deleting a potentially unique generation. If the site was fully +verified before cleanup becomes unsafe, the render succeeds and reports the +retained workspace as a warning. diff --git a/autoform_cli/__init__.py b/autoform_cli/__init__.py index ec3d057c..f7a5d77f 100644 --- a/autoform_cli/__init__.py +++ b/autoform_cli/__init__.py @@ -1,5 +1,19 @@ """Command-line support for Autoform blueprints.""" +from .execution_input import ExecutionInput, ExecutionInputError, load_execution_input from .graph import Graph, GraphValidationError, Node, load_graph +from .ready import ReadyBlock, ReadyItem, ReadyResult, list_ready_work -__all__ = ["Graph", "GraphValidationError", "Node", "load_graph"] +__all__ = [ + "ExecutionInput", + "ExecutionInputError", + "Graph", + "GraphValidationError", + "Node", + "ReadyBlock", + "ReadyItem", + "ReadyResult", + "list_ready_work", + "load_execution_input", + "load_graph", +] diff --git a/autoform_cli/__main__.py b/autoform_cli/__main__.py index d1c661ed..617502dd 100644 --- a/autoform_cli/__main__.py +++ b/autoform_cli/__main__.py @@ -6,21 +6,116 @@ import hashlib import json import os +import re +import secrets import socket +import stat import subprocess import sys -from collections.abc import Sequence -from pathlib import Path +import tempfile +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path, PurePosixPath from . import status from .article_identity import plan_article_ids from .audit import audit_blueprint -from .claims import CLAIM_TTL_S, ClaimBoard, ClaimTransportError, author_claim_key +from .claims import ( + CLAIM_TTL_S, + ClaimBoard, + ClaimTransportError, + _claim_git_environment, + author_claim_key, + claim_repository_is_remote, + pin_claim_repository, + pin_claim_scratch, + resource_claim_key, + workspace_author_claim_key, +) from .doctor import diagnose_project -from .graph import GraphValidationError, load_graph +from .execution_input import ExecutionInputError +from .graph import ARTICLE_ID_PATTERN, GraphValidationError from .lean import build_linker, declaration_names +from .project import ( + ProjectCatalogError, + ProjectCreateError, + ProjectRepairError, + create_project, + inspect_project, + load_release_catalog, + repair_project, +) +from .provenance import ProvenanceError, verify_plugin_provenance +from .ready import READY_SCHEMA, list_ready_work from .render import PublicationError, render_site +from .runtime import ( + RuntimePaths, + RuntimeProjectionError, + bind_runtime_paths, + load_bound_graph, +) from .scaffold import ScaffoldError, scaffold_project +from .workspace_cli import add_workspace_parsers, run_blueprint_command, run_workspace_command + +_CLAIM_TEMP_DIRECTORY = Path(tempfile.gettempdir()).resolve() + + +@dataclass(frozen=True, slots=True) +class _ClaimBoardIdentity: + repo: str + repo_identity: tuple[int, int] | None + session_id: str + scratch: Path + scratch_identity: tuple[int, int] | None + + +@dataclass(frozen=True, slots=True) +class _PinnedDirectory: + path: Path + identity: tuple[tuple[int, int, int | None], ...] + + @staticmethod + def _snapshot(path: Path, *, label: str) -> tuple[tuple[int, int, int | None], ...]: + snapshot: list[tuple[int, int, int | None]] = [] + temp_ancestors = frozenset( + (_CLAIM_TEMP_DIRECTORY, *_CLAIM_TEMP_DIRECTORY.parents) + ) + for component in reversed((path, *path.parents)): + try: + info = component.stat(follow_symlinks=False) + except OSError as exc: + raise ValueError(f"{label} cannot be inspected safely") from exc + if not stat.S_ISDIR(info.st_mode): + raise ValueError(f"{label} must have only real directory components") + changed_at_ns = None if component in temp_ancestors else info.st_ctime_ns + snapshot.append((info.st_dev, info.st_ino, changed_at_ns)) + return tuple(snapshot) + + @classmethod + def capture(cls, path: Path, *, label: str) -> _PinnedDirectory: + before = cls._snapshot(path, label=label) + after = cls._snapshot(path, label=label) + if before != after: + raise ValueError(f"{label} changed while its path was being inspected") + return cls(path=path, identity=after) + + def verify(self, *, label: str) -> None: + try: + current = self._snapshot(self.path, label=label) + except ValueError as exc: + raise ValueError(f"{label} was replaced while resolving the claim") from exc + if current != self.identity: + raise ValueError(f"{label} was replaced while resolving the claim") + + +@dataclass(frozen=True, slots=True) +class _ResolvedClaimTarget: + key: str + label: str + compatibility_keys: tuple[str, ...] + canonical_keys: tuple[str, ...] + board_identity: _ClaimBoardIdentity def main(argv: Sequence[str] | None = None) -> int: @@ -34,18 +129,19 @@ def main(argv: Sequence[str] | None = None) -> int: init.add_argument( "--autoform-source", default="", - help="Autoform Git source the generated workflows install from (default: this checkout's origin)", + help="Autoform Git source for generated workflows (default: verified installation source)", ) init.add_argument( "--autoform-ref", default="", - help="immutable ref the workflows pin (default: this checkout's HEAD commit)", + help="full commit for generated workflows (default: verified installation revision)", ) init.add_argument("--force", action="store_true", help="overwrite files that already exist") init.add_argument("--json", action="store_true", help="write stable machine-readable output") check = subparsers.add_parser("check", help="validate a Markdown blueprint") check.add_argument("blueprint_dir") + check.add_argument("--project", help="registered workspace project id") check.add_argument( "--lean-root", type=Path, @@ -54,19 +150,103 @@ def main(argv: Sequence[str] | None = None) -> int: audit = subparsers.add_parser("audit", help="audit roadmap completeness and checked facts") audit.add_argument("blueprint_dir") + audit.add_argument("--project", help="registered workspace project id") audit.add_argument("--lean-root", type=Path, help="Lean project to resolve local targets against") audit.add_argument("--json", action="store_true", help="write stable machine-readable output") doctor = subparsers.add_parser("doctor", help="diagnose the local Markdown runtime contract") doctor.add_argument("project_or_blueprint") + doctor.add_argument("--project", help="registered workspace project id") doctor.add_argument("--lean-root", type=Path, help="Lean project to resolve local targets against") doctor.add_argument("--json", action="store_true", help="write stable machine-readable output") - claim = subparsers.add_parser("claim", help="coordinate temporary node ownership through Git refs") + ready = subparsers.add_parser( + "ready", help="list formalization work whose authored prerequisites are satisfied" + ) + ready.add_argument("project_or_blueprint") + ready.add_argument("--project", help="registered workspace project id") + ready.add_argument("--lean-root", type=Path, help="Lean project to bind into the execution input") + ready.add_argument("--json", action="store_true", help="write stable machine-readable output") + + project = subparsers.add_parser("project", help="inspect local project configuration and releases") + project_subparsers = project.add_subparsers(dest="project_command", required=True) + project_new = project_subparsers.add_parser( + "new", help="atomically create a complete Lean and Autoform project" + ) + project_new.add_argument( + "target", + nargs="?", + help="new absent directory, or '.' for the empty current directory", + ) + project_new.add_argument("--package", help="UpperCamelCase Lean package name") + project_new.add_argument("--release", help="release id from 'project versions'") + project_new.add_argument( + "--autoform-source", + default="", + help="trusted Autoform Git source for generated workflows", + ) + project_new.add_argument( + "--autoform-ref", + default="", + help="full 40-character Autoform commit for generated workflows", + ) + project_new.add_argument("--json", action="store_true", help="write stable machine-readable output") + project_repair = project_subparsers.add_parser( + "repair", help="conservatively add unambiguous missing project files" + ) + project_repair.add_argument("target", help="existing project directory") + project_repair.add_argument( + "--title", help="exact human project title for missing generated files" + ) + project_repair.add_argument( + "--repository-url", + help="exact project URL for a missing site configuration (empty is allowed)", + ) + project_repair.add_argument( + "--autoform-source", + help="exact Autoform Git source for missing workflows", + ) + project_repair.add_argument( + "--autoform-ref", + help="exact immutable Autoform commit for missing workflows", + ) + project_repair.add_argument("--dry-run", action="store_true", help="report without writing") + project_repair.add_argument("--json", action="store_true", help="write stable machine-readable output") + project_inspect = project_subparsers.add_parser( + "inspect", help="inspect a project without running Lake, Git, or network operations" + ) + project_inspect.add_argument( + "target", nargs="?", default=".", help="a path inside the project (default: current directory)" + ) + project_inspect.add_argument("--json", action="store_true", help="write stable machine-readable output") + project_versions = project_subparsers.add_parser( + "versions", help="list bundled known-good Lean and Mathlib releases" + ) + project_versions.add_argument("--json", action="store_true", help="write stable machine-readable output") + project_provenance = project_subparsers.add_parser( + "provenance", + help="verify immutable provenance for this Autoform installation", + ) + project_provenance.add_argument( + "--json", action="store_true", help="write stable machine-readable output" + ) + + add_workspace_parsers(subparsers) + + claim = subparsers.add_parser( + "claim", help="coordinate temporary article and resource ownership through Git refs" + ) claim_subparsers = claim.add_subparsers(dest="claim_command", required=True) for operation in ("acquire", "renew", "release"): command = claim_subparsers.add_parser(operation) - command.add_argument("node_id") + command.add_argument("node_id", nargs="?", help="roadmap path id or exact article_id") + command.add_argument("--resource", help="claim a raw shared resource instead of an article") + command.add_argument( + "--blueprint", + default=".", + help="project or blueprint directory used to resolve the article (default: current directory)", + ) + command.add_argument("--project", help="registered workspace project id") _add_claim_board_arguments(command) if operation in {"acquire", "renew"}: command.add_argument("--ttl", type=int, default=CLAIM_TTL_S) @@ -76,6 +256,11 @@ def main(argv: Sequence[str] | None = None) -> int: _add_claim_board_arguments(claim_list) claim_cleanup = claim_subparsers.add_parser("cleanup") _add_claim_board_arguments(claim_cleanup) + claim_cleanup.add_argument( + "--blueprint", + help="project or blueprint directory required to retire legacy author refs safely", + ) + claim_cleanup.add_argument("--project", help="registered workspace project id") migrate = subparsers.add_parser("migrate", help="inspect authored migration contracts") migrate_subparsers = migrate.add_subparsers(dest="migrate_command", required=True) @@ -84,6 +269,7 @@ def main(argv: Sequence[str] | None = None) -> int: help="plan durable roadmap article identifiers without writing files", ) article_ids.add_argument("blueprint_dir") + article_ids.add_argument("--project", help="registered workspace project id") article_ids.add_argument( "--check", action="store_true", @@ -91,11 +277,14 @@ def main(argv: Sequence[str] | None = None) -> int: ) article_ids.add_argument("--json", action="store_true", help="write stable machine-readable output") - render = subparsers.add_parser - render = subparsers.add_parser("render", help="build the publishable blueprint") render.add_argument("blueprint_dir") - render.add_argument("-o", "--output", default="site-src", help="output directory") + render.add_argument("--project", help="registered workspace project id") + render.add_argument( + "-o", + "--output", + help="output directory (default: site-src, or site-src/ in a workspace)", + ) render.add_argument("--lean-root", type=Path, help="Lean project to link code from") render.add_argument("--repository-url", help="project URL, e.g. https://github.com/owner/repo") render.add_argument("--ref", help="commit or branch the code links should pin") @@ -115,6 +304,14 @@ def main(argv: Sequence[str] | None = None) -> int: return _audit(args) if args.command == "doctor": return _doctor(args) + if args.command == "ready": + return _ready(args) + if args.command == "project": + return _project(args) + if args.command == "workspace": + return run_workspace_command(args) + if args.command == "blueprint": + return run_blueprint_command(args) if args.command == "claim": return _claim(args) if args.command == "migrate": @@ -129,9 +326,20 @@ def _add_claim_board_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--worker-id", default=os.environ.get("AUTOFORM_WORKER_ID"), - help="stable identity for this agent (or set AUTOFORM_WORKER_ID)", + help="display identity for this agent (or set AUTOFORM_WORKER_ID)", + ) + parser.add_argument( + "--session-id", + default=os.environ.get("AUTOFORM_CLAIM_SESSION_ID"), + help="stable work session identity (or set AUTOFORM_CLAIM_SESSION_ID)", ) parser.add_argument("--scratch", type=Path, help="local bare Git object cache") + parser.add_argument( + "--object-format", + choices=("sha1", "sha256"), + default=os.environ.get("AUTOFORM_GIT_OBJECT_FORMAT"), + help="Git object format for an empty network claim repository", + ) def _init(args: argparse.Namespace) -> int: @@ -169,9 +377,10 @@ def _init(args: argparse.Namespace) -> int: sys.stdout.flush() print( "\nCI was not written: generated workflows install Autoform from a Git\n" - "ref, and this Autoform is not running from a checkout, so there is\n" - "nothing to pin. Re-run with the commit to add them:\n" - " autoform init --autoform-ref <40-char-sha>", + "ref, and this installation has no verified source and commit.\n" + "Re-run with the complete pair to add them:\n" + " autoform init --autoform-source " + "--autoform-ref <40-char-sha>", file=sys.stderr, ) return 0 @@ -179,28 +388,38 @@ def _init(args: argparse.Namespace) -> int: def _check(args: argparse.Namespace) -> int: try: - graph = load_graph(args.blueprint_dir) - except GraphValidationError as exc: + with _resolved_blueprint(args.blueprint_dir, args.project) as paths: + graph = load_bound_graph(paths) + + statuses = status.derive(graph) + summary = " · ".join( + f"{count} {state.label}" for state, count in status.summarize(statuses) + ) + missing: list[str] = [] + if args.lean_root is None: + paths.verify() + else: + linker = build_linker(args.lean_root) + missing = [ + f"{node.id}: declaration not found in {args.lean_root}: {name}" + for node in graph.nodes.values() + for name in declaration_names(node.lean or "") + if linker.location(name) is None + ] + paths.verify() + except (GraphValidationError, RuntimeProjectionError) as exc: for issue in exc.issues: print(f"error: {issue}") return 1 + except OSError as exc: + print(f"error: {exc}") + return 1 - statuses = status.derive(graph) - summary = " · ".join(f"{count} {state.label}" for state, count in status.summarize(statuses)) print(f"OK: {len(graph.nodes)} articles, {graph.edge_count} dependencies") if summary: print(f" {summary}") - if args.lean_root is None: return 0 - - linker = build_linker(args.lean_root) - missing = [ - f"{node.id}: declaration not found in {args.lean_root}: {name}" - for node in graph.nodes.values() - for name in declaration_names(node.lean or "") - if linker.location(name) is None - ] for issue in missing: print(f"error: {issue}") if missing: @@ -211,7 +430,22 @@ def _check(args: argparse.Namespace) -> int: def _audit(args: argparse.Namespace) -> int: - result = audit_blueprint(args.blueprint_dir, lean_root=args.lean_root) + try: + with _resolved_blueprint(args.blueprint_dir, args.project) as paths: + result = audit_blueprint( + paths.blueprint_dir, + lean_root=args.lean_root, + _expected_blueprint_identity=paths.blueprint_identity, + _expected_roadmap_identity=paths.roadmap_identity, + ) + paths.verify() + except RuntimeProjectionError as error: + if args.json: + print(json.dumps({"clean": False, "errors": list(error.issues)}, sort_keys=True, separators=(",", ":"))) + else: + for issue in error.issues: + print(f"error: {issue}") + return 1 if args.json: print(result.to_json()) else: @@ -232,7 +466,11 @@ def _audit(args: argparse.Namespace) -> int: def _doctor(args: argparse.Namespace) -> int: - result = diagnose_project(args.project_or_blueprint, lean_root=args.lean_root) + result = diagnose_project( + args.project_or_blueprint, + lean_root=args.lean_root, + project_id=args.project, + ) if args.json: print(result.to_json()) else: @@ -242,29 +480,304 @@ def _doctor(args: argparse.Namespace) -> int: return 0 if result.clean else 1 +def _ready(args: argparse.Namespace) -> int: + try: + result = list_ready_work( + args.project_or_blueprint, + lean_root=args.lean_root, + project_id=args.project, + ) + except ExecutionInputError as error: + if args.json: + print( + json.dumps( + { + "blocked_items": [], + "errors": [ + {"code": issue.code, "reason": issue.reason} + for issue in error.issues + ], + "items": [], + "schema": READY_SCHEMA, + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + else: + for issue in error.issues: + print(f"error: {issue.code}: {issue.reason}") + return 1 + if args.json: + print(result.to_json()) + return 0 + for item in result.items: + print( + f"READY: {item.phase}: {item.article_id}: " + f"{item.node_id} ({item.article_path})" + ) + if not result.items: + print("BLOCKED: no ready work" if result.blocked_items else "OK: no ready work") + for item in result.blocked_items: + dependencies = ", ".join(item.blocked_by) or "none" + print( + f"BLOCKED: {item.phase}: {item.article_id}: {item.node_id} " + f"({item.article_path}): {', '.join(item.reasons)}; blocked by: {dependencies}" + ) + print( + f" {len(result.items)} ready · {result.blocked} blocked · " + f"{result.complete} complete" + ) + return 0 + + +@contextmanager +def _resolved_blueprint( + target: str | Path, + project_id: str | None, +) -> Iterator[RuntimePaths]: + with bind_runtime_paths(target, project_id=project_id) as paths: + yield paths + + +def _project(args: argparse.Namespace) -> int: + try: + if args.project_command == "new": + result = create_project( + args.target, + package=args.package, + release_id=args.release, + autoform_source=args.autoform_source, + autoform_ref=args.autoform_ref, + ) + if args.json: + print(result.to_json()) + else: + print(f"Created {result.package} at {result.target} ({result.release})") + if not result.workflows_pinned: + print("warning: workflows were omitted because no immutable Autoform pin was available") + return 0 + if args.project_command == "repair": + result = repair_project( + args.target, + dry_run=args.dry_run, + title=args.title, + repository_url=args.repository_url, + autoform_source=args.autoform_source, + autoform_ref=args.autoform_ref, + ) + if args.json: + print(result.to_json()) + else: + action = "Would add" if result.dry_run else "Added" + print(f"{action} {len(result.planned if result.dry_run else result.written)} file(s)") + for path in result.planned if result.dry_run else result.written: + print(f" {path}") + return 0 + if args.project_command == "inspect": + result = inspect_project(args.target) + if args.json: + print(result.to_json()) + else: + _print_project_inspection(result) + return 0 if result.ok else 1 + if args.project_command == "versions": + catalog = load_release_catalog() + if args.json: + print(catalog.to_json()) + else: + print("Supported Lean/Mathlib releases:") + for release in catalog.releases: + suffix = " [recommended]" if release.recommended else "" + print(f" {release.id}{suffix}") + print(f" Lean: {release.lean.toolchain}") + print(f" Mathlib: {release.mathlib.revision} ({release.mathlib.git})") + return 0 + if args.project_command == "provenance": + result = verify_plugin_provenance() + if args.json: + print(json.dumps(result.as_dict(), sort_keys=True, separators=(",", ":"))) + else: + print(f"Source: {result.source}") + print(f"Revision: {result.revision}") + return 0 + except ProjectRepairError as error: + if getattr(args, "json", False): + print(error.to_json()) + else: + print(f"error[{error.code}]: {error.message}", file=sys.stderr) + for conflict in error.conflicts: + location = f" {conflict.path}" if conflict.path else "" + print(f" {conflict.code}{location}: {conflict.message}", file=sys.stderr) + if error.written: + print(" files already published:", file=sys.stderr) + for path in error.written: + print(f" {path}", file=sys.stderr) + return 1 + except ProjectCreateError as error: + if getattr(args, "json", False): + print(error.to_json()) + else: + print(f"error[{error.code}]: {error.message}", file=sys.stderr) + return 1 + except ProjectCatalogError: + if getattr(args, "json", False): + print( + json.dumps( + { + "error": { + "code": "project-catalog-invalid", + "message": "The bundled project release catalog is invalid.", + }, + "ok": False, + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + else: + print("error: bundled project release catalog is invalid", file=sys.stderr) + return 1 + except ProvenanceError as error: + if getattr(args, "json", False): + print( + json.dumps( + { + "error": {"code": error.code, "message": error.message}, + "ok": False, + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + else: + print(f"error[{error.code}]: {error.message}", file=sys.stderr) + return 1 + return 2 + + +def _print_project_inspection(result) -> None: + if result.project_root is not None: + print(f"Project: {result.project_root}") + if result.lake is not None: + package = result.lake.name or "unknown package" + version = f" {result.lake.version}" if result.lake.version else "" + print(f"Lake: {package}{version} ({result.lake.path})") + for target in result.lake.targets: + source_parts = [ + part + for part in (result.lake.package_src_dir, target.src_dir) + if part is not None + ] + source = PurePosixPath(*source_parts).as_posix() if source_parts else "." + modules = target.roots or ((target.root,) if target.root is not None else ()) + module_note = f", roots: {', '.join(modules)}" if modules else "" + print(f" {target.kind} {target.name} (srcDir: {source}{module_note})") + if result.lean is not None: + print(f"Lean: {result.lean.toolchain}") + if result.mathlib is not None: + print(f"Mathlib: {result.mathlib.revision or 'none'} ({result.mathlib.git or 'none'})") + if result.autoform.manifest_path is not None: + print(f"Autoform workspace: {result.autoform.manifest_path}") + for path in result.autoform.blueprint_paths: + print(f" blueprint: {path}") + elif result.autoform.blueprint_path is not None: + print(f"Autoform blueprint: {result.autoform.blueprint_path}") + print( + f"Compatibility: {result.compatibility.status}" + + (f" ({result.compatibility.release})" if result.compatibility.release else "") + ) + for diagnostic in result.diagnostics: + location = f" {diagnostic.path}" if diagnostic.path else "" + print( + f"{diagnostic.severity}[{diagnostic.code}]{location}: {diagnostic.message}", + file=sys.stderr, + ) + + def _claim(args: argparse.Namespace) -> int: try: - board = _claim_board(args) operation = args.claim_command if operation == "list": + board = _claim_board(args, require_identity=False) print(json.dumps(board.list(), sort_keys=True, separators=(",", ":"))) return 0 if operation == "cleanup": - print(f"removed {board.cleanup()} expired claim(s)") + canonical_keys = None + board_identity = None + if args.blueprint is not None: + try: + with bind_runtime_paths(args.blueprint, project_id=args.project) as paths: + paths.require_strong_binding(operation="claim cleanup") + blueprint = paths.blueprint_dir + project_pin = _PinnedDirectory.capture( + paths.project_root, + label="claim project", + ) + blueprint_pin = _PinnedDirectory.capture( + blueprint, + label="claim blueprint", + ) + graph = load_bound_graph(paths) + board_identity = _resolve_claim_board_identity( + args, + context=paths.project_root, + require_identity=False, + ) + project_pin.verify(label="claim project") + blueprint_pin.verify(label="claim blueprint") + key_factory = ( + (lambda article_id: workspace_author_claim_key( + paths.workspace_project_id, article_id + )) + if paths.workspace_project_id is not None + else author_claim_key + ) + canonical_keys = tuple( + key_factory(node.article_id) + for node in graph.nodes.values() + if node.article_id is not None + ) + paths.verify() + except RuntimeProjectionError as exc: + raise ValueError(str(exc)) from exc + except GraphValidationError as exc: + raise ValueError("; ".join(exc.issues)) from exc + board = _claim_board( + args, + identity=board_identity, + require_identity=False, + ) + print( + f"recovered {board.cleanup(canonical_keys=canonical_keys)} " + "expired or unsafe-timestamp claim(s)" + ) return 0 - key = author_claim_key(args.node_id) + target = _resolve_claim_target(args) + board = _claim_board(args, identity=target.board_identity) + if operation in {"acquire", "renew"} and target.compatibility_keys: + if not board.prepare_v2_claim( + target.key, + target.compatibility_keys, + canonical_keys=target.canonical_keys, + ): + print( + f"error: could not {operation} {target.label}; " + "a live legacy v1 claim or incompatible path claim blocks v2 rollout" + ) + return 1 if operation == "acquire": - succeeded = board.acquire(key, ttl=args.ttl, note=args.note) + succeeded = board.acquire(target.key, ttl=args.ttl, note=args.note) elif operation == "renew": - succeeded = board.renew(key, ttl=args.ttl) + succeeded = board.renew(target.key, ttl=args.ttl) else: - succeeded = board.release(key) + succeeded = board.release(target.key) if succeeded: past_tense = {"acquire": "acquired", "renew": "renewed", "release": "released"} - print(f"{past_tense[operation]} {args.node_id} ({key})") + print(f"{past_tense[operation]} {target.label} ({target.key})") return 0 - print(f"error: could not {operation} {args.node_id}; ownership is held or unverifiable") + print(f"error: could not {operation} {target.label}; ownership is held or unverifiable") return 1 except (ClaimTransportError, ValueError) as exc: print(f"error: {exc}") @@ -275,8 +788,11 @@ def _migrate(args: argparse.Namespace) -> int: if args.migrate_command != "article-ids": return 2 try: - plan = plan_article_ids(args.blueprint_dir) - except GraphValidationError as error: + with _resolved_blueprint(args.blueprint_dir, args.project) as paths: + graph = load_bound_graph(paths) + plan = plan_article_ids(paths.blueprint_dir, _graph=graph) + paths.verify() + except (GraphValidationError, RuntimeProjectionError) as error: for issue in error.issues: print(f"error: {issue}", file=sys.stderr) return 2 @@ -293,45 +809,343 @@ def _migrate(args: argparse.Namespace) -> int: return 1 if args.check and not plan.complete else 0 -def _claim_board(args: argparse.Namespace) -> ClaimBoard: - worker_id = args.worker_id - if not worker_id: +def _resolve_claim_board_identity( + args: argparse.Namespace, + *, + context: str | Path | None = None, + require_identity: bool = True, +) -> _ClaimBoardIdentity: + repo = args.repo + session_id = args.session_id + context_pin = None + if repo is None or (require_identity and session_id is None): + if context is None: + context = getattr(args, "blueprint", None) or "." + context = Path(context).expanduser().resolve() + context_pin = _PinnedDirectory.capture(context, label="claim context") + if repo is None: + assert context is not None + repo = _origin_url(context) + if session_id is None and require_identity: + assert context is not None + session_id = _worktree_claim_session_id(context) + if session_id is None: + session_id = "claim-maintenance" + if context_pin is not None: + context_pin.verify(label="claim context") + normalized_repo, repo_identity = pin_claim_repository(repo) + scratch, scratch_identity = pin_claim_scratch( + args.scratch or _default_claim_scratch(normalized_repo, session_id) + ) + return _ClaimBoardIdentity( + repo=normalized_repo, + repo_identity=repo_identity, + session_id=session_id, + scratch=scratch, + scratch_identity=scratch_identity, + ) + + +def _claim_board( + args: argparse.Namespace, + *, + identity: _ClaimBoardIdentity | None = None, + require_identity: bool = True, +) -> ClaimBoard: + worker_id = args.worker_id or ("claim-maintenance" if not require_identity else None) + if worker_id is None: raise ValueError("--worker-id or AUTOFORM_WORKER_ID is required") - repo = args.repo or _origin_url() - scratch = args.scratch or _default_claim_scratch(repo, worker_id) - return ClaimBoard(repo, worker_id, scratch) + identity = identity or _resolve_claim_board_identity(args, require_identity=require_identity) + return ClaimBoard( + identity.repo, + worker_id, + identity.scratch, + session_id=identity.session_id, + expected_object_format=args.object_format, + expected_repo_identity=identity.repo_identity, + expected_scratch_identity=identity.scratch_identity, + ) + + +def _resolve_claim_target( + args: argparse.Namespace, +) -> _ResolvedClaimTarget: + article_target = args.node_id + resource = args.resource + if article_target and resource: + raise ValueError("article target and --resource are mutually exclusive") + if resource: + if ARTICLE_ID_PATTERN.fullmatch(resource): + raise ValueError("resource names must not use the reserved article_id format") + identity = _resolve_claim_board_identity(args) + return _ResolvedClaimTarget( + resource_claim_key(resource), + resource, + (author_claim_key(resource),), + (), + identity, + ) + if not article_target: + raise ValueError("an article target or --resource is required") + + try: + with bind_runtime_paths(args.blueprint, project_id=args.project) as paths: + paths.require_strong_binding(operation="claim mutation") + blueprint = paths.blueprint_dir + project_pin = _PinnedDirectory.capture(paths.project_root, label="claim project") + blueprint_pin = _PinnedDirectory.capture(blueprint, label="claim blueprint") + graph = load_bound_graph(paths) + matches = [ + node + for node in graph.nodes.values() + if article_target == node.id or article_target == node.article_id + ] + if not matches: + if article_target == "lake-build": + raise ValueError( + f"article target {article_target!r} does not exist in {blueprint}; " + "use --resource lake-build for the shared build lock" + ) + raise ValueError( + f"article target {article_target!r} does not exist in {blueprint}" + ) + if len(matches) != 1: + matching_paths = ", ".join(sorted(node.id for node in matches)) + raise ValueError( + f"article target {article_target!r} is ambiguous: {matching_paths}" + ) + node = matches[0] + if node.article_id is None: + raise ValueError( + f"article {node.id!r} has no durable article_id; " + f"run 'autoform migrate article-ids {blueprint}' and add the proposed ID" + ) + if paths.workspace_project_id is None: + key = author_claim_key(node.article_id) + compatibility_keys = (author_claim_key(node.id),) + canonical_keys = tuple( + author_claim_key(candidate.article_id) + for candidate in graph.nodes.values() + if candidate.article_id is not None + ) + else: + key = workspace_author_claim_key(paths.workspace_project_id, node.article_id) + compatibility_keys = ( + author_claim_key(node.article_id), + author_claim_key(node.id), + ) + canonical_keys = tuple( + workspace_author_claim_key( + paths.workspace_project_id, + candidate.article_id, + ) + for candidate in graph.nodes.values() + if candidate.article_id is not None + ) + identity = _resolve_claim_board_identity(args, context=paths.project_root) + project_pin.verify(label="claim project") + blueprint_pin.verify(label="claim blueprint") + paths.verify() + return _ResolvedClaimTarget( + key, + node.id, + compatibility_keys, + canonical_keys, + identity, + ) + except RuntimeProjectionError as exc: + raise ValueError(str(exc)) from exc + except GraphValidationError as exc: + raise ValueError("; ".join(exc.issues)) from exc -def _origin_url() -> str: +def _origin_url(project_or_blueprint: str | Path = ".") -> str: + target = Path(project_or_blueprint).expanduser().resolve() try: result = subprocess.run( - ["git", "remote", "get-url", "origin"], + [ + "git", + "-C", + str(target), + "config", + "--local", + "--no-includes", + "--get", + "remote.origin.url", + ], capture_output=True, text=True, check=True, timeout=10, + env=_claim_git_environment(), ) except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: raise ValueError("--repo is required outside a Git checkout with an origin remote") from exc - return result.stdout.strip() + origin = result.stdout.strip() + if not claim_repository_is_remote(origin): + origin_path = Path(origin).expanduser() + if not origin_path.is_absolute(): + try: + root_result = subprocess.run( + ["git", "-C", str(target), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + timeout=10, + env=_claim_git_environment(), + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + raise ValueError("could not resolve the relative origin repository") from exc + origin_path = Path(root_result.stdout.strip()) / origin_path + return str(origin_path.resolve()) + return origin + + +def _worktree_claim_session_id(project_or_blueprint: str | Path = ".") -> str: + target = Path(project_or_blueprint).expanduser().resolve() + try: + result = subprocess.run( + [ + "git", + "-C", + str(target), + "rev-parse", + "--show-toplevel", + "--absolute-git-dir", + ], + capture_output=True, + text=True, + check=True, + timeout=10, + env=_claim_git_environment(), + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + raise ValueError( + "--session-id or AUTOFORM_CLAIM_SESSION_ID is required outside a Git worktree" + ) from exc + lines = result.stdout.splitlines() + if len(lines) != 2: + raise ValueError("could not determine a stable Git worktree identity") + root = Path(lines[0]).resolve() + git_dir = Path(lines[1]).resolve() + try: + root_stat = root.stat(follow_symlinks=False) + git_dir_stat = git_dir.stat(follow_symlinks=False) + except OSError as exc: + raise ValueError("could not inspect the Git worktree identity") from exc + token = _worktree_claim_token(git_dir) + identity = ( + f"{socket.gethostname()}\0{token}\0{root_stat.st_dev}:{root_stat.st_ino}" + f"\0{git_dir_stat.st_dev}:{git_dir_stat.st_ino}" + ) + digest = hashlib.sha256(identity.encode()).hexdigest() + return f"worktree-{digest}" + + +def _worktree_claim_token(git_dir: Path) -> str: + token_path = git_dir / "autoform-claim-session" + stored_token = _read_worktree_claim_token(token_path) + if stored_token is not None: + return stored_token + + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + token = secrets.token_hex(32) + temporary_path = git_dir / f".autoform-claim-session-{secrets.token_hex(16)}" + try: + descriptor = os.open(temporary_path, flags, 0o600) + except OSError as exc: + raise ValueError("could not create the Git worktree claim identity") from exc + try: + try: + os.write(descriptor, f"{token}\n".encode()) + os.fsync(descriptor) + finally: + os.close(descriptor) + try: + os.link(temporary_path, token_path, follow_symlinks=False) + except FileExistsError: + pass + except OSError as exc: + raise ValueError("could not install the Git worktree claim identity") from exc + finally: + try: + temporary_path.unlink() + except FileNotFoundError: + pass + + stored_token = _read_worktree_claim_token(token_path) + if stored_token is None: + raise ValueError("could not install the Git worktree claim identity") + return stored_token -def _default_claim_scratch(repo: str, worker_id: str) -> Path: +def _read_worktree_claim_token(token_path: Path) -> str | None: + read_flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + read_flags |= os.O_NOFOLLOW + if hasattr(os, "O_NONBLOCK"): + read_flags |= os.O_NONBLOCK + try: + descriptor = os.open(token_path, read_flags) + except FileNotFoundError: + return None + except OSError as exc: + raise ValueError("could not read the Git worktree claim identity") from exc + try: + try: + token_info = os.fstat(descriptor) + path_info = token_path.stat(follow_symlinks=False) + raw_token = os.read(descriptor, 256) + finally: + os.close(descriptor) + except OSError as exc: + raise ValueError("could not read the Git worktree claim identity") from exc + if not stat.S_ISREG(token_info.st_mode): + raise ValueError("Git worktree claim identity must be a regular file") + if (token_info.st_dev, token_info.st_ino) != (path_info.st_dev, path_info.st_ino): + raise ValueError("Git worktree claim identity changed while it was read") + try: + stored_token = raw_token.decode("ascii") + except UnicodeDecodeError as exc: + raise ValueError("Git worktree claim identity is malformed") from exc + if len(raw_token) != token_info.st_size or not re.fullmatch( + r"[0-9a-f]{64}\n?", + stored_token, + ): + raise ValueError("Git worktree claim identity is malformed") + return stored_token.rstrip("\n") + + +def _default_claim_scratch(repo: str, session_id: str) -> Path: cache = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) - identity = hashlib.sha256(f"{repo}\0{worker_id}\0{socket.gethostname()}".encode()).hexdigest()[:24] + identity = hashlib.sha256(f"{repo}\0{session_id}\0{socket.gethostname()}".encode()).hexdigest()[:24] return cache / "autoform" / "claims" / identity def _render(args: argparse.Namespace) -> int: try: - report = render_site( - args.blueprint_dir, - args.output, - lean_root=args.lean_root, - repository_url=args.repository_url, - ref=args.ref, - ) - except (GraphValidationError, PublicationError) as exc: + with bind_runtime_paths(args.blueprint_dir, project_id=args.project) as paths: + blueprint_dir = paths.blueprint_dir + output = args.output + if output is None: + output = ( + str(Path("site-src") / paths.workspace_project_id) + if paths.workspace_project_id is not None + else "site-src" + ) + report = render_site( + blueprint_dir, + output, + lean_root=args.lean_root, + repository_url=args.repository_url, + ref=args.ref, + _expected_blueprint_identity=paths.blueprint_identity, + _expected_roadmap_identity=paths.roadmap_identity, + ) + paths.verify() + except (GraphValidationError, PublicationError, RuntimeProjectionError) as exc: for issue in exc.issues: print(f"error: {issue}") return 1 @@ -339,6 +1153,8 @@ def _render(args: argparse.Namespace) -> int: print(f"{report.output_dir}: {report.pages} pages, {report.nodes} nodes, {report.linked} code links") for issue in report.unresolved: print(f"warning: declaration not found in the Lean sources: {issue}") + for issue in report.warnings: + print(f"warning: {issue}") if report.unresolved and args.require_declarations: return 1 return 0 diff --git a/autoform_cli/_git_fd_transport.py b/autoform_cli/_git_fd_transport.py new file mode 100644 index 00000000..ddfc6e90 --- /dev/null +++ b/autoform_cli/_git_fd_transport.py @@ -0,0 +1,22 @@ +"""Run one side of Git's local smart transport from a pinned directory FD.""" + +from __future__ import annotations + +import os +import sys + + +def main() -> None: + if len(sys.argv) < 3 or sys.argv[1] not in {"upload", "receive"}: + raise SystemExit("usage: _git_fd_transport.py {upload|receive} DIRECTORY_FD") + mode = sys.argv[1] + try: + directory_fd = int(sys.argv[2]) + os.fchdir(directory_fd) + except (OSError, ValueError) as exc: + raise SystemExit(f"could not enter pinned Git repository: {exc}") from exc + os.execvp("git", ["git", f"{mode}-pack", "."]) + + +if __name__ == "__main__": + main() diff --git a/autoform_cli/_tree_snapshot.py b/autoform_cli/_tree_snapshot.py new file mode 100644 index 00000000..0ca76847 --- /dev/null +++ b/autoform_cli/_tree_snapshot.py @@ -0,0 +1,725 @@ +"""Capture one immutable directory-tree generation for internal consumers.""" + +from __future__ import annotations + +import hashlib +import os +import stat +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Callable, Iterator + +from . import workspace as workspace_module +from .workspace import _WorkspaceRootBinding, _open_workspace_root +from .workspace_manifest import WorkspaceError + +_DIRECTORY_FLAGS = ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) +) +_FILE_FLAGS = ( + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NONBLOCK", 0) +) + + +class TreeSnapshotError(ValueError): + """A directory tree could not be captured as one stable generation.""" + + +@dataclass(frozen=True, slots=True) +class TreeSelection: + """Select which paths are descended into and captured as bytes.""" + + include: Callable[[PurePosixPath, int], bool] + descend: Callable[[PurePosixPath], bool] + placeholder: Callable[[PurePosixPath, int], bool] = lambda _path, _mode: False + byte_limit: Callable[[PurePosixPath], int | None] = lambda _path: None + record_omitted: bool = True + + +ALL_ENTRIES = TreeSelection( + include=lambda _path, _mode: True, + descend=lambda _path: True, +) + + +@dataclass(frozen=True, slots=True) +class TreeSnapshot: + """Immutable names, types, and bytes captured below one directory root.""" + + root_identity: tuple[int, int] + directories: tuple[str, ...] + files: tuple[tuple[str, bytes], ...] + symlinks: tuple[tuple[str, str], ...] + special: tuple[tuple[str, int], ...] + placeholders: tuple[str, ...] + omitted: tuple[tuple[str, str], ...] + identities: tuple[tuple[str, tuple[int, ...]], ...] + + @property + def revision(self) -> str: + """Return a framed digest of every captured entry and regular-file byte.""" + + digest = hashlib.sha256(b"autoform-directory-snapshot/v1\0") + for relative in self.directories: + _update_digest(digest, b"directory", relative, b"") + for relative, data in self.files: + _update_digest(digest, b"file", relative, data) + for relative, target in self.symlinks: + _update_digest(digest, b"symlink", relative, os.fsencode(target)) + for relative, mode in self.special: + _update_digest(digest, b"special", relative, str(mode).encode("ascii")) + for relative in self.placeholders: + _update_digest(digest, b"placeholder", relative, b"") + for relative, kind in self.omitted: + _update_digest(digest, b"omitted", relative, kind.encode("ascii")) + return digest.hexdigest() + + @property + def generation_revision(self) -> str: + """Return a digest that also distinguishes filesystem generations.""" + + digest = hashlib.sha256(self.revision.encode("ascii")) + directory_paths = set(self.directories) + for relative, identity in self.identities: + stable_identity = identity[:3] if relative in directory_paths else identity + encoded = ",".join(str(field) for field in stable_identity).encode("ascii") + _update_digest(digest, b"identity", relative, encoded) + return digest.hexdigest() + + def materialize(self, destination: Path) -> None: + """Write captured regular files below a fresh private directory.""" + + issues = self.unsupported_entries() + if issues: + relative, reason = issues[0] + raise TreeSnapshotError(f"{relative}: {reason}") + self.materialize_regular_files(destination) + + def materialize_regular_files(self, destination: Path) -> None: + """Materialize safe content after a caller has recorded invalid entries.""" + + destination.mkdir(parents=True, exist_ok=False) + for relative in self.directories: + if relative: + destination.joinpath(*PurePosixPath(relative).parts).mkdir() + for relative, data in self.files: + target = destination.joinpath(*PurePosixPath(relative).parts) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + for relative in self.placeholders: + target = destination.joinpath(*PurePosixPath(relative).parts) + target.parent.mkdir(parents=True, exist_ok=True) + target.touch() + + def unsupported_entries(self) -> tuple[tuple[str, str], ...]: + """Return path-specific reasons for entries that cannot be copied safely.""" + + issues = [ + (relative, "symbolic links are not supported") + for relative, _target in self.symlinks + ] + issues.extend( + ( + relative, + f"{_special_file_kind(mode)} is not a regular file or directory", + ) + for relative, mode in self.special + ) + return tuple(sorted(issues)) + + +@dataclass(frozen=True, slots=True) +class _DirectoryRecord: + relative: str + identity: tuple[int, ...] + names: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _EntryRecord: + relative: str + identity: tuple[int, ...] + ignored: bool = False + + +class BoundDirectoryTree: + """One retained directory generation that can be recaptured and compared.""" + + def __init__( + self, + root: Path, + *, + expected_identity: tuple[int, int] | None = None, + expected_children: dict[str, tuple[int, int]] | None = None, + selection: TreeSelection = ALL_ENTRIES, + ) -> None: + self.root = root + self.expected_identity = expected_identity + self.expected_children = expected_children or {} + self.selection = selection + self._binding: _WorkspaceRootBinding | None = None + self._portable_identity: tuple[int, int] | None = None + self._closed = False + if workspace_module._DIRECTORY_BINDING_SUPPORTED: + try: + binding = _open_workspace_root(root) + except WorkspaceError as error: + raise TreeSnapshotError("directory tree cannot be inspected safely") from error + if expected_identity is not None and binding.identity != expected_identity: + binding.close() + raise TreeSnapshotError("directory tree changed before it was captured") + self._binding = binding + try: + self._verify_expected_children(binding.descriptor) + except BaseException: + binding.close() + self._binding = None + raise + return + try: + metadata = root.stat(follow_symlinks=False) + except OSError as error: + raise TreeSnapshotError("directory tree cannot be inspected safely") from error + identity = (metadata.st_dev, metadata.st_ino) + if not stat.S_ISDIR(metadata.st_mode) or ( + expected_identity is not None and identity != expected_identity + ): + raise TreeSnapshotError("directory tree changed before it was captured") + self._portable_identity = identity + self._verify_expected_children(None) + + @property + def identity(self) -> tuple[int, int]: + if self._closed: + raise TreeSnapshotError("directory tree binding is closed") + if self._binding is not None: + return self._binding.identity + assert self._portable_identity is not None + return self._portable_identity + + def _verify_expected_children(self, descriptor: int | None) -> None: + try: + for name, expected in self.expected_children.items(): + if Path(name).name != name or name in {"", ".", ".."}: + raise TreeSnapshotError("invalid bound child name") + metadata = ( + os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if descriptor is not None + else (self.root / name).stat(follow_symlinks=False) + ) + if not stat.S_ISDIR(metadata.st_mode) or ( + metadata.st_dev, + metadata.st_ino, + ) != expected: + raise TreeSnapshotError("directory tree changed before it was captured") + except OSError as error: + raise TreeSnapshotError("directory tree changed before it was captured") from error + + def verify(self) -> None: + """Verify the retained generation is still selected by its public path.""" + + if self._closed: + raise TreeSnapshotError("directory tree binding is closed") + try: + if self._binding is not None: + self._binding.verify() + self._verify_expected_children(self._binding.descriptor) + return + metadata = self.root.stat(follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode) or ( + metadata.st_dev, + metadata.st_ino, + ) != self.identity: + raise TreeSnapshotError("directory tree changed while it was in use") + self._verify_expected_children(None) + except (OSError, WorkspaceError) as error: + raise TreeSnapshotError("directory tree changed while it was in use") from error + + def capture(self) -> TreeSnapshot: + """Capture one stable tree through the retained directory generation.""" + + self.verify() + try: + if self._binding is not None: + snapshot = capture_directory_descriptor( + self._binding.descriptor, + expected_identity=self.identity, + selection=self.selection, + ) + else: + first = _capture_portable(self.root, selection=self.selection) + _tree_snapshot_checkpoint("between-portable-captures", "") + snapshot = _capture_portable(self.root, selection=self.selection) + if first != snapshot: + raise TreeSnapshotError("directory tree changed while it was captured") + except (OSError, RuntimeError) as error: + raise TreeSnapshotError("directory tree changed while it was captured") from error + self.verify() + return snapshot + + def close(self) -> None: + if self._closed: + return + if self._binding is not None: + self._binding.close() + self._binding = None + self._closed = True + + +@contextmanager +def bind_directory_tree( + root: Path, + *, + expected_identity: tuple[int, int] | None = None, + expected_children: dict[str, tuple[int, int]] | None = None, + selection: TreeSelection = ALL_ENTRIES, +) -> Iterator[BoundDirectoryTree]: + """Retain *root* while callers capture and verify its content generation.""" + + bound = BoundDirectoryTree( + root, + expected_identity=expected_identity, + expected_children=expected_children, + selection=selection, + ) + try: + yield bound + finally: + bound.close() + + +def capture_directory_descriptor( + descriptor: int, + *, + expected_identity: tuple[int, int] | None = None, + selection: TreeSelection = ALL_ENTRIES, +) -> TreeSnapshot: + """Capture a tree below an already retained directory descriptor.""" + + try: + root = os.fstat(descriptor) + except OSError as error: + raise TreeSnapshotError("directory tree cannot be inspected safely") from error + if not stat.S_ISDIR(root.st_mode) or ( + expected_identity is not None + and (root.st_dev, root.st_ino) != expected_identity + ): + raise TreeSnapshotError("directory tree changed before it was captured") + directories: list[_DirectoryRecord] = [] + entries: list[_EntryRecord] = [] + files: list[tuple[str, bytes]] = [] + symlinks: list[tuple[str, str]] = [] + special: list[tuple[str, int]] = [] + placeholders: list[str] = [] + omitted: list[tuple[str, str]] = [] + try: + _scan_directory( + descriptor, + relative="", + identity=_stat_signature(root), + directories=directories, + entries=entries, + files=files, + symlinks=symlinks, + special=special, + placeholders=placeholders, + omitted=omitted, + selection=selection, + ) + _tree_snapshot_checkpoint("before-final-verification", "") + _verify_snapshot(descriptor, directories, entries) + except (OSError, _TreeChanged) as error: + raise TreeSnapshotError("directory tree changed while it was captured") from error + return TreeSnapshot( + root_identity=(root.st_dev, root.st_ino), + directories=tuple(record.relative for record in directories), + files=tuple(sorted(files)), + symlinks=tuple(sorted(symlinks)), + special=tuple(sorted(special)), + placeholders=tuple(sorted(placeholders)), + omitted=tuple(sorted(omitted)), + identities=_included_identities(directories, entries), + ) + + +class _TreeChanged(Exception): + """The retained directory did not remain one stable generation.""" + + +def _tree_snapshot_checkpoint(_event: str, _relative: str) -> None: + """Deterministic concurrency boundary used by adversarial tests.""" + + +def _stat_signature(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _valid_name(name: object) -> bool: + return ( + isinstance(name, str) + and bool(name) + and name not in {".", ".."} + and "/" not in name + and "\\" not in name + ) + + +def _scan_directory( + descriptor: int, + *, + relative: str, + identity: tuple[int, ...], + directories: list[_DirectoryRecord], + entries: list[_EntryRecord], + files: list[tuple[str, bytes]], + symlinks: list[tuple[str, str]], + special: list[tuple[str, int]], + placeholders: list[str], + omitted: list[tuple[str, str]], + selection: TreeSelection, +) -> None: + names = tuple(sorted(os.listdir(descriptor))) + if any(not _valid_name(name) for name in names): + raise _TreeChanged + directories.append(_DirectoryRecord(relative, identity, names)) + _tree_snapshot_checkpoint("after-directory-list", relative) + for name in names: + child_relative = f"{relative}/{name}" if relative else name + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + child_identity = _stat_signature(metadata) + relative_path = PurePosixPath(child_relative) + if stat.S_ISDIR(metadata.st_mode): + if not selection.descend(relative_path): + entries.append(_EntryRecord(child_relative, child_identity, ignored=True)) + if selection.record_omitted: + omitted.append((child_relative, "directory")) + continue + child_descriptor: int | None = None + try: + child_descriptor = os.open(name, _DIRECTORY_FLAGS, dir_fd=descriptor) + opened = os.fstat(child_descriptor) + if _stat_signature(opened) != child_identity: + raise _TreeChanged + _scan_directory( + child_descriptor, + relative=child_relative, + identity=child_identity, + directories=directories, + entries=entries, + files=files, + symlinks=symlinks, + special=special, + placeholders=placeholders, + omitted=omitted, + selection=selection, + ) + if _stat_signature( + os.stat(name, dir_fd=descriptor, follow_symlinks=False) + ) != child_identity: + raise _TreeChanged + finally: + if child_descriptor is not None: + os.close(child_descriptor) + continue + if not selection.include(relative_path, metadata.st_mode): + if stat.S_ISREG(metadata.st_mode) and selection.placeholder( + relative_path, + metadata.st_mode, + ): + entries.append(_EntryRecord(child_relative, child_identity)) + placeholders.append(child_relative) + continue + entries.append(_EntryRecord(child_relative, child_identity, ignored=True)) + kind = ( + "file" + if stat.S_ISREG(metadata.st_mode) + else "symlink" + if stat.S_ISLNK(metadata.st_mode) + else "special" + ) + if selection.record_omitted: + omitted.append((child_relative, kind)) + continue + entries.append(_EntryRecord(child_relative, child_identity)) + if stat.S_ISREG(metadata.st_mode): + files.append( + ( + child_relative, + _read_file( + descriptor, + name, + child_identity, + max_bytes=selection.byte_limit(relative_path), + ), + ) + ) + elif stat.S_ISLNK(metadata.st_mode): + target = os.readlink(name, dir_fd=descriptor) + if _stat_signature( + os.stat(name, dir_fd=descriptor, follow_symlinks=False) + ) != child_identity: + raise _TreeChanged + symlinks.append((child_relative, target)) + else: + special.append((child_relative, stat.S_IFMT(metadata.st_mode))) + if ( + _stat_signature(os.fstat(descriptor)) != identity + or tuple(sorted(os.listdir(descriptor))) != names + ): + raise _TreeChanged + + +def _read_file( + parent_descriptor: int, + name: str, + expected: tuple[int, ...], + *, + max_bytes: int | None = None, +) -> bytes: + descriptor: int | None = None + try: + descriptor = os.open(name, _FILE_FLAGS, dir_fd=parent_descriptor) + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _stat_signature(opened) != expected: + raise _TreeChanged + stream = os.fdopen(descriptor, "rb", buffering=0, closefd=False) + try: + data = stream.read() if max_bytes is None else _read_prefix(stream, max_bytes + 1) + finally: + stream.close() + if ( + _stat_signature(os.fstat(descriptor)) != expected + or _stat_signature( + os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + ) + != expected + ): + raise _TreeChanged + return data + finally: + if descriptor is not None: + os.close(descriptor) + + +def _read_prefix(stream, length: int) -> bytes: + """Read through *length* bytes or EOF despite legal short reads.""" + + chunks: list[bytes] = [] + remaining = length + while remaining: + chunk = stream.read(remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def _verify_snapshot( + root_descriptor: int, + directories: list[_DirectoryRecord], + entries: list[_EntryRecord], +) -> None: + expected_directories = {record.relative: record for record in directories} + expected_entries = {record.relative: record for record in entries} + visited_directories: set[str] = set() + visited_entries: set[str] = set() + + def verify_directory(descriptor: int, relative: str) -> None: + expected = expected_directories.get(relative) + if expected is None: + raise _TreeChanged + visited_directories.add(relative) + names = tuple(sorted(os.listdir(descriptor))) + if _stat_signature(os.fstat(descriptor)) != expected.identity or names != expected.names: + raise _TreeChanged + for name in names: + child_relative = f"{relative}/{name}" if relative else name + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + directory = expected_directories.get(child_relative) + if directory is None: + entry = expected_entries.get(child_relative) + if entry is None or ( + not entry.ignored and _stat_signature(metadata) != entry.identity + ): + raise _TreeChanged + if entry.ignored and _entry_kind(metadata.st_mode) != _entry_kind( + entry.identity[2] + ): + raise _TreeChanged + visited_entries.add(child_relative) + continue + if not stat.S_ISDIR(metadata.st_mode) or _stat_signature(metadata) != directory.identity: + raise _TreeChanged + child_descriptor: int | None = None + try: + child_descriptor = os.open(name, _DIRECTORY_FLAGS, dir_fd=descriptor) + if _stat_signature(os.fstat(child_descriptor)) != directory.identity: + raise _TreeChanged + verify_directory(child_descriptor, child_relative) + if _stat_signature( + os.stat(name, dir_fd=descriptor, follow_symlinks=False) + ) != directory.identity: + raise _TreeChanged + finally: + if child_descriptor is not None: + os.close(child_descriptor) + if ( + _stat_signature(os.fstat(descriptor)) != expected.identity + or tuple(sorted(os.listdir(descriptor))) != expected.names + ): + raise _TreeChanged + + verify_directory(root_descriptor, "") + if visited_directories != set(expected_directories) or visited_entries != set( + expected_entries + ): + raise _TreeChanged + + +def _capture_portable( + root: Path, + *, + selection: TreeSelection, +) -> TreeSnapshot: + """Best-effort double-captured fallback for read-only non-POSIX clients.""" + + root_before = root.stat(follow_symlinks=False) + if not stat.S_ISDIR(root_before.st_mode) or _is_reparse_point(root_before): + raise TreeSnapshotError("directory tree cannot be inspected safely") + directories: list[str] = [""] + files: list[tuple[str, bytes]] = [] + symlinks: list[tuple[str, str]] = [] + special: list[tuple[str, int]] = [] + placeholders: list[str] = [] + omitted: list[tuple[str, str]] = [] + identities: list[tuple[str, tuple[int, ...]]] = [("", _stat_signature(root_before))] + + def visit(directory: Path, relative: str) -> None: + entries = sorted(os.scandir(directory), key=lambda entry: entry.name) + for entry in entries: + if not _valid_name(entry.name): + raise TreeSnapshotError("directory tree cannot be inspected safely") + child_relative = f"{relative}/{entry.name}" if relative else entry.name + metadata = entry.stat(follow_symlinks=False) + path = directory / entry.name + relative_path = PurePosixPath(child_relative) + if stat.S_ISDIR(metadata.st_mode) and not _is_reparse_point(metadata): + if not selection.descend(relative_path): + if selection.record_omitted: + omitted.append((child_relative, "directory")) + continue + directories.append(child_relative) + identities.append((child_relative, _stat_signature(metadata))) + visit(path, child_relative) + elif stat.S_ISREG(metadata.st_mode) and not _is_reparse_point(metadata): + if not selection.include(relative_path, metadata.st_mode): + if selection.placeholder(relative_path, metadata.st_mode): + placeholders.append(child_relative) + identities.append((child_relative, _stat_signature(metadata))) + else: + if selection.record_omitted: + omitted.append((child_relative, "file")) + continue + before = _stat_signature(metadata) + with path.open("rb") as stream: + opened = os.fstat(stream.fileno()) + limit = selection.byte_limit(relative_path) + data = stream.read() if limit is None else _read_prefix(stream, limit + 1) + after = os.fstat(stream.fileno()) + final = path.stat(follow_symlinks=False) + if not (before == _stat_signature(opened) == _stat_signature(after) == _stat_signature(final)): + raise TreeSnapshotError("directory tree changed while it was captured") + files.append((child_relative, data)) + identities.append((child_relative, before)) + elif stat.S_ISLNK(metadata.st_mode) or _is_reparse_point(metadata): + if selection.include(relative_path, metadata.st_mode): + symlinks.append((child_relative, os.readlink(path))) + identities.append((child_relative, _stat_signature(metadata))) + else: + if selection.record_omitted: + omitted.append((child_relative, "symlink")) + else: + if selection.include(relative_path, metadata.st_mode): + special.append((child_relative, stat.S_IFMT(metadata.st_mode))) + identities.append((child_relative, _stat_signature(metadata))) + else: + if selection.record_omitted: + omitted.append((child_relative, "special")) + + visit(root, "") + root_after = root.stat(follow_symlinks=False) + if _stat_signature(root_before) != _stat_signature(root_after): + raise TreeSnapshotError("directory tree changed while it was captured") + return TreeSnapshot( + root_identity=(root_after.st_dev, root_after.st_ino), + directories=tuple(sorted(directories)), + files=tuple(sorted(files)), + symlinks=tuple(sorted(symlinks)), + special=tuple(sorted(special)), + placeholders=tuple(sorted(placeholders)), + omitted=tuple(sorted(omitted)), + identities=tuple(sorted(identities)), + ) + + +def _is_reparse_point(metadata: os.stat_result) -> bool: + attributes = getattr(metadata, "st_file_attributes", 0) + marker = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(attributes & marker) + + +def _special_file_kind(mode: int) -> str: + if stat.S_ISFIFO(mode): + return "named pipe" + if stat.S_ISSOCK(mode): + return "socket" + if stat.S_ISBLK(mode): + return "block device" + if stat.S_ISCHR(mode): + return "character device" + return "special filesystem entry" + + +def _entry_kind(mode: int) -> int: + return stat.S_IFMT(mode) + + +def _update_digest(digest, kind: bytes, relative: str, data: bytes) -> None: + path = os.fsencode(relative) + for field in (kind, path, data): + digest.update(len(field).to_bytes(8, "big")) + digest.update(field) + + +def _included_identities( + directories: list[_DirectoryRecord], + entries: list[_EntryRecord], +) -> tuple[tuple[str, tuple[int, ...]], ...]: + return tuple( + sorted( + [ + *((record.relative, record.identity) for record in directories), + *( + (record.relative, record.identity) + for record in entries + if not record.ignored + ), + ] + ) + ) diff --git a/autoform_cli/article_identity.py b/autoform_cli/article_identity.py index a469d102..6fb23ff2 100644 --- a/autoform_cli/article_identity.py +++ b/autoform_cli/article_identity.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from pathlib import Path -from .graph import GraphValidationError, load_graph +from .graph import Graph, GraphValidationError, load_graph IDENTITY_PLAN_SCHEMA = "autoform-article-id-plan/v1" @@ -61,10 +61,14 @@ def to_json(self) -> str: return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) -def plan_article_ids(blueprint_dir: str | Path) -> ArticleIdentityPlan: +def plan_article_ids( + blueprint_dir: str | Path, + *, + _graph: Graph | None = None, +) -> ArticleIdentityPlan: """Validate a blueprint and propose IDs for articles that do not have one.""" - graph = load_graph(blueprint_dir) + graph = _graph if _graph is not None else load_graph(blueprint_dir) entries = [] owners: dict[str, str] = {} for node in sorted(graph.nodes.values(), key=lambda candidate: candidate.id): diff --git a/autoform_cli/audit.py b/autoform_cli/audit.py index 35e427cc..781164ff 100644 --- a/autoform_cli/audit.py +++ b/autoform_cli/audit.py @@ -8,15 +8,29 @@ from __future__ import annotations import json +import stat import statistics +import tempfile from bisect import bisect_right -from dataclasses import asdict, dataclass -from pathlib import Path +from dataclasses import asdict, dataclass, replace +from pathlib import Path, PurePosixPath from . import status +from ._tree_snapshot import ( + TreeSelection, + TreeSnapshot, + TreeSnapshotError, + bind_directory_tree, +) from .coverage import CoverageSummary, load_coverage from .graph import Graph, GraphValidationError, Node, load_graph -from .lean import SourceIndex, declaration_names, index_project +from .lean import ( + SourceIndex, + declaration_keywords, + declaration_names, + index_project, + mathlib_module_name, +) from .markdown import FENCE as _FENCE from .markdown import frontmatter_end as _frontmatter_end from .markdown import HEADING as _HEADING @@ -35,22 +49,26 @@ _NODE_SIZE_FLOOR = 200 _NODE_SIZE_MULTIPLE = 4 -_DECLARATION_KEYWORDS = { - "abbrev": frozenset({"abbrev"}), - "axiom": frozenset({"axiom"}), - "class": frozenset({"class"}), - "corollary": frozenset({"lemma", "theorem"}), - "def": frozenset({"def"}), - "definition": frozenset({"def"}), - "inductive": frozenset({"inductive"}), - "instance": frozenset({"instance"}), - "lemma": frozenset({"lemma", "theorem"}), - "opaque": frozenset({"opaque"}), - "proposition": frozenset({"lemma", "theorem"}), - "structure": frozenset({"structure"}), - "theorem": frozenset({"lemma", "theorem"}), -} +def _visible_snapshot_path(relative: PurePosixPath) -> bool: + return not any(part.startswith(".") for part in relative.parts) + + +def _audit_snapshot_includes(relative: PurePosixPath, mode: int) -> bool: + return _visible_snapshot_path(relative) and ( + not stat.S_ISREG(mode) + or relative.suffix.casefold() == ".md" + or relative.parts[:1] == ("sources",) + ) + + +_AUDIT_SNAPSHOT_SELECTION = TreeSelection( + include=_audit_snapshot_includes, + descend=_visible_snapshot_path, + placeholder=lambda relative, mode: ( + _visible_snapshot_path(relative) and stat.S_ISREG(mode) + ), +) @dataclass(frozen=True, order=True, slots=True) class AuditFinding: @@ -93,6 +111,8 @@ def audit_blueprint( blueprint_dir: str | Path, *, lean_root: str | Path | None = None, + _expected_blueprint_identity: tuple[int, int] | None = None, + _expected_roadmap_identity: tuple[int, int] | None = None, ) -> AuditResult: """Audit *blueprint_dir* using only local, committed-style source files. @@ -100,13 +120,155 @@ def audit_blueprint( Semantic checks run only after :func:`load_graph` has produced a valid graph. """ + _graph, result = load_audit_graph( + blueprint_dir, + lean_root=lean_root, + _expected_blueprint_identity=_expected_blueprint_identity, + _expected_roadmap_identity=_expected_roadmap_identity, + ) + return result + + +def load_audit_graph( + blueprint_dir: str | Path, + *, + lean_root: str | Path | None = None, + lean_index: SourceIndex | None = None, + _expected_blueprint_identity: tuple[int, int] | None = None, + _expected_roadmap_identity: tuple[int, int] | None = None, +) -> tuple[Graph | None, AuditResult]: + """Return a graph and audit derived from one immutable blueprint capture.""" + blueprint = Path(blueprint_dir).expanduser().resolve() + if not blueprint.is_dir(): + return _audit_snapshot_graph( + blueprint, + lean_root=lean_root, + lean_index=lean_index, + expected_blueprint_identity=_expected_blueprint_identity, + expected_roadmap_identity=_expected_roadmap_identity, + ) + expected_children = ( + {"roadmap": _expected_roadmap_identity} + if _expected_roadmap_identity is not None + else None + ) + try: + with bind_directory_tree( + blueprint, + expected_identity=_expected_blueprint_identity, + expected_children=expected_children, + selection=_AUDIT_SNAPSHOT_SELECTION, + ) as bound: + snapshot = bound.capture() + entry_findings = _snapshot_entry_findings(snapshot) + with tempfile.TemporaryDirectory(prefix="autoform-audit-") as temporary: + snapshot_root = Path(temporary) / "blueprint" + snapshot.materialize_regular_files(snapshot_root) + graph, result = _audit_snapshot_graph( + snapshot_root, + lean_root=lean_root, + lean_index=lean_index, + ) + if graph is not None: + graph = _rebase_captured_graph(graph, blueprint) + bound.verify() + graph_is_invalid = any( + finding.code == "invalid-graph" for finding in entry_findings + ) + return ( + None if graph_is_invalid else graph, + _result( + [*result.findings, *entry_findings], + coverage=result.coverage, + ), + ) + except TreeSnapshotError as error: + return None, _result( + [AuditFinding(".", "invalid-graph", str(error))] + ) + + +def _rebase_captured_graph(graph: Graph, blueprint: Path) -> Graph: + """Return a captured graph whose public paths name the requested blueprint.""" + + nodes = { + node_id: replace( + node, + path=blueprint / node.path.relative_to(graph.blueprint_dir), + ) + for node_id, node in graph.nodes.items() + } + rebased = Graph(blueprint_dir=blueprint, nodes=nodes) + object.__setattr__( + rebased, + "_source_bytes", + { + node_id: content + for node_id in graph.nodes + if (content := graph.source_bytes(node_id)) is not None + }, + ) + return rebased + + +def _snapshot_entry_findings(snapshot: TreeSnapshot) -> list[AuditFinding]: + """Preserve unsafe-entry diagnostics that cannot survive materialization.""" + + findings: list[AuditFinding] = [] + for relative, reason in snapshot.unsupported_entries(): + roadmap_entry = relative == "roadmap" or relative.startswith("roadmap/") + findings.append( + AuditFinding( + relative, + "invalid-graph" if roadmap_entry else "unsafe-blueprint-entry", + ( + f"roadmap path is invalid: {reason}" + if roadmap_entry + else f"blueprint path is unsafe: {reason}" + ), + ) + ) + return findings + + +def _audit_snapshot( + blueprint: Path, + *, + lean_root: str | Path | None, + expected_blueprint_identity: tuple[int, int] | None = None, + expected_roadmap_identity: tuple[int, int] | None = None, +) -> AuditResult: + """Audit one immutable, private blueprint snapshot.""" + + return _audit_snapshot_graph( + blueprint, + lean_root=lean_root, + expected_blueprint_identity=expected_blueprint_identity, + expected_roadmap_identity=expected_roadmap_identity, + )[1] + + +def _audit_snapshot_graph( + blueprint: Path, + *, + lean_root: str | Path | None, + lean_index: SourceIndex | None = None, + expected_blueprint_identity: tuple[int, int] | None = None, + expected_roadmap_identity: tuple[int, int] | None = None, +) -> tuple[Graph | None, AuditResult]: + """Audit one immutable tree and retain its parsed graph for sibling checks.""" + if blueprint.is_dir(): coverage, coverage_findings = _coverage_findings(blueprint) else: coverage, coverage_findings = None, [] try: - graph = load_graph(blueprint) + graph = load_graph( + blueprint, + _expected_blueprint_identity=expected_blueprint_identity, + _expected_roadmap_identity=expected_roadmap_identity, + ) except GraphValidationError as error: findings = [ AuditFinding( @@ -116,12 +278,16 @@ def audit_blueprint( ) for issue in error.issues ] - return _result([*findings, *coverage_findings], coverage=coverage) - return audit_graph( + return None, _result([*findings, *coverage_findings], coverage=coverage) + return ( graph, - lean_root=lean_root, - coverage=coverage, - coverage_findings=coverage_findings, + audit_graph( + graph, + lean_root=lean_root, + lean_index=lean_index, + coverage=coverage, + coverage_findings=coverage_findings, + ), ) @@ -129,6 +295,7 @@ def audit_graph( graph: Graph, *, lean_root: str | Path | None = None, + lean_index: SourceIndex | None = None, coverage: CoverageSummary | None = None, coverage_findings: list[AuditFinding] | None = None, ) -> AuditResult: @@ -141,7 +308,7 @@ def audit_graph( node = graph.nodes[node_id] article_path = _relative_path(node.path, graph.blueprint_dir) children = graph.children(node_id) - article = _read_article(node.path) + article = _read_article(node.path, content=graph.source_bytes(node.id)) if node.formalizable: if children: @@ -204,6 +371,22 @@ def audit_graph( "mathlib is true but mathlib_declaration metadata is missing", ) ) + if node.mathlib and not node.mathlib_file: + findings.append( + AuditFinding( + article_path, + "mathlib-without-file", + "mathlib is true but mathlib_file metadata is missing", + ) + ) + elif node.mathlib and mathlib_module_name(node.mathlib_file or "") is None: + findings.append( + AuditFinding( + article_path, + "invalid-mathlib-file", + "mathlib_file must be a canonical Mathlib/**/*.lean source path", + ) + ) formalization_evidence = ( bool(node.lean) @@ -228,8 +411,10 @@ def audit_graph( if coverage_findings is None: coverage, coverage_findings = _coverage_findings(graph.blueprint_dir) findings.extend(coverage_findings) - if lean_root is not None: - findings.extend(_lean_findings(graph, lean_root)) + if lean_index is not None: + findings.extend(_lean_findings(graph, lean_index=lean_index)) + elif lean_root is not None: + findings.extend(_lean_findings(graph, lean_root=lean_root)) return _result(findings, coverage=coverage) @@ -239,11 +424,17 @@ class _ArticleShape: has_depends_section: bool -def _read_article(path: Path) -> _ArticleShape: - try: - text = path.read_text(encoding="utf-8") - except (OSError, UnicodeError): - return _ArticleShape(False, False) +def _read_article(path: Path, *, content: bytes | None = None) -> _ArticleShape: + if content is None: + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + return _ArticleShape(False, False) + else: + try: + text = content.decode("utf-8") + except UnicodeError: + return _ArticleShape(False, False) lines = text.splitlines() start = _frontmatter_end(lines) @@ -302,11 +493,7 @@ def _coverage_findings( findings = [ AuditFinding( "coverage/README.md", - ( - "missing-coverage-contract" - if issue.reason == "coverage contract is missing" - else "invalid-coverage-contract" - ), + issue.code, f"{issue.reason}{f' (line {issue.line})' if issue.line else ''}", ) for issue in issues @@ -351,20 +538,29 @@ def _coverage_findings( return coverage, findings -def _lean_findings(graph: Graph, lean_root: str | Path) -> list[AuditFinding]: - root = Path(lean_root).expanduser().resolve() - if not root.is_dir(): - return [ - AuditFinding( - ".", - "invalid-lean-root", - "Lean root does not exist or is not a directory", - ) - ] - +def _lean_findings( + graph: Graph, + *, + lean_root: str | Path | None = None, + lean_index: SourceIndex | None = None, +) -> list[AuditFinding]: + if lean_index is None: + assert lean_root is not None + root = Path(lean_root).expanduser().resolve() + if not root.is_dir(): + return [ + AuditFinding( + ".", + "invalid-lean-root", + "Lean root does not exist or is not a directory", + ) + ] + try: + lean_index = index_project(root) + except OSError as error: + return [AuditFinding(".", "invalid-lean-root", str(error))] findings: list[AuditFinding] = [] - index = index_project(root) - spans = _source_spans(index) + spans = _source_spans(lean_index) sizes: dict[str, int] = {} for node_id in sorted(graph.nodes): node = graph.nodes[node_id] @@ -382,7 +578,7 @@ def _lean_findings(graph: Graph, lean_root: str | Path) -> list[AuditFinding]: resolved = [] for name in names: - declaration = index.find(name) + declaration = lean_index.find(name) if declaration is None: findings.append( AuditFinding( @@ -394,9 +590,14 @@ def _lean_findings(graph: Graph, lean_root: str | Path) -> list[AuditFinding]: else: resolved.append(declaration) - expected = _DECLARATION_KEYWORDS.get((node.declaration or "").casefold()) - if expected and resolved and not any(declaration.keyword in expected for declaration in resolved): - actual = ", ".join(sorted({declaration.keyword for declaration in resolved})) + expected = declaration_keywords(node.declaration) + mismatched = [ + declaration + for declaration in resolved + if expected is not None and declaration.keyword not in expected + ] + if mismatched: + actual = ", ".join(sorted({declaration.keyword for declaration in mismatched})) findings.append( AuditFinding( article_path, @@ -425,7 +626,11 @@ def _source_spans(index: SourceIndex) -> dict[str, int]: tails: dict[Path, int] = {} for path, lines in starts.items(): lines.sort() - tails[path] = _line_count(index.root / path) + tails[path] = ( + index.line_counts[path] + if path in index.line_counts + else _line_count(index.root / path) + ) spans: dict[str, int] = {} for declaration in index.declarations.values(): @@ -498,4 +703,10 @@ def _result( return AuditResult(tuple(sorted(set(findings))), coverage) -__all__ = ["AuditFinding", "AuditResult", "audit_blueprint", "audit_graph"] +__all__ = [ + "AuditFinding", + "AuditResult", + "audit_blueprint", + "audit_graph", + "load_audit_graph", +] diff --git a/autoform_cli/claims.py b/autoform_cli/claims.py index 09901325..7325d903 100644 --- a/autoform_cli/claims.py +++ b/autoform_cli/claims.py @@ -13,31 +13,97 @@ import math import os import re +import secrets +import shlex import socket +import stat import subprocess +import sys import threading import time +import weakref +from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Mapping +from typing import Any, Iterable, Mapping +from urllib.parse import urlsplit +from urllib.request import url2pathname CLAIM_REF_PREFIX = "refs/autoform-claims/" -CLAIM_SCHEMA = "autoform-claim/v1" +CLAIM_RECEIPT_REF_PREFIX = "refs/autoform-claim-receipts/" +CLAIM_SCHEMA = "autoform-claim/v2" +LEGACY_CLAIM_SCHEMA = "autoform-claim/v1" +LEGACY_BLOCK_SCHEMA = "autoform-claim/legacy-block/v1" +_PERMANENT_BLOCK_SCHEMAS = frozenset({LEGACY_BLOCK_SCHEMA}) CLAIM_TTL_S = 1500 CLAIM_HEARTBEAT_S = 300 +CLAIM_MAX_TTL_S = 3600 +CLAIM_CLOCK_SKEW_S = 300 CLAIM_KEY_RE = re.compile(r"^[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*$") +LEASE_ID_RE = re.compile(r"^[0-9a-f]{64}$") +OBJECT_ID_RE = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") +_OBJECT_FORMAT_LENGTHS = {"sha1": 40, "sha256": 64} + +_SCP_REPOSITORY_RE = re.compile( + r"^(?:[^/@:]+@)?(?:\[[^\]]+\]|[^/:]+):.+$" +) +_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]") _GIT_ENV = { "GIT_AUTHOR_NAME": "autoform", "GIT_AUTHOR_EMAIL": "autoform@localhost", "GIT_COMMITTER_NAME": "autoform", "GIT_COMMITTER_EMAIL": "autoform@localhost", + "GIT_CONFIG_COUNT": "0", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0", } +_GIT_ENV_ALLOWLIST = frozenset( + { + "ALL_PROXY", + "COMSPEC", + "CURL_CA_BUNDLE", + "HOME", + "HOMEDRIVE", + "HOMEPATH", + "HTTP_PROXY", + "HTTPS_PROXY", + "LANG", + "LANGUAGE", + "LOGNAME", + "NO_PROXY", + "PATH", + "PATHEXT", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SSH_AUTH_SOCK", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "USER", + "USERPROFILE", + "WINDIR", + "all_proxy", + "http_proxy", + "https_proxy", + "no_proxy", + } +) _CAS_REJECTIONS = ( "stale info", "fetch first", "remote ref updated since checkout", "cannot lock ref", ) +_UNPINNED_REPOSITORY = object() +_UNPINNED_SCRATCH = object() +_FCHDIR_EXEC = ( + "import os,sys; os.fchdir(int(sys.argv[1])); " + "os.execvp(sys.argv[2], sys.argv[2:])" +) class ClaimTransportError(RuntimeError): @@ -48,6 +114,30 @@ class MalformedLeaseError(ClaimTransportError): """A claim ref exists, but its lease cannot be verified safely.""" +@dataclass(frozen=True, slots=True) +class ClaimFence: + """One coherent, exact remote ownership receipt for an acquired claim.""" + + key: str + ref: str + oid: str + lease_id: str + + def __post_init__(self) -> None: + key = _validate_key(self.key) + if self.ref != CLAIM_REF_PREFIX + key: + raise ValueError("claim fence ref does not match its key") + if not isinstance(self.oid, str) or OBJECT_ID_RE.fullmatch(self.oid) is None: + raise ValueError("claim fence OID must be a full Git object ID") + if set(self.oid) == {"0"}: + raise ValueError("claim fence OID must identify an object") + if not isinstance(self.lease_id, str) or LEASE_ID_RE.fullmatch(self.lease_id) is None: + raise ValueError("claim fence lease_id must be 64 lowercase hexadecimal characters") + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + def _validate_key(key: str) -> str: if not isinstance(key, str) or not CLAIM_KEY_RE.fullmatch(key) or ".." in key: raise ValueError(f"invalid claim key {key!r}") @@ -68,9 +158,31 @@ def _is_finite_number(value: object) -> bool: def _validate_ttl(ttl: int | float) -> int | float: if not _is_finite_number(ttl) or ttl <= 0: raise ValueError("claim TTL must be a finite positive number") + if ttl > CLAIM_MAX_TTL_S: + raise ValueError(f"claim TTL must not exceed {CLAIM_MAX_TTL_S} seconds") return ttl +def _validate_object_format(value: str) -> str: + if not isinstance(value, str) or value not in _OBJECT_FORMAT_LENGTHS: + choices = ", ".join(sorted(_OBJECT_FORMAT_LENGTHS)) + raise ValueError(f"Git object format must be one of: {choices}") + return value + + +def _canonical_scratch_config(object_format: str) -> bytes: + object_format = _validate_object_format(object_format) + version = 0 if object_format == "sha1" else 1 + extension = "" if object_format == "sha1" else "[extensions]\n\tobjectFormat = sha256\n" + return ( + "[core]\n" + f"\trepositoryformatversion = {version}\n" + "\tbare = true\n" + f"\thooksPath = {os.devnull}\n" + f"{extension}" + ).encode() + + def _reject_json_constant(value: str) -> None: raise ValueError(f"non-finite JSON number {value!r}") @@ -84,6 +196,128 @@ def _strict_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: return value +def _resolve_local_path(value: str | os.PathLike[str], *, label: str) -> Path: + try: + return Path(value).expanduser().resolve() + except (OSError, RuntimeError) as exc: + raise ValueError(f"{label} path cannot be resolved safely") from exc + + +def _directory_identity( + path: Path, + *, + label: str, + allow_missing: bool, +) -> tuple[int, int] | None: + try: + info = path.stat(follow_symlinks=False) + except FileNotFoundError: + if allow_missing: + return None + raise ClaimTransportError(f"{label} directory is no longer available") from None + except OSError as exc: + raise ClaimTransportError(f"{label} directory cannot be inspected safely") from exc + if not stat.S_ISDIR(info.st_mode): + raise ClaimTransportError(f"{label} path must be a real directory") + return info.st_dev, info.st_ino + + +def _directory_path_snapshot( + path: Path, + *, + anchor: Path, + label: str, +) -> tuple[tuple[int, int, int | None], ...]: + try: + relative = path.relative_to(anchor) + except ValueError as exc: + raise ClaimTransportError(f"{label} escaped its pinned filesystem boundary") from exc + components = [anchor] + for part in relative.parts: + components.append(components[-1] / part) + snapshot: list[tuple[int, int, int | None]] = [] + for component in components: + try: + info = component.stat(follow_symlinks=False) + except OSError as exc: + raise ClaimTransportError( + f"{label} path component cannot be inspected safely" + ) from exc + if not stat.S_ISDIR(info.st_mode): + raise ClaimTransportError(f"{label} path component must be a real directory") + changed_at_ns = None if component == path else info.st_ctime_ns + snapshot.append((info.st_dev, info.st_ino, changed_at_ns)) + return tuple(snapshot) + + +def _directory_operation_guard( + path: Path, + *, + anchor: Path, + label: str, +) -> tuple[tuple[int, int, int | None], ...]: + """Stably capture every path component around one Git subprocess.""" + before = _directory_path_snapshot(path, anchor=anchor, label=label) + after = _directory_path_snapshot(path, anchor=anchor, label=label) + if before != after: + raise ClaimTransportError(f"{label} changed while its path was being inspected") + return after + + +def claim_repository_is_remote(repo_url: str | os.PathLike[str]) -> bool: + """Return whether Git will treat this repository name as a remote transport.""" + value = os.fspath(repo_url) + if _WINDOWS_DRIVE_RE.match(value): + return False + return "://" in value or bool(_SCP_REPOSITORY_RE.match(value)) + + +def normalize_claim_repository(repo_url: str | os.PathLike[str]) -> str: + """Return a stable transport identity, resolving local paths and file URLs.""" + raw_repo_url = os.fspath(repo_url) + parsed = urlsplit(raw_repo_url) + if parsed.scheme.lower() == "file": + if parsed.query or parsed.fragment or parsed.netloc.lower() not in {"", "localhost"}: + raise ValueError("file repository URL must identify an absolute local path") + local_path = Path(url2pathname(parsed.path)) + if not local_path.is_absolute(): + raise ValueError("file repository URL must identify an absolute local path") + return str(_resolve_local_path(local_path, label="claim repository")) + if not claim_repository_is_remote(raw_repo_url): + return str(_resolve_local_path(raw_repo_url, label="claim repository")) + return raw_repo_url + + +def pin_claim_repository( + repo_url: str | os.PathLike[str], +) -> tuple[str, tuple[int, int] | None]: + """Resolve a claim repository and capture its local filesystem identity.""" + normalized = normalize_claim_repository(repo_url) + local_path = None if claim_repository_is_remote(normalized) else Path(normalized) + identity = ( + _directory_identity( + local_path, + label="local claim repository", + allow_missing=True, + ) + if local_path is not None + else None + ) + return normalized, identity + + +def pin_claim_scratch( + scratch: str | os.PathLike[str], +) -> tuple[Path, tuple[int, int] | None]: + """Resolve a scratch path and capture an existing directory's identity.""" + path = _resolve_local_path(scratch, label="claim scratch") + return path, _directory_identity( + path, + label="claim scratch", + allow_missing=True, + ) + + def author_claim_key(node_id: str) -> str: """Return a readable, ref-safe, collision-resistant author claim key.""" if not isinstance(node_id, str): @@ -93,18 +327,179 @@ def author_claim_key(node_id: str) -> str: return f"author/{slug}-{digest}" +def workspace_author_claim_key(project_id: str, article_id: str) -> str: + """Return an article claim key scoped to one stable workspace project id.""" + + if not isinstance(project_id, str) or not project_id: + raise ValueError("workspace project id must not be empty") + project = author_claim_key(project_id).removeprefix("author/") + article = author_claim_key(article_id).removeprefix("author/") + return f"author/workspace-{project}/{article}" + + +def resource_claim_key(resource: str) -> str: + """Return a ref-safe key in the namespace for non-article resources.""" + if not isinstance(resource, str): + raise TypeError("resource must be a string") + if not resource: + raise ValueError("resource must not be empty") + slug = re.sub(r"[^a-z0-9-]+", "-", resource.lower()).strip("-")[:48] or "resource" + digest = hashlib.sha256(resource.encode("utf-8")).hexdigest()[:16] + return f"resource/{slug}-{digest}" + + +def _open_pinned_directory( + path: Path, + identity: tuple[int, int] | None, + *, + label: str, +) -> int | None: + if identity is None or os.name != "posix" or not hasattr(os, "fchdir"): + return None + flags = os.O_RDONLY + if hasattr(os, "O_DIRECTORY"): + flags |= os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor: int | None = None + try: + descriptor = os.open(path, flags) + info = os.fstat(descriptor) + except OSError as exc: + if descriptor is not None: + os.close(descriptor) + raise ClaimTransportError(f"{label} cannot be pinned safely") from exc + if not stat.S_ISDIR(info.st_mode) or (info.st_dev, info.st_ino) != identity: + os.close(descriptor) + raise ClaimTransportError(f"{label} was replaced") + return descriptor + + +def _claim_git_environment() -> dict[str, str]: + environment = { + key: value + for key, value in os.environ.items() + if key in _GIT_ENV_ALLOWLIST or key.startswith("LC_") + } + environment.setdefault("PATH", os.defpath) + environment.update(_GIT_ENV) + return environment + + +def _parse_ls_remote_output( + output: str, + *, + allow_head: bool = False, +) -> list[tuple[str, str]]: + if not output: + return [] + entries: list[tuple[str, str]] = [] + lines = output.split("\n") + if lines[-1] == "": + lines.pop() + for line in lines: + oid, separator, ref = line.partition("\t") + if ( + not separator + or not OBJECT_ID_RE.fullmatch(oid) + or (not ref.startswith("refs/") and not (allow_head and ref == "HEAD")) + or any( + character == " " or ord(character) < 32 or ord(character) == 127 + for character in ref + ) + ): + raise ClaimTransportError("claim board returned malformed ls-remote output") + entries.append((oid, ref)) + return entries + + class ClaimBoard: """Lease operations against a Git repository via a local bare object store.""" - def __init__(self, repo_url: str | os.PathLike[str], worker_id: str, scratch: str | os.PathLike[str]): + def __init__( + self, + repo_url: str | os.PathLike[str], + worker_id: str, + scratch: str | os.PathLike[str], + *, + session_id: str | None = None, + expected_object_format: str | None = None, + expected_repo_identity: object = _UNPINNED_REPOSITORY, + expected_scratch_identity: object = _UNPINNED_SCRATCH, + ): if not worker_id: raise ValueError("worker_id must not be empty") - raw_repo_url = os.fspath(repo_url) - if "://" not in raw_repo_url and not re.match(r"^[^/]+@[^:]+:", raw_repo_url): - raw_repo_url = str(Path(raw_repo_url).expanduser().resolve()) - self.repo_url = raw_repo_url + validated_object_format = ( + _validate_object_format(expected_object_format) + if expected_object_format is not None + else None + ) + self.repo_url, current_repo_identity = pin_claim_repository(repo_url) + self._repo_path = ( + None if claim_repository_is_remote(self.repo_url) else Path(self.repo_url) + ) + if ( + expected_repo_identity is not _UNPINNED_REPOSITORY + and current_repo_identity != expected_repo_identity + ): + raise ClaimTransportError("local claim repository was replaced") + self._repo_identity = current_repo_identity self.worker_id = worker_id - self.scratch = Path(scratch) + self.scratch, current_scratch_identity = pin_claim_scratch(scratch) + if ( + expected_scratch_identity is not _UNPINNED_SCRATCH + and current_scratch_identity != expected_scratch_identity + ): + raise ClaimTransportError("claim scratch directory was replaced") + self._scratch_identity = current_scratch_identity + if self._scratch_identity is None: + self.scratch.parent.mkdir(parents=True, exist_ok=True) + try: + self.scratch.mkdir() + except FileExistsError as exc: + raise ClaimTransportError( + "claim scratch directory appeared while its identity was being pinned" + ) from exc + self._scratch_identity = _directory_identity( + self.scratch, + label="claim scratch", + allow_missing=False, + ) + self._path_anchor = ( + Path(os.path.commonpath((self._repo_path, self.scratch))) + if self._repo_path is not None + else self.scratch + ) + self._scratch_fd = _open_pinned_directory( + self.scratch, + self._scratch_identity, + label="claim scratch directory", + ) + self._repo_fd = ( + _open_pinned_directory( + self._repo_path, + self._repo_identity, + label="local claim repository", + ) + if self._repo_path is not None + else None + ) + self._fd_finalizers: list[weakref.finalize] = [] + for descriptor in (self._scratch_fd, self._repo_fd): + if descriptor is not None: + self._fd_finalizers.append(weakref.finalize(self, os.close, descriptor)) + self._transport_helper = Path(__file__).with_name("_git_fd_transport.py").resolve() + self._scratch_ready = False + self._expected_object_format = validated_object_format + self._object_format: str | None = None + if session_id is None: + session_id = f"scratch:{self.scratch}" + if not isinstance(session_id, str) or not session_id: + raise ValueError("session_id must not be empty") + self.session_id = session_id + self._session_key = hashlib.sha256( + f"{self.repo_url}\0{session_id}".encode("utf-8") + ).hexdigest() def _git( self, @@ -112,43 +507,456 @@ def _git( *, check: bool = True, input_text: str | None = None, + remote: bool = False, ) -> subprocess.CompletedProcess[str]: + display_args = args + if remote and self._repo_fd is not None: + args = self._local_transport_args(args) + self._verify_scratch_identity() + scratch_guard = None + repo_guard = None + if self._scratch_fd is None: + scratch_guard = _directory_operation_guard( + self.scratch, + anchor=self._path_anchor, + label="claim scratch", + ) + if remote: + self._verify_repo_identity() + if ( + self._repo_fd is None + and self._repo_path is not None + and self._repo_identity is not None + ): + repo_guard = _directory_operation_guard( + self._repo_path, + anchor=self._path_anchor, + label="local claim repository", + ) try: + environment = {**_claim_git_environment(), "GIT_DIR": "."} + command = ["git", *args] + run_options: dict[str, Any] = {"cwd": self.scratch} + descriptors = tuple( + descriptor + for descriptor in (self._scratch_fd, self._repo_fd if remote else None) + if descriptor is not None + ) + if self._scratch_fd is not None: + command = [ + sys.executable, + "-c", + _FCHDIR_EXEC, + str(self._scratch_fd), + "git", + *args, + ] + run_options = {"pass_fds": descriptors} proc = subprocess.run( - ["git", *args], - cwd=self.scratch, + command, capture_output=True, text=True, + errors="surrogateescape", input=input_text, timeout=120, - env={**os.environ, **_GIT_ENV}, + env=environment, + **run_options, ) except (OSError, subprocess.TimeoutExpired) as exc: raise ClaimTransportError(f"git claim-board operation failed: {exc}") from exc + self._verify_scratch_identity() + if remote: + self._verify_repo_identity() + if repo_guard is not None and ( + _directory_operation_guard( + self._repo_path, + anchor=self._path_anchor, + label="local claim repository", + ) + != repo_guard + ): + raise ClaimTransportError( + "local claim repository changed during a Git operation" + ) + if scratch_guard is not None and ( + _directory_operation_guard( + self.scratch, + anchor=self._path_anchor, + label="claim scratch", + ) + != scratch_guard + ): + raise ClaimTransportError("claim scratch changed during a Git operation") if check and proc.returncode != 0: detail = (proc.stderr or proc.stdout).strip()[:300] - raise ClaimTransportError(f"git {' '.join(args[:2])} failed against claim board: {detail}") + raise ClaimTransportError( + f"git {' '.join(display_args[:2])} failed against claim board: {detail}" + ) + return proc + + def _local_transport_args(self, args: list[str]) -> list[str]: + if self._repo_fd is None or not args: + return args + operation = args[0] + if operation in {"ls-remote", "fetch"}: + mode = "upload" + option = "--upload-pack" + elif operation == "push": + mode = "receive" + option = "--receive-pack" + else: + raise ClaimTransportError( + f"unsupported local claim transport operation {operation!r}" + ) + helper = shlex.join( + ( + sys.executable, + os.fspath(self._transport_helper), + mode, + str(self._repo_fd), + ) + ) + rewritten = ["." if arg == self.repo_url else arg for arg in args] + if rewritten == args: + raise ClaimTransportError("local claim transport target was not explicit") + rewritten.insert(1, f"{option}={helper}") + return rewritten + + def _verify_repo_identity(self) -> None: + if self._repo_path is None: + return + current = _directory_identity( + self._repo_path, + label="local claim repository", + allow_missing=True, + ) + if current != self._repo_identity: + raise ClaimTransportError("local claim repository was replaced") + + def _remote_git( + self, + args: list[str], + *, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + self._verify_repo_identity() + proc = self._git(args, check=check, remote=True) + self._verify_repo_identity() return proc + def _verify_scratch_identity(self) -> None: + current = _directory_identity( + self.scratch, + label="claim scratch", + allow_missing=True, + ) + if current != self._scratch_identity: + raise ClaimTransportError("claim scratch directory was replaced") + + def _repository_object_format(self) -> str | None: + self._verify_repo_identity() + if self._repo_path is not None: + command = ["git", "rev-parse", "--show-object-format"] + run_options: dict[str, Any] = {"cwd": self._repo_path} + if self._repo_fd is not None: + command = [ + sys.executable, + "-c", + _FCHDIR_EXEC, + str(self._repo_fd), + "git", + "rev-parse", + "--show-object-format", + ] + run_options = {"pass_fds": (self._repo_fd,)} + try: + proc = subprocess.run( + command, + capture_output=True, + text=True, + timeout=120, + env=_claim_git_environment(), + **run_options, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ClaimTransportError( + f"cannot inspect claim repository object format: {exc}" + ) from exc + self._verify_repo_identity() + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip()[:300] + raise ClaimTransportError( + f"cannot inspect claim repository object format: {detail}" + ) + detected = proc.stdout.strip() + else: + entries: list[tuple[str, str]] = [] + commands = ( + (["git", "ls-remote", "--refs", self.repo_url], False), + (["git", "ls-remote", self.repo_url, "HEAD"], True), + ) + for command, allow_head in commands: + try: + proc = subprocess.run( + command, + capture_output=True, + text=True, + errors="surrogateescape", + timeout=120, + env=_claim_git_environment(), + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ClaimTransportError( + f"cannot inspect claim repository object format: {exc}" + ) from exc + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip()[:300] + raise ClaimTransportError( + f"cannot inspect claim repository object format: {detail}" + ) + entries = _parse_ls_remote_output(proc.stdout, allow_head=allow_head) + if entries: + break + widths = {len(oid) for oid, _ref in entries} + if not widths: + return self._expected_object_format + if len(widths) != 1: + raise ClaimTransportError("claim repository returned mixed object formats") + width = widths.pop() + detected = next( + name for name, length in _OBJECT_FORMAT_LENGTHS.items() if length == width + ) + try: + detected = _validate_object_format(detected) + except ValueError as exc: + raise ClaimTransportError("claim repository has an unsupported object format") from exc + if ( + self._expected_object_format is not None + and detected != self._expected_object_format + ): + raise ClaimTransportError( + f"claim repository object format {detected!r} does not match expected " + f"{self._expected_object_format!r}" + ) + return detected + + def _scratch_object_format(self) -> str: + proc = self._git(["rev-parse", "--show-object-format"], check=False) + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip()[:300] + raise ClaimTransportError( + f"cannot inspect claim scratch object format: {detail}" + ) + try: + return _validate_object_format(proc.stdout.strip()) + except ValueError as exc: + raise ClaimTransportError("claim scratch has an unsupported object format") from exc + + def _install_canonical_scratch_config(self, object_format: str) -> None: + canonical_config = _canonical_scratch_config(object_format) + read_flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + read_flags |= os.O_NOFOLLOW + existing: int | None = None + try: + if self._scratch_fd is not None: + existing = os.open("config", read_flags, dir_fd=self._scratch_fd) + else: + existing = os.open(self.scratch / "config", read_flags) + info = os.fstat(existing) + content = os.read(existing, len(canonical_config) + 1) + if stat.S_ISREG(info.st_mode) and content == canonical_config: + return + except OSError: + pass + finally: + if existing is not None: + try: + os.close(existing) + except OSError: + pass + temporary_name = f".autoform-config-{os.getpid()}-{secrets.token_hex(8)}" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor: int | None = None + try: + if self._scratch_fd is not None: + descriptor = os.open( + temporary_name, + flags, + 0o600, + dir_fd=self._scratch_fd, + ) + else: + descriptor = os.open(self.scratch / temporary_name, flags, 0o600) + remaining = memoryview(canonical_config) + while remaining: + written = os.write(descriptor, remaining) + if written <= 0: + raise OSError("short write while installing claim scratch config") + remaining = remaining[written:] + os.fsync(descriptor) + os.close(descriptor) + descriptor = None + if self._scratch_fd is not None: + os.replace( + temporary_name, + "config", + src_dir_fd=self._scratch_fd, + dst_dir_fd=self._scratch_fd, + ) + os.fsync(self._scratch_fd) + else: + os.replace(self.scratch / temporary_name, self.scratch / "config") + except OSError as exc: + if descriptor is not None: + os.close(descriptor) + try: + if self._scratch_fd is not None: + os.unlink(temporary_name, dir_fd=self._scratch_fd) + else: + (self.scratch / temporary_name).unlink() + except OSError: + pass + raise ClaimTransportError( + "claim scratch Git configuration cannot be installed safely" + ) from exc + def _ensure_scratch(self) -> None: + if self._scratch_identity is not None: + self._verify_scratch_identity() + else: + self.scratch.mkdir(parents=True, exist_ok=True) + self._scratch_identity = _directory_identity( + self.scratch, + label="claim scratch", + allow_missing=False, + ) + if self._scratch_ready: + if (self.scratch / "HEAD").is_symlink(): + raise ClaimTransportError("claim scratch HEAD must not be a symbolic link") + if not (self.scratch / "HEAD").is_file(): + raise ClaimTransportError("claim scratch is no longer a bare Git repository") + object_format = self._scratch_object_format() + if self._object_format != object_format: + raise ClaimTransportError("claim scratch object format changed") + self._install_canonical_scratch_config(object_format) + return + if (self.scratch / "HEAD").is_symlink(): + raise ClaimTransportError("claim scratch HEAD must not be a symbolic link") if (self.scratch / "HEAD").is_file(): + proc = self._git(["rev-parse", "--is-bare-repository"], check=False) + if proc.returncode != 0 or proc.stdout.strip() != "true": + raise ClaimTransportError("claim scratch must be a bare Git repository") + object_format = self._scratch_object_format() + expected = self._repository_object_format() + if expected is not None and object_format != expected: + raise ClaimTransportError( + f"claim scratch object format {object_format!r} does not match repository " + f"object format {expected!r}" + ) + self._install_canonical_scratch_config(object_format) + self._object_format = object_format + self._scratch_ready = True return - self.scratch.mkdir(parents=True, exist_ok=True) - self._git(["init", "--bare", "--quiet", "."]) + object_format = self._repository_object_format() + if object_format is None: + raise ClaimTransportError( + "cannot determine an empty remote claim repository's object format; " + "pass expected_object_format" + ) + self._git( + [ + "init", + "--bare", + "--quiet", + "--template=", + f"--object-format={object_format}", + ] + ) + if (self.scratch / "HEAD").is_symlink() or not (self.scratch / "HEAD").is_file(): + raise ClaimTransportError("claim scratch initialization could not be verified") + actual_format = self._scratch_object_format() + if actual_format != object_format: + raise ClaimTransportError("claim scratch initialized with the wrong object format") + self._install_canonical_scratch_config(actual_format) + self._object_format = actual_format + self._scratch_ready = True @staticmethod def _ref(key: str) -> str: return CLAIM_REF_PREFIX + _validate_key(key) + def _receipt_ref(self, key: str) -> str: + return f"{CLAIM_RECEIPT_REF_PREFIX}{self._session_key}/{_validate_key(key)}" + + def _verify_object_id_format(self, oid: str) -> None: + if ( + self._object_format is None + or len(oid) != _OBJECT_FORMAT_LENGTHS[self._object_format] + ): + raise ClaimTransportError("claim repository object format changed") + + def _remote_ref_oid(self, ref: str) -> str | None: + proc = self._remote_git(["ls-remote", self.repo_url, ref]) + entries = _parse_ls_remote_output(proc.stdout) + if not entries: + return None + if len(entries) != 1 or entries[0][1] != ref: + raise ClaimTransportError( + f"claim board did not resolve exact requested ref {ref!r}" + ) + self._verify_object_id_format(entries[0][0]) + return entries[0][0] + def _remote_oid(self, key: str) -> str | None: - proc = self._git(["ls-remote", self.repo_url, self._ref(key)]) - line = proc.stdout.strip() - return line.split("\t", 1)[0] if line else None + return self._remote_ref_oid(self._ref(key)) + + def _receipt_oid(self, key: str) -> str | None: + proc = self._git( + ["rev-parse", "--verify", "--quiet", self._receipt_ref(key)], + check=False, + ) + if proc.returncode == 0: + return proc.stdout.strip() or None + if proc.returncode == 1: + return None + detail = (proc.stderr or proc.stdout).strip()[:300] + raise ClaimTransportError(f"could not read local claim receipt: {detail}") + + def _record_receipt(self, key: str, oid: str, *, expected: str | None = None) -> None: + args = ["update-ref", self._receipt_ref(key), oid] + if expected is not None: + args.append(expected) + proc = self._git(args, check=False) + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip()[:300] + raise ClaimTransportError( + "remote claim changed but its exact local ownership receipt could not be recorded" + + (f": {detail}" if detail else "") + ) + + def _clear_receipt(self, key: str, *, expected: str | None) -> None: + object_id_width = len( + self._git(["hash-object", "--stdin"], input_text="").stdout.strip() + ) + zero_oid = "0" * object_id_width + args = ["update-ref", self._receipt_ref(key), zero_oid, expected or zero_oid] + proc = self._git(args, check=False) + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip()[:300] + raise ClaimTransportError( + "remote claim changed but its local ownership receipt could not be cleared" + + (f": {detail}" if detail else "") + ) def _read_lease(self, key: str, oid: str) -> dict[str, Any]: ref = self._ref(key) if self._git(["cat-file", "-e", f"{oid}^{{commit}}"], check=False).returncode != 0: - self._git(["fetch", "--quiet", self.repo_url, f"+{ref}:{ref}"]) + self._remote_git( + ["fetch", "--quiet", "--no-write-fetch-head", self.repo_url, f"+{ref}:{ref}"] + ) proc = self._git(["cat-file", "commit", oid], check=False) if proc.returncode != 0: raise MalformedLeaseError(f"claim {key!r} does not point to a readable commit") @@ -169,10 +977,18 @@ def _read_lease(self, key: str, oid: str) -> dict[str, Any]: @staticmethod def _lease_is_valid(lease: Mapping[str, Any], key: str | None = None) -> bool: + if lease.get("schema") == LEGACY_BLOCK_SCHEMA: + return bool( + isinstance(lease.get("resource"), str) + and _is_finite_number(lease.get("blocked_at")) + and isinstance(lease.get("canonical_resource"), str) + and (key is None or lease.get("resource") == key) + ) acquired_at = lease.get("acquired_at") + renewed_at = lease.get("renewed_at", acquired_at) expires_at = lease.get("expires_at") valid = ( - lease.get("schema") == CLAIM_SCHEMA + lease.get("schema") in {CLAIM_SCHEMA, LEGACY_CLAIM_SCHEMA} and isinstance(lease.get("owner"), str) and bool(lease.get("owner")) and isinstance(lease.get("resource"), str) @@ -180,31 +996,93 @@ def _lease_is_valid(lease: Mapping[str, Any], key: str | None = None) -> bool: and _is_finite_number(expires_at) and acquired_at <= expires_at ) + if lease.get("schema") == CLAIM_SCHEMA: + valid = ( + valid + and _is_finite_number(renewed_at) + and acquired_at <= renewed_at <= expires_at + and isinstance(lease.get("lease_id"), str) + and bool(LEASE_ID_RE.fullmatch(str(lease.get("lease_id")))) + ) return bool(valid and (key is None or lease.get("resource") == key)) - def _make_lease_commit(self, key: str, ttl: int | float, note: str = "") -> str: + def _make_legacy_block_commit(self, key: str, canonical_key: str) -> str: + key = _validate_key(key) + canonical_key = _validate_key(canonical_key) + now = time.time() + if not math.isfinite(now): + raise ValueError("claim timestamp must be finite") + block = { + "blocked_at": now, + "canonical_resource": canonical_key, + "resource": key, + "schema": LEGACY_BLOCK_SCHEMA, + } + tree = self._git(["mktree"], input_text="").stdout.strip() + message = json.dumps(block, sort_keys=True, separators=(",", ":"), allow_nan=False) + return self._git(["commit-tree", tree, "-m", message]).stdout.strip() + + def _make_lease_commit( + self, + key: str, + ttl: int | float, + note: str = "", + *, + lease_id: str | None = None, + acquired_at: int | float | None = None, + previous_renewed_at: int | float | None = None, + previous_expires_at: int | float | None = None, + ) -> str: key = _validate_key(key) ttl = _validate_ttl(ttl) now = time.time() if not math.isfinite(now): raise ValueError("claim timestamp must be finite") + if acquired_at is None: + acquired_at = now + if not _is_finite_number(acquired_at) or acquired_at > now + CLAIM_CLOCK_SKEW_S: + raise ValueError( + "claim acquisition timestamp must be finite and within the allowed clock skew" + ) + if previous_renewed_at is None: + previous_renewed_at = acquired_at + if ( + not _is_finite_number(previous_renewed_at) + or previous_renewed_at < acquired_at + or previous_renewed_at > now + CLAIM_CLOCK_SKEW_S + ): + raise ValueError( + "claim renewal timestamp must be monotonic and within the allowed clock skew" + ) + renewed_at = max(now, acquired_at, previous_renewed_at) + if previous_expires_at is not None and not _is_finite_number(previous_expires_at): + raise ValueError("claim expiry timestamp must be finite") + expiry_floor = renewed_at if previous_expires_at is None else previous_expires_at try: - expires_at = now + ttl + expires_at = max(renewed_at + ttl, expiry_floor) except OverflowError as exc: raise ValueError("claim expiry must be finite") from exc if not math.isfinite(expires_at): raise ValueError("claim expiry must be finite") + if lease_id is None: + lease_id = secrets.token_hex(32) + if not isinstance(lease_id, str) or not LEASE_ID_RE.fullmatch(lease_id): + raise ValueError("claim lease_id must be 64 lowercase hexadecimal characters") lease: dict[str, Any] = { "schema": CLAIM_SCHEMA, + "lease_id": lease_id, "owner": self.worker_id, "host": socket.gethostname(), "pid": os.getpid(), - "acquired_at": now, + "acquired_at": acquired_at, + "renewed_at": renewed_at, "expires_at": expires_at, "resource": key, } if note: lease["note"] = note + if not self._lease_is_valid(lease, key): + raise ValueError("generated claim lease violates the claim schema") tree = self._git(["mktree"], input_text="").stdout.strip() message = json.dumps(lease, sort_keys=True, separators=(",", ":"), allow_nan=False) return self._git(["commit-tree", tree, "-m", message]).stdout.strip() @@ -212,7 +1090,7 @@ def _make_lease_commit(self, key: str, ttl: int | float, note: str = "") -> str: def _cas_push(self, key: str, old: str | None, new: str) -> bool: ref = self._ref(key) source = new if new else "" - proc = self._git( + proc = self._remote_git( [ "push", "--quiet", @@ -228,6 +1106,18 @@ def _cas_push(self, key: str, old: str | None, new: str) -> bool: detail = f"{proc.stdout}\n{proc.stderr}".strip() if any(marker in detail.lower() for marker in _CAS_REJECTIONS): return False + # Some Git transports report a compare-and-swap loss only as a generic + # remote "failed to update ref" error. Re-read the ref: a value that + # differs from our lease proves another claimant won, while an unchanged + # value remains a genuine transport failure. + try: + current = self._remote_oid(key) + except ClaimTransportError: + current = old + if current == (new or None): + return True + if current != old: + return False raise ClaimTransportError(f"claim CAS push failed: {detail[:300]}") def read(self, key: str) -> dict[str, Any] | None: @@ -238,84 +1128,351 @@ def read(self, key: str) -> dict[str, Any] | None: @classmethod def expired(cls, lease: Mapping[str, Any], now: float | None = None) -> bool: - """Return whether a lease is malformed or no longer live.""" + """Return whether a lease is reclaimable after bounded clock-skew grace.""" + comparison_time = time.time() if now is None else now + if not _is_finite_number(comparison_time): + raise ValueError("claim expiry comparison clock must be finite") + if lease.get("schema") in _PERMANENT_BLOCK_SCHEMAS: + return False expires_at = lease.get("expires_at") if not _is_finite_number(expires_at): return True + return expires_at <= comparison_time - CLAIM_CLOCK_SKEW_S + + @classmethod + def _holder_expired(cls, lease: Mapping[str, Any], now: float | None = None) -> bool: + """Return whether the lease holder's nominal authority has elapsed.""" comparison_time = time.time() if now is None else now if not _is_finite_number(comparison_time): raise ValueError("claim expiry comparison clock must be finite") + if lease.get("schema") in _PERMANENT_BLOCK_SCHEMAS: + return False + expires_at = lease.get("expires_at") + if not _is_finite_number(expires_at): + return True return expires_at <= comparison_time - def acquire(self, key: str, ttl: int | float = CLAIM_TTL_S, steal: bool = False, note: str = "") -> bool: - """CAS-acquire a free, expired, malformed, owned, or explicitly stolen lease.""" + @classmethod + def recovery_required(cls, lease: Mapping[str, Any], now: float | None = None) -> bool: + """Return whether bounded lease timing was violated and explicit cleanup is required.""" + comparison_time = time.time() if now is None else now + if not _is_finite_number(comparison_time): + raise ValueError("claim recovery comparison clock must be finite") + if lease.get("schema") in _PERMANENT_BLOCK_SCHEMAS: + return False + acquired_at = lease.get("acquired_at") + renewed_at = lease.get("renewed_at", acquired_at) + expires_at = lease.get("expires_at") + if ( + not _is_finite_number(acquired_at) + or not _is_finite_number(renewed_at) + or not _is_finite_number(expires_at) + ): + return False + return bool( + renewed_at > comparison_time + CLAIM_CLOCK_SKEW_S + or ( + lease.get("schema") == CLAIM_SCHEMA + and expires_at - renewed_at > CLAIM_MAX_TTL_S + ) + ) + + def install_legacy_compatibility(self, key: str, *, canonical_key: str) -> bool: + """Permanently fence a v1 path key before a v2 canonical claim is used.""" key = _validate_key(key) - _validate_ttl(ttl) + canonical_key = _validate_key(canonical_key) self._ensure_scratch() old = self._remote_oid(key) if old is not None: lease = self._read_lease(key, old) + if lease.get("schema") == LEGACY_BLOCK_SCHEMA: + return True if ( - lease is not None - and self._lease_is_valid(lease, key) - and lease.get("owner") != self.worker_id - and not self.expired(lease) - and not steal + lease.get("schema") not in {LEGACY_CLAIM_SCHEMA, CLAIM_SCHEMA} + or self.recovery_required(lease) + or not self.expired(lease) ): return False - new = self._make_lease_commit(key, ttl, note) + new = self._make_legacy_block_commit(key, canonical_key) return self._cas_push(key, old, new) - def renew(self, key: str, ttl: int | float = CLAIM_TTL_S) -> bool: - """CAS-renew this worker's lease, returning ``False`` if ownership was lost.""" + def prepare_v2_claim( + self, + canonical_key: str, + compatibility_keys: Iterable[str], + *, + canonical_keys: Iterable[str] = (), + ) -> bool: + """Retire observable v1 keys, then install permanent compatibility fences.""" + canonical_key = _validate_key(canonical_key) + keys = tuple( + key + for key in dict.fromkeys(_validate_key(key) for key in compatibility_keys) + if key != canonical_key + ) + protected = {_validate_key(key) for key in canonical_keys} + protected.add(canonical_key) + collisions = sorted(set(keys) & (protected - {canonical_key})) + if collisions: + raise ValueError( + "legacy compatibility key collides with a durable canonical claim key: " + + ", ".join(collisions) + ) + + for lease in self.list(): + key = str(lease["_key"]) + if not key.startswith("author/"): + continue + if lease["_malformed"]: + raise MalformedLeaseError( + f"legacy rollout is blocked by unreadable author claim {key!r}: " + f"{lease['_error']}" + ) + if lease.get("schema") != LEGACY_CLAIM_SCHEMA: + continue + if lease["_recovery_required"]: + raise ClaimTransportError( + f"legacy rollout is blocked by unsafe-timestamp claim {key!r}; " + "inspect it and run claim cleanup --blueprint PROJECT to recover" + ) + if not lease["_expired"]: + return False + if key in protected: + continue + if not self.install_legacy_compatibility(key, canonical_key=canonical_key): + return False + + for key in keys: + if not self.install_legacy_compatibility(key, canonical_key=canonical_key): + return False + + for lease in self.list(): + key = str(lease["_key"]) + if not key.startswith("author/"): + continue + if lease["_malformed"]: + return False + if lease.get("schema") == LEGACY_CLAIM_SCHEMA and not ( + key in protected and lease["_expired"] + ): + return False + return True + + def _legacy_author_claim_blocks_v2(self, key: str) -> bool: + if not key.startswith("author/"): + return False + for lease in self.list(): + if not str(lease["_key"]).startswith("author/"): + continue + if lease["_malformed"]: + raise MalformedLeaseError(str(lease["_error"])) + if lease.get("schema") == LEGACY_CLAIM_SCHEMA and ( + lease["_recovery_required"] or not lease["_expired"] + ): + return True + return False + + def acquire( + self, + key: str, + ttl: int | float = CLAIM_TTL_S, + steal: bool = False, + note: str = "", + ) -> bool: + """CAS-acquire a free or expired lease, or refresh this exact session's lease.""" + key = _validate_key(key) + _validate_ttl(ttl) + self._ensure_scratch() + if self._legacy_author_claim_blocks_v2(key): + return False + old = self._remote_oid(key) + lease_id: str | None = None + acquired_at: int | float | None = None + previous_renewed_at: int | float | None = None + previous_expires_at: int | float | None = None + if old is not None: + lease = self._read_lease(key, old) + if lease.get("schema") in _PERMANENT_BLOCK_SCHEMAS or self.recovery_required(lease): + return False + if not self.expired(lease): + if lease.get("schema") == LEGACY_CLAIM_SCHEMA: + return False + if self._holder_expired(lease): + return False + if not self._receipt_matches(key, old, lease): + return False + else: + lease_id = str(lease["lease_id"]) + acquired_at = lease["acquired_at"] + previous_renewed_at = lease.get("renewed_at", acquired_at) + previous_expires_at = lease["expires_at"] + new = self._make_lease_commit( + key, + ttl, + note, + lease_id=lease_id, + acquired_at=acquired_at, + previous_renewed_at=previous_renewed_at, + previous_expires_at=previous_expires_at, + ) + if not self._cas_push(key, old, new): + return False + if self._legacy_author_claim_blocks_v2(key): + if not self._cas_push(key, new, old or ""): + raise ClaimTransportError( + "a legacy v1 claim appeared while a v2 claim was acquired, and " + "the v2 claim could not be rolled back" + ) + return False + self._record_receipt(key, new, expected=old if lease_id is not None else None) + return True + + def renew( + self, + key: str, + ttl: int | float = CLAIM_TTL_S, + *, + lease_id: str | None = None, + ) -> bool: + """CAS-renew this session's exact lease, returning ``False`` if it was lost.""" key = _validate_key(key) _validate_ttl(ttl) self._ensure_scratch() + if self._legacy_author_claim_blocks_v2(key): + return False old = self._remote_oid(key) if old is None: return False lease = self._read_lease(key, old) - if lease is None or not self._lease_is_valid(lease, key) or lease.get("owner") != self.worker_id: + if ( + lease.get("schema") != CLAIM_SCHEMA + or self.recovery_required(lease) + or self._holder_expired(lease) + or not self._receipt_matches(key, old, lease) + or (lease_id is not None and lease.get("lease_id") != lease_id) + ): return False - new = self._make_lease_commit(key, ttl, str(lease.get("note", ""))) - return self._cas_push(key, old, new) + new = self._make_lease_commit( + key, + ttl, + str(lease.get("note", "")), + lease_id=str(lease["lease_id"]), + acquired_at=lease["acquired_at"], + previous_renewed_at=lease.get("renewed_at", lease["acquired_at"]), + previous_expires_at=lease["expires_at"], + ) + if not self._cas_push(key, old, new): + return False + if self._legacy_author_claim_blocks_v2(key): + if not self._cas_push(key, new, old): + raise ClaimTransportError( + "a legacy v1 claim appeared while a v2 claim was renewed, and " + "the prior lease could not be restored" + ) + return False + self._record_receipt(key, new, expected=old) + return True def release(self, key: str) -> bool: - """CAS-delete this worker's lease; refuse foreign or unverifiable ownership.""" + """CAS-delete this session's lease; refuse stale or unverifiable ownership.""" key = _validate_key(key) self._ensure_scratch() + receipt = self._receipt_oid(key) old = self._remote_oid(key) if old is None: + self._clear_receipt(key, expected=receipt) return True lease = self._read_lease(key, old) - if lease is None or not self._lease_is_valid(lease, key) or lease.get("owner") != self.worker_id: + if ( + lease.get("schema") != CLAIM_SCHEMA + or self.recovery_required(lease) + or self._holder_expired(lease) + or not self._receipt_matches(key, old, lease) + ): + return False + if not self._cas_push(key, old, ""): return False - return self._cas_push(key, old, "") + self._clear_receipt(key, expected=old) + return True def holds(self, key: str) -> bool: - """Return whether this worker verifiably owns the current live lease.""" - lease = self.read(key) - return bool( - lease is not None - and self._lease_is_valid(lease, key) - and lease.get("owner") == self.worker_id - and not self.expired(lease) + """Return whether this session has the exact receipt for the live lease.""" + return self.held_claim_oid(key) is not None + + def held_claim_oid(self, key: str) -> str | None: + """Return the exact live claim commit owned by this session, or ``None``. + + Callers must still use this object ID as a remote compare-and-swap lease. + Ownership can change immediately after this point-in-time validation. + """ + fence = self.held_claim_fence(key) + return fence.oid if fence is not None else None + + def held_lease_id(self, key: str) -> str | None: + """Return the fenced lease id held by this session, or ``None``.""" + fence = self.held_claim_fence(key) + return fence.lease_id if fence is not None else None + + def held_claim_fence(self, key: str) -> ClaimFence | None: + """Return one coherent ref/OID/lease receipt for this session's live claim.""" + + held = self._held_claim(key) + if held is None: + return None + oid, lease = held + return ClaimFence( + key=key, + ref=self._ref(key), + oid=oid, + lease_id=str(lease["lease_id"]), ) + def _held_claim(self, key: str) -> tuple[str, dict[str, Any]] | None: + key = _validate_key(key) + self._ensure_scratch() + if self._legacy_author_claim_blocks_v2(key): + return None + oid = self._remote_oid(key) + if oid is None: + return None + lease = self._read_lease(key, oid) + if ( + lease.get("schema") != CLAIM_SCHEMA + or self.recovery_required(lease) + or self._holder_expired(lease) + or not self._receipt_matches(key, oid, lease) + ): + return None + return oid, lease + + def _receipt_matches(self, key: str, oid: str, lease: Mapping[str, Any]) -> bool: + """Return whether this session recorded this exact v2 lease commit.""" + receipt_oid = self._receipt_oid(key) + if receipt_oid != oid or lease.get("schema") != CLAIM_SCHEMA: + return False + receipt = self._read_lease(key, receipt_oid) + return bool(receipt.get("lease_id") == lease.get("lease_id")) + def list(self) -> list[dict[str, Any]]: """Return all claim refs, including malformed and expired entries.""" self._ensure_scratch() - proc = self._git(["ls-remote", self.repo_url, CLAIM_REF_PREFIX + "*"]) + proc = self._remote_git(["ls-remote", self.repo_url, CLAIM_REF_PREFIX + "*"]) leases: list[dict[str, Any]] = [] - for line in proc.stdout.splitlines(): - oid, separator, ref = line.partition("\t") - if not separator or not ref.startswith(CLAIM_REF_PREFIX): - continue + seen_refs: set[str] = set() + for oid, ref in _parse_ls_remote_output(proc.stdout): + self._verify_object_id_format(oid) + if not ref.startswith(CLAIM_REF_PREFIX) or ref in seen_refs: + raise ClaimTransportError( + "claim board returned an unexpected or duplicate claim ref" + ) + seen_refs.add(ref) key = ref[len(CLAIM_REF_PREFIX) :] try: _validate_key(key) - except ValueError: - continue + except ValueError as exc: + raise ClaimTransportError( + f"claim board returned invalid claim ref {ref!r}" + ) from exc try: lease = dict(self._read_lease(key, oid)) except MalformedLeaseError as exc: @@ -327,21 +1484,55 @@ def list(self) -> list[dict[str, Any]]: } else: lease["_malformed"] = False + lease["_legacy"] = lease.get("schema") == LEGACY_CLAIM_SCHEMA + lease["_legacy_block"] = lease.get("schema") == LEGACY_BLOCK_SCHEMA lease["_key"] = key lease["_oid"] = oid lease["_expired"] = not lease["_malformed"] and self.expired(lease) + lease["_recovery_required"] = not lease["_malformed"] and self.recovery_required( + lease + ) leases.append(lease) return sorted(leases, key=lambda lease: str(lease["_key"])) - def cleanup(self) -> int: - """CAS-delete leases expired at snapshot time and return the deletion count.""" - removed = 0 - for lease in self.list(): - if not lease["_malformed"] and lease["_expired"] and self._cas_push( - str(lease["_key"]), str(lease["_oid"]), "" + def cleanup(self, *, canonical_keys: Iterable[str] | None = None) -> int: + """CAS-recover expired or unsafe leases and return the changed-ref count.""" + protected = ( + None + if canonical_keys is None + else {_validate_key(key) for key in canonical_keys} + ) + leases = self.list() + if protected is None and any( + lease.get("schema") == LEGACY_CLAIM_SCHEMA + and str(lease["_key"]).startswith("author/") + and (lease["_expired"] or lease["_recovery_required"]) + for lease in leases + ): + raise ValueError( + "a blueprint is required to recover legacy author claims without " + "blocking durable article IDs" + ) + recovered = 0 + for lease in leases: + if lease["_malformed"] or not ( + lease["_expired"] or lease["_recovery_required"] ): - removed += 1 - return removed + continue + key = str(lease["_key"]) + old = str(lease["_oid"]) + if lease.get("schema") == LEGACY_CLAIM_SCHEMA and key.startswith("author/"): + assert protected is not None + new = ( + "" + if key in protected + else self._make_legacy_block_commit(key, "legacy-rollout") + ) + else: + new = "" + if self._cas_push(key, old, new): + recovered += 1 + return recovered def gc(self) -> int: """Compatibility alias for :meth:`cleanup`.""" @@ -380,6 +1571,7 @@ def __init__( self.ttl = ttl self.lost = threading.Event() self.error: Exception | None = None + self.lease_id: str | None = None self._stop = threading.Event() self._thread: threading.Thread | None = None @@ -387,7 +1579,12 @@ def __enter__(self) -> Heartbeat: if self._thread is not None: raise RuntimeError("heartbeat cannot be started more than once") try: - renewed = self.board.renew(self.key, ttl=self.ttl) + self.lease_id = self.board.held_lease_id(self.key) + renewed = self.lease_id is not None and self.board.renew( + self.key, + ttl=self.ttl, + lease_id=self.lease_id, + ) except Exception as exc: self.error = exc self.lost.set() @@ -402,12 +1599,16 @@ def __enter__(self) -> Heartbeat: def __exit__(self, *exc: object) -> None: self._stop.set() if self._thread is not None: - self._thread.join(timeout=5) + self._thread.join() def _run(self) -> None: while not self._stop.wait(self.interval): try: - renewed = self.board.renew(self.key, ttl=self.ttl) + renewed = self.board.renew( + self.key, + ttl=self.ttl, + lease_id=self.lease_id, + ) except Exception as exc: self.error = exc self.lost.set() @@ -420,12 +1621,25 @@ def _run(self) -> None: __all__ = [ "CLAIM_HEARTBEAT_S", "CLAIM_KEY_RE", + "CLAIM_CLOCK_SKEW_S", + "CLAIM_MAX_TTL_S", + "CLAIM_RECEIPT_REF_PREFIX", "CLAIM_REF_PREFIX", "CLAIM_SCHEMA", "CLAIM_TTL_S", "ClaimBoard", + "ClaimFence", "ClaimTransportError", "Heartbeat", + "LEGACY_CLAIM_SCHEMA", + "LEGACY_BLOCK_SCHEMA", + "LEASE_ID_RE", "MalformedLeaseError", "author_claim_key", + "claim_repository_is_remote", + "normalize_claim_repository", + "pin_claim_repository", + "pin_claim_scratch", + "resource_claim_key", + "workspace_author_claim_key", ] diff --git a/autoform_cli/coverage.py b/autoform_cli/coverage.py index 763cca50..0eaa41af 100644 --- a/autoform_cli/coverage.py +++ b/autoform_cli/coverage.py @@ -10,12 +10,17 @@ import hashlib import json +import os import re +import stat from collections import Counter -from dataclasses import asdict, dataclass -from pathlib import Path +from collections.abc import Iterable +from dataclasses import asdict, dataclass, field +from pathlib import Path, PurePosixPath, PureWindowsPath from urllib.parse import unquote, urlsplit +from .graph import GraphValidationError, SOURCE_UNIT_PATTERN, load_graph + from .markdown import ( INLINE_CODE, Content, @@ -28,9 +33,31 @@ ) COVERAGE_SCHEMA = "autoform-coverage/v1" +COVERAGE_V2_SCHEMA = "autoform-coverage/v2" COVERAGE_DISPOSITIONS = ("MAPPED", "DECOMPOSED", "DEFERRED", "OUT") _EXPECTED_HEADER = ("Area", "Coverage", "Evidence") +_V2_EXPECTED_HEADER = ( + "Unit", + "Area", + "Lines", + "Locator", + "Unit SHA-256", + "Coverage", + "Evidence", +) _SEPARATOR = re.compile(r"^:?-{3,}:?$") +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") +_LINE_SPAN = re.compile(r"([1-9][0-9]*)-([1-9][0-9]*)\Z") +_FRONTMATTER_LIKE_FENCE = re.compile( + r"(?:--(?:[ \t]*(?:ya?ml|#.*))?|-{3,}.*)\Z", + re.IGNORECASE, +) +_FRONTMATTER_KEY = re.compile(r"^[ \t]*[\"']?(?P[A-Za-z][A-Za-z0-9_-]*)[\"']?") +_YAML_HEX_ESCAPE = re.compile( + r"\\U(?P[0-9A-Fa-f]{8})|" + r"\\u(?P[0-9A-Fa-f]{4})|" + r"\\x(?P[0-9A-Fa-f]{2})" +) #: Stem of the marker that stands in for a row's cells when tracing which #: published table those source lines became. Grown by `_unique_marker` until @@ -49,6 +76,7 @@ class CoverageIssue: line: int reason: str + code: str = "invalid-coverage-contract" @dataclass(frozen=True, slots=True) @@ -64,6 +92,47 @@ def as_dict(self) -> dict[str, int | str]: return asdict(self) +@dataclass(frozen=True, slots=True) +class CoverageUnit: + """One exact, LF-terminated span in a v2 source artifact.""" + + unit: str + area: str + start_line: int + end_line: int + locator: str + unit_sha256: str + disposition: str + evidence: str + line: int + roadmap_nodes: tuple[str, ...] = () + + def as_dict(self) -> dict[str, object]: + return { + "area": self.area, + "coverage": self.disposition, + "end_line": self.end_line, + "evidence": self.evidence, + "line": self.line, + "locator": self.locator, + "roadmap_nodes": list(self.roadmap_nodes), + "start_line": self.start_line, + "unit": self.unit, + "unit_sha256": self.unit_sha256, + } + + +@dataclass(frozen=True, order=True, slots=True) +class CoverageNodeBinding: + """A reciprocal source-unit to roadmap-leaf binding.""" + + node_id: str + unit: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + @dataclass(frozen=True, slots=True) class CoverageSummary: """Canonical coverage rows, counts, and source binding. @@ -77,6 +146,11 @@ class CoverageSummary: source_path: str source_sha256: str entries: tuple[CoverageEntry, ...] + artifact_path: str | None = None + artifact_sha256: str | None = None + units: tuple[CoverageUnit, ...] = () + node_bindings: tuple[CoverageNodeBinding, ...] = () + _roadmap_sha256: str | None = field(default=None, repr=False, compare=False) @property def counts(self) -> dict[str, int]: @@ -101,7 +175,7 @@ def complete(self) -> bool: return bool(self.entries) and not self.counts["MAPPED"] def as_dict(self) -> dict[str, object]: - return { + result: dict[str, object] = { "complete": self.complete, "counts": self.counts, "entries": [entry.as_dict() for entry in self.entries], @@ -109,6 +183,17 @@ def as_dict(self) -> dict[str, object]: "source_path": self.source_path, "source_sha256": self.source_sha256, } + if self.schema == COVERAGE_V2_SCHEMA: + result.update( + { + "artifact_path": self.artifact_path, + "artifact_sha256": self.artifact_sha256, + "contract_sha256": self.source_sha256, + "node_bindings": [binding.as_dict() for binding in self.node_bindings], + "units": [unit.as_dict() for unit in self.units], + } + ) + return result def to_json(self) -> str: return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) @@ -123,12 +208,69 @@ def load_coverage(blueprint_dir: str | Path) -> tuple[CoverageSummary | None, tu content = path.read_bytes() text = content.decode("utf-8") except FileNotFoundError: - return None, (CoverageIssue(0, "coverage contract is missing"),) + return None, ( + CoverageIssue(0, "coverage contract is missing", "missing-coverage-contract"), + ) except UnicodeError: return None, (CoverageIssue(0, "coverage contract cannot be read as UTF-8"),) except OSError: return None, (CoverageIssue(0, "coverage contract cannot be read"),) + schema_values, frontmatter, frontmatter_end, frontmatter_issues = _coverage_frontmatter(text) + ambiguous_schema_line = _ambiguous_v2_schema_line(text) + if frontmatter_issues and (schema_values or ambiguous_schema_line is not None): + return None, tuple(frontmatter_issues) + if schema_values: + if len(schema_values) != 1: + return None, ( + CoverageIssue( + schema_values[1][0], + "coverage contract declares more than one schema", + "coverage-schema-mixed", + ), + ) + schema_line, schema = schema_values[0] + if schema != COVERAGE_V2_SCHEMA: + return None, ( + CoverageIssue( + schema_line, + f"unsupported coverage schema {schema!r}", + "coverage-schema-unknown", + ), + ) + return _load_coverage_v2( + blueprint, + path, + content, + text, + frontmatter, + frontmatter_end, + ) + + if ambiguous_schema_line is not None: + return None, ( + CoverageIssue( + ambiguous_schema_line, + "autoform-coverage/v2 appears in malformed or unsupported coverage frontmatter", + "coverage-schema-ambiguous", + ), + ) + + published_headers = {table.headers for table in published_tables(text)} + if _V2_EXPECTED_HEADER in published_headers: + code = ( + "coverage-schema-mixed" + if _EXPECTED_HEADER in published_headers + else "coverage-v2-schema-required" + ) + return None, ( + CoverageIssue( + 0, + "a rendered v2 coverage table requires exact 'schema: autoform-coverage/v2' frontmatter", + code, + ), + ) + rows, issues = _parse_table(text) issues.extend(_validate_evidence(rows, blueprint=blueprint, coverage_path=path)) if issues: @@ -144,6 +286,749 @@ def load_coverage(blueprint_dir: str | Path) -> tuple[CoverageSummary | None, tu ) +def _coverage_frontmatter( + text: str, +) -> tuple[list[tuple[int, str]], dict[str, tuple[int, str]], int, list[CoverageIssue]]: + """Read the intentionally small coverage frontmatter language. + + V1 remains schema-less. The presence of any ``schema`` declaration opts + into strict schema selection, so a typo or two competing declarations can + never be interpreted as the legacy contract. + """ + + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return [], {}, 0, [] + try: + end = next(index for index in range(1, len(lines)) if lines[index].strip() == "---") + except StopIteration: + return [], {}, len(lines), [ + CoverageIssue(1, "coverage frontmatter is unterminated", "coverage-frontmatter-invalid") + ] + + schemas: list[tuple[int, str]] = [] + values: dict[str, tuple[int, str]] = {} + issues: list[CoverageIssue] = [] + for line_number, raw in enumerate(lines[1:end], start=2): + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + if ":" not in stripped: + issues.append( + CoverageIssue( + line_number, + "expected 'key: value' in coverage frontmatter", + "coverage-frontmatter-invalid", + ) + ) + continue + key, value = (part.strip() for part in stripped.split(":", 1)) + value = _unquote_frontmatter_scalar(value) + if key == "schema": + schemas.append((line_number, value)) + if key in values: + continue + if key in values: + issues.append( + CoverageIssue( + line_number, + f"duplicate coverage frontmatter key {key!r}", + "coverage-frontmatter-duplicate-key", + ) + ) + continue + values[key] = (line_number, value) + return schemas, values, end + 1, issues + + +def _ambiguous_v2_schema_line(text: str) -> int | None: + """Find v2 intent inside a frontmatter block we could not select. + + Exact frontmatter is strict about the selector's spelling, quoting, and + separator. Detection is deliberately broader only inside that block: any + un-commented v2 schema token means a malformed selector must not downgrade + to permissive legacy v1. A near frontmatter fence at the start gets the same + treatment, while prose and fenced examples in the Markdown body do not. + """ + + lines = text.splitlines() + if not lines: + return None + opening = lines[0].lstrip("\ufeff").strip() + if _FRONTMATTER_LIKE_FENCE.fullmatch(opening) is None: + return None + try: + end = next(index for index in range(1, len(lines)) if lines[index].strip() == "---") + except StopIteration: + end = len(lines) + for line_number, line in enumerate(lines[1:end], start=2): + stripped = line.strip() + if stripped.startswith("#"): + continue + if _frontmatter_line_signals_v2(stripped): + return line_number + return None + + +def _frontmatter_line_signals_v2(line: str) -> bool: + def decode_hex_escape(match: re.Match[str]) -> str: + value = match.group("short") or match.group("long") or match.group("byte") + assert value is not None + try: + return chr(int(value, 16)) + except ValueError: + return match.group(0) + + folded = _YAML_HEX_ESCAPE.sub(decode_hex_escape, line).casefold() + if COVERAGE_V2_SCHEMA.casefold() in folded: + return True + normalized = re.sub(r"[\s\"'\\]", "", folded).replace("_", "-") + if COVERAGE_V2_SCHEMA.casefold() in normalized: + return True + key_match = _FRONTMATTER_KEY.match(folded) + return ( + key_match is not None + and _within_one_schema_edit(key_match.group("key").casefold()) + and "autoform-coverage/" in normalized + ) + + +def _within_one_schema_edit(value: str) -> bool: + expected = "schema" + if value == expected: + return True + if abs(len(value) - len(expected)) > 1: + return False + if len(value) == len(expected): + differences = [ + index + for index, pair in enumerate(zip(value, expected)) + if pair[0] != pair[1] + ] + if len(differences) == 1: + return True + return ( + len(differences) == 2 + and differences[1] == differences[0] + 1 + and value[differences[0]] == expected[differences[1]] + and value[differences[1]] == expected[differences[0]] + ) + shorter, longer = (value, expected) if len(value) < len(expected) else (expected, value) + mismatch = next( + (index for index, pair in enumerate(zip(shorter, longer)) if pair[0] != pair[1]), + len(shorter), + ) + return shorter[mismatch:] == longer[mismatch + 1 :] + + +def _unquote_frontmatter_scalar(value: str) -> str: + if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: + return value[1:-1] + return value + + +def _load_coverage_v2( + blueprint: Path, + path: Path, + contract_bytes: bytes, + text: str, + frontmatter: dict[str, tuple[int, str]], + frontmatter_end: int, +) -> tuple[CoverageSummary | None, tuple[CoverageIssue, ...]]: + issues: list[CoverageIssue] = [] + allowed = {"schema", "artifact", "artifact_sha256"} + for key, (line, _) in frontmatter.items(): + if key not in allowed: + issues.append( + CoverageIssue( + line, + f"unsupported v2 coverage frontmatter key {key!r}", + "coverage-frontmatter-unknown-key", + ) + ) + for key in ("artifact", "artifact_sha256"): + if key not in frontmatter: + issues.append( + CoverageIssue( + 1, + f"v2 coverage frontmatter is missing {key!r}", + f"coverage-{key.replace('_', '-')}-missing", + ) + ) + if issues: + return None, tuple(issues) + + artifact_line, artifact_value = frontmatter["artifact"] + hash_line, declared_artifact_hash = frontmatter["artifact_sha256"] + artifact_relative, artifact_issue = _artifact_relative_path(artifact_value) + if artifact_issue is not None: + return None, (CoverageIssue(artifact_line, artifact_issue, "coverage-artifact-path-invalid"),) + if _SHA256.fullmatch(declared_artifact_hash) is None: + return None, ( + CoverageIssue( + hash_line, + "artifact_sha256 must be exactly 64 lowercase hexadecimal characters", + "coverage-artifact-hash-invalid", + ), + ) + assert artifact_relative is not None + artifact_path = blueprint.joinpath(*artifact_relative.parts) + artifact_bytes, read_issue = _read_source_artifact(blueprint, artifact_path) + if read_issue is not None: + return None, (CoverageIssue(artifact_line, read_issue[1], read_issue[0]),) + assert artifact_bytes is not None + format_issue = _canonical_artifact_issue(artifact_bytes) + if format_issue is not None: + return None, (CoverageIssue(artifact_line, format_issue[1], format_issue[0]),) + actual_artifact_hash = hashlib.sha256(artifact_bytes).hexdigest() + if actual_artifact_hash != declared_artifact_hash: + return None, ( + CoverageIssue( + hash_line, + "artifact_sha256 does not match the named source artifact", + "coverage-artifact-hash-stale", + ), + ) + + units, table_issues = _parse_v2_table(text, frontmatter_end) + issues.extend(table_issues) + if not table_issues: + issues.extend(_validate_unit_partition(units, artifact_bytes)) + bindings: tuple[CoverageNodeBinding, ...] = () + roadmap_sha256: str | None = None + if not issues: + units, bindings, roadmap_sha256, binding_issues = _validate_v2_bindings( + units, + blueprint=blueprint, + coverage_path=path, + ) + issues.extend(binding_issues) + if issues: + return None, tuple(issues) + + entries = tuple( + CoverageEntry(unit.area, unit.disposition, unit.evidence, unit.line) for unit in units + ) + return ( + CoverageSummary( + schema=COVERAGE_V2_SCHEMA, + source_path="coverage/README.md", + source_sha256=hashlib.sha256(contract_bytes).hexdigest(), + entries=entries, + artifact_path=artifact_relative.as_posix(), + artifact_sha256=actual_artifact_hash, + units=tuple(units), + node_bindings=bindings, + _roadmap_sha256=roadmap_sha256, + ), + (), + ) + + +def _artifact_relative_path(value: str) -> tuple[PurePosixPath | None, str | None]: + windows = PureWindowsPath(value) + path = PurePosixPath(value) + if ( + not value + or "\\" in value + or path.is_absolute() + or windows.is_absolute() + or path.parts[:1] != ("sources",) + or len(path.parts) < 2 + or any(part in {"", ".", ".."} for part in path.parts) + ): + return None, "artifact must be a portable relative file below sources/" + return path, None + + +def _read_source_artifact( + blueprint: Path, artifact: Path +) -> tuple[bytes | None, tuple[str, str] | None]: + """Read one regular artifact without following a symlink in its path.""" + + try: + relative = artifact.relative_to(blueprint) + current = blueprint + identities: list[tuple[Path, tuple[int, int]]] = [] + for part in relative.parts[:-1]: + current /= part + metadata = current.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + return None, ( + "coverage-artifact-symlink", + "source artifact path contains a symbolic link or non-directory component", + ) + identities.append((current, (metadata.st_dev, metadata.st_ino))) + final_metadata = artifact.lstat() + if stat.S_ISLNK(final_metadata.st_mode): + return None, ( + "coverage-artifact-symlink", + "source artifact is a symbolic link", + ) + if not stat.S_ISREG(final_metadata.st_mode): + return None, ( + "coverage-artifact-not-regular", + "source artifact is not a regular file", + ) + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(artifact, flags) + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + return None, ( + "coverage-artifact-not-regular", + "source artifact is not a regular file", + ) + stream = os.fdopen(descriptor, "rb", buffering=0, closefd=False) + try: + artifact_bytes = stream.read() + finally: + stream.close() + after = os.fstat(descriptor) + finally: + os.close(descriptor) + current_metadata = artifact.lstat() + if stat.S_ISLNK(current_metadata.st_mode) or ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) or (after.st_dev, after.st_ino) != ( + current_metadata.st_dev, + current_metadata.st_ino, + ): + return None, ( + "coverage-artifact-changed", + "source artifact changed while it was read", + ) + for parent, identity in identities: + metadata = parent.lstat() + if stat.S_ISLNK(metadata.st_mode) or (metadata.st_dev, metadata.st_ino) != identity: + return None, ( + "coverage-artifact-changed", + "source artifact path changed while it was read", + ) + return artifact_bytes, None + except FileNotFoundError: + return None, ("coverage-artifact-missing", "source artifact does not exist") + except OSError: + return None, ("coverage-artifact-unreadable", "source artifact cannot be read safely") + + +def _canonical_artifact_issue(data: bytes) -> tuple[str, str] | None: + if not data: + return "coverage-artifact-empty", "source artifact is empty" + if data.startswith(b"\xef\xbb\xbf"): + return "coverage-artifact-bom", "source artifact must not contain a UTF-8 BOM" + if b"\x00" in data: + return "coverage-artifact-nul", "source artifact contains a NUL byte" + if b"\r" in data: + return "coverage-artifact-cr", "source artifact must use LF line endings" + if not data.endswith(b"\n"): + return "coverage-artifact-final-lf", "source artifact must end with LF" + try: + decoded = data.decode("utf-8") + except UnicodeDecodeError: + return "coverage-artifact-utf8", "source artifact is not canonical UTF-8" + if decoded.encode("utf-8") != data: + return "coverage-artifact-utf8", "source artifact is not canonical UTF-8" + return None + + +def _parse_v2_table(text: str, frontmatter_end: int) -> tuple[list[CoverageUnit], list[CoverageIssue]]: + view = content(text) + lines = view.lines + source_lines = text.splitlines() + header_indexes: list[int] = [] + for index in range(frontmatter_end, len(lines) - 1): + if view.is_hidden(index) or view.is_hidden(index + 1): + continue + if _cells(lines[index]) != _V2_EXPECTED_HEADER: + continue + separator = _cells(lines[index + 1]) + if len(separator) == len(_V2_EXPECTED_HEADER) and all( + _SEPARATOR.fullmatch(cell) for cell in separator + ): + header_indexes.append(index) + page_tables = published_tables(text) + if any(table.headers == _EXPECTED_HEADER for table in page_tables): + return [], [ + CoverageIssue( + 0, + "coverage contract mixes v1 and v2 rendered tables", + "coverage-schema-mixed", + ) + ] + contract_tables = [table for table in page_tables if table.headers == _V2_EXPECTED_HEADER] + if not header_indexes: + return [], [ + CoverageIssue( + 0, + "v2 coverage contract has no 'Unit | Area | Lines | Locator | Unit SHA-256 | Coverage | Evidence' table", + "coverage-table-missing", + ) + ] + if len(header_indexes) == 1 and not contract_tables: + return [], [ + CoverageIssue(header_indexes[0] + 1, "v2 coverage table does not render as a table", "coverage-table-unrendered") + ] + if len(header_indexes) != 1 or len(contract_tables) != 1: + line = header_indexes[1] + 1 if len(header_indexes) > 1 else header_indexes[0] + 1 + return [], [ + CoverageIssue(line, "v2 coverage contract must have exactly one coverage table", "coverage-table-ambiguous") + ] + header_index = header_indexes[0] + units: list[CoverageUnit] = [] + issues: list[CoverageIssue] = [] + seen: dict[str, int] = {} + parsed_rows: list[tuple[str, ...]] = [] + row_indexes: list[int] = [] + for index in range(header_index + 2, len(lines)): + raw = lines[index] + if view.ends_block(index) or not raw.strip(): + break + cells = _cells(raw) + line_number = index + 1 + if len(_cells(source_lines[index])) != len(cells): + issues.append( + CoverageIssue( + line_number, + "an HTML comment changes this v2 coverage row's column layout", + "coverage-row-hidden-layout", + ) + ) + continue + if len(cells) != len(_V2_EXPECTED_HEADER): + issues.append( + CoverageIssue(line_number, "v2 coverage row must have exactly seven columns", "coverage-row-columns") + ) + continue + parsed_rows.append(cells) + row_indexes.append(index) + unit, area, span_text, locator, unit_hash, disposition_text, evidence = cells + disposition = _inline_code(disposition_text).upper() + span_match = _LINE_SPAN.fullmatch(_inline_code(span_text)) + valid = True + if SOURCE_UNIT_PATTERN.fullmatch(unit) is None: + issues.append(CoverageIssue(line_number, f"invalid source unit id {unit!r}", "coverage-unit-id-invalid")) + valid = False + elif unit in seen: + issues.append( + CoverageIssue( + line_number, + f"duplicate source unit id {unit!r}; first declared at line {seen[unit]}", + "coverage-unit-id-duplicate", + ) + ) + valid = False + else: + seen[unit] = line_number + for label, value in (("area", area), ("locator", locator), ("evidence", evidence)): + visible = _visible_markdown(value) + if not _has_substance(visible): + issues.append( + CoverageIssue( + line_number, + f"coverage {label} has no substantive visible text", + f"coverage-{label}-empty", + ) + ) + valid = False + elif label == "evidence" and _is_placeholder(visible): + issues.append(CoverageIssue(line_number, "coverage evidence is a placeholder", "coverage-evidence-placeholder")) + valid = False + if span_match is None: + issues.append( + CoverageIssue( + line_number, + "source line span must use inclusive one-based START-END syntax", + "coverage-unit-span-invalid", + ) + ) + valid = False + start_line = end_line = 0 + else: + start_line, end_line = (int(value) for value in span_match.groups()) + if end_line < start_line: + issues.append(CoverageIssue(line_number, "source line span ends before it starts", "coverage-unit-span-invalid")) + valid = False + if _SHA256.fullmatch(unit_hash) is None: + issues.append( + CoverageIssue( + line_number, + "unit SHA-256 must be exactly 64 lowercase hexadecimal characters", + "coverage-unit-hash-invalid", + ) + ) + valid = False + if disposition not in COVERAGE_DISPOSITIONS: + issues.append( + CoverageIssue( + line_number, + f"unknown coverage disposition {disposition_text!r}", + "coverage-disposition-invalid", + ) + ) + valid = False + if valid: + units.append( + CoverageUnit( + unit, + area, + start_line, + end_line, + locator, + unit_hash, + disposition, + evidence, + line_number, + ) + ) + if not units and not issues: + issues.append(CoverageIssue(header_index + 1, "v2 coverage table has no rows", "coverage-table-empty")) + if not issues: + issues.extend( + _correlation_issues( + source_lines, + contract_tables[0], + page_tables, + row_indexes, + parsed_rows, + header_index, + ) + ) + return units, issues + + +def _validate_unit_partition(units: list[CoverageUnit], artifact: bytes) -> list[CoverageIssue]: + issues: list[CoverageIssue] = [] + source_lines = artifact.splitlines(keepends=True) + expected_start = 1 + for unit in units: + if unit.start_line > expected_start: + issues.append( + CoverageIssue( + unit.line, + f"source coverage has a gap before line {unit.start_line}", + "coverage-unit-gap", + ) + ) + elif unit.start_line < expected_start: + issues.append( + CoverageIssue( + unit.line, + f"source coverage overlaps or is out of order at line {unit.start_line}", + "coverage-unit-overlap", + ) + ) + if unit.end_line > len(source_lines): + issues.append( + CoverageIssue( + unit.line, + f"source line span exceeds the artifact's {len(source_lines)} lines", + "coverage-unit-bounds", + ) + ) + elif unit.start_line <= unit.end_line: + actual = hashlib.sha256( + b"".join(source_lines[unit.start_line - 1 : unit.end_line]) + ).hexdigest() + if actual != unit.unit_sha256: + issues.append( + CoverageIssue( + unit.line, + f"unit SHA-256 is stale for {unit.unit!r}", + "coverage-unit-hash-stale", + ) + ) + expected_start = max(expected_start, unit.end_line + 1) + if units and expected_start <= len(source_lines): + issues.append( + CoverageIssue( + units[-1].line, + f"source coverage stops at line {expected_start - 1} of {len(source_lines)}", + "coverage-unit-gap", + ) + ) + return issues + + +def _validate_v2_bindings( + units: list[CoverageUnit], + *, + blueprint: Path, + coverage_path: Path, +) -> tuple[ + list[CoverageUnit], + tuple[CoverageNodeBinding, ...], + str | None, + list[CoverageIssue], +]: + issues: list[CoverageIssue] = [] + try: + graph = load_graph(blueprint) + except GraphValidationError as error: + return ( + units, + (), + None, + [ + CoverageIssue( + 0, + f"roadmap cannot be validated for source bindings: {reason}", + "coverage-roadmap-invalid", + ) + for reason in error.issues + ], + ) + roadmap_sha256 = _roadmap_source_provenance( + ( + node.path.resolve().relative_to(graph.blueprint_dir.resolve()).as_posix(), + node.source_sha256 or "", + ) + for node in graph.nodes.values() + ) + by_path = {node.path.resolve(): node for node in graph.nodes.values()} + units_by_id = {unit.unit: unit for unit in units} + expected: set[tuple[str, str]] = set() + updated: list[CoverageUnit] = [] + for unit in units: + node_ids: list[str] = [] + targets = link_targets(unit.evidence) if unit.disposition == "DECOMPOSED" else () + if unit.disposition == "DECOMPOSED" and not targets: + issues.append( + CoverageIssue( + unit.line, + "DECOMPOSED coverage evidence must link to at least one roadmap leaf", + "coverage-decomposed-target-missing", + ) + ) + seen_targets: set[str] = set() + for target in targets: + split = urlsplit(target) + raw_path = unquote(split.path) + if split.scheme or split.netloc or not raw_path: + issues.append( + CoverageIssue( + unit.line, + f"DECOMPOSED evidence target is not a local roadmap article: {target!r}", + "coverage-decomposed-target-outside-roadmap", + ) + ) + continue + problem = local_target_issue(coverage_path, target, blueprint, label="coverage") + if problem is not None: + issues.append(CoverageIssue(unit.line, problem[1], "coverage-decomposed-target-invalid")) + continue + candidate = (coverage_path.parent / raw_path).resolve() + node = by_path.get(candidate) + if node is None: + issues.append( + CoverageIssue( + unit.line, + f"DECOMPOSED evidence target is outside blueprint/roadmap: {target!r}", + "coverage-decomposed-target-outside-roadmap", + ) + ) + continue + if node.id in seen_targets: + issues.append( + CoverageIssue( + unit.line, + f"duplicate source-unit mapping to roadmap node {node.id!r}", + "coverage-node-binding-duplicate", + ) + ) + continue + seen_targets.add(node.id) + if not node.formalizable or graph.children(node.id): + issues.append( + CoverageIssue( + unit.line, + f"DECOMPOSED evidence target {node.id!r} is not a formalizable roadmap leaf", + "coverage-decomposed-target-not-leaf", + ) + ) + continue + node_ids.append(node.id) + expected.add((unit.unit, node.id)) + updated.append( + CoverageUnit( + unit.unit, + unit.area, + unit.start_line, + unit.end_line, + unit.locator, + unit.unit_sha256, + unit.disposition, + unit.evidence, + unit.line, + tuple(sorted(node_ids)), + ) + ) + + authored: set[tuple[str, str]] = set() + for node in graph.nodes.values(): + for unit_id in node.source_units: + if unit_id not in units_by_id: + issues.append( + CoverageIssue( + 0, + f"roadmap node {node.id!r} names unknown source unit {unit_id!r}", + "coverage-node-binding-unknown-unit", + ) + ) + continue + if not node.formalizable or graph.children(node.id): + issues.append( + CoverageIssue( + 0, + f"roadmap node {node.id!r} binds source units but is not a formalizable leaf", + "coverage-node-binding-not-leaf", + ) + ) + authored.add((unit_id, node.id)) + for unit_id, node_id in sorted(expected - authored): + issues.append( + CoverageIssue( + units_by_id[unit_id].line, + f"roadmap node {node_id!r} does not reciprocally list source unit {unit_id!r}", + "coverage-node-binding-missing-reciprocal", + ) + ) + for unit_id, node_id in sorted(authored - expected): + issues.append( + CoverageIssue( + 0, + f"roadmap node {node_id!r} lists source unit {unit_id!r} without reciprocal DECOMPOSED evidence", + "coverage-node-binding-one-way", + ) + ) + bindings = tuple( + CoverageNodeBinding(node_id=node_id, unit=unit_id) + for unit_id, node_id in sorted(expected & authored) + ) + return updated, bindings, roadmap_sha256, issues + + +def _roadmap_source_provenance(sources: Iterable[tuple[str, str]]) -> str: + """Identify exact roadmap bytes without retaining another source copy.""" + + digest = hashlib.sha256(b"autoform-roadmap-provenance/v1\0") + for relative, source_sha256 in sorted(sources): + encoded_path = os.fsencode(relative) + encoded_sha256 = source_sha256.encode("ascii") + digest.update(len(encoded_path).to_bytes(8, "big")) + digest.update(encoded_path) + digest.update(encoded_sha256) + return digest.hexdigest() + + def _parse_table(text: str) -> tuple[list[CoverageEntry], list[CoverageIssue]]: # Only published Markdown can carry the contract. Commented-out and # code-block tables are masked to blank lines first, which keeps every @@ -314,7 +1199,7 @@ def _correlation_issues( for position, index in enumerate(row_indexes): token = f"{marker}{position}" markers.append(token) - marked[index] = f"| {token} | {token} | {token} |" + marked[index] = f"| {' | '.join(token for _ in published.headers)} |" untraceable = [ CoverageIssue( header_index + 1, @@ -609,8 +1494,11 @@ def _inline_code(value: str) -> str: __all__ = [ "COVERAGE_DISPOSITIONS", "COVERAGE_SCHEMA", + "COVERAGE_V2_SCHEMA", "CoverageEntry", "CoverageIssue", + "CoverageNodeBinding", "CoverageSummary", + "CoverageUnit", "load_coverage", ] diff --git a/autoform_cli/doctor.py b/autoform_cli/doctor.py index 43b2440a..a53cfdbf 100644 --- a/autoform_cli/doctor.py +++ b/autoform_cli/doctor.py @@ -13,17 +13,17 @@ from dataclasses import asdict, dataclass from pathlib import Path, PureWindowsPath -from .audit import AuditFinding, audit_graph -from .graph import GraphValidationError, load_graph +from .audit import AuditFinding, load_audit_graph +from .lean import snapshot_project_sources from .runtime import ( RUNTIME_AUTHORITY, RUNTIME_SCHEMA, - RuntimeGraph, RuntimePaths, RuntimeProjectionError, build_runtime_graph, resolve_runtime_paths, ) +from .workspace_manifest import WorkspaceError _LEAN_FINDING_CODES = frozenset( { @@ -70,16 +70,19 @@ def diagnose_project( project_or_blueprint: str | Path, *, lean_root: str | Path | None = None, + project_id: str | None = None, ) -> DoctorResult: """Diagnose one project using only its local authored source files.""" paths: RuntimePaths | None = None - graph = None - runtime: RuntimeGraph | None = None checks: list[DoctorCheck] = [] try: - paths = resolve_runtime_paths(project_or_blueprint) + paths = resolve_runtime_paths( + project_or_blueprint, + project_id=project_id, + _retain_workspace=True, + ) except RuntimeProjectionError as error: checks.append(DoctorCheck("blueprint", False, _issues(error.issues))) except (OSError, RuntimeError, ValueError): @@ -97,9 +100,51 @@ def diagnose_project( return _blocked_result(checks, "blueprint resolution failed", lean_root=lean_root) try: - graph = load_graph(paths.blueprint_dir) - except GraphValidationError as error: - reason = _sanitize_issues(error.issues, paths) + try: + return _diagnose_bound_project(paths, checks, lean_root=lean_root) + except (RuntimeProjectionError, WorkspaceError) as error: + reason = _sanitize_issues(error.issues, paths) + return _blocked_result( + [DoctorCheck("blueprint", False, reason)], + "project changed during diagnosis", + lean_root=lean_root, + ) + finally: + paths.close() + + +def _diagnose_bound_project( + paths: RuntimePaths, + checks: list[DoctorCheck], + *, + lean_root: str | Path | None, +) -> DoctorResult: + """Run diagnostics while retaining one selected filesystem generation.""" + + lean_snapshot = None + lean_root_valid = True + lean_root_error: str | None = None + if lean_root is not None: + try: + lean_snapshot = snapshot_project_sources(lean_root) + except (OSError, RuntimeError, ValueError) as error: + lean_root_valid = False + lean_root_error = _sanitize_issues((str(error),), paths) + + runtime = None + graph, audit = load_audit_graph( + paths.blueprint_dir, + lean_index=lean_snapshot.index if lean_snapshot is not None else None, + _expected_blueprint_identity=paths.blueprint_identity, + _expected_roadmap_identity=paths.roadmap_identity, + ) + if graph is None: + reasons = tuple( + finding.reason + for finding in audit.findings + if finding.code == "invalid-graph" + ) + reason = _sanitize_issues(reasons, paths) if reasons else "canonical graph is invalid" checks.append(DoctorCheck("runtime", False, "canonical graph is invalid")) checks.append(DoctorCheck("graph", False, reason)) checks.append(DoctorCheck("references", False, "not checked because graph validation failed")) @@ -107,12 +152,11 @@ def diagnose_project( checks.append(_blocked_lean_check("graph validation failed", lean_root)) return _result(checks) - resolved_lean_root, lean_root_valid = _resolve_lean_root(lean_root) try: runtime = build_runtime_graph( graph, project_root=paths.project_root, - lean_root=resolved_lean_root, + _lean_index=lean_snapshot.index if lean_snapshot is not None else None, ) except RuntimeProjectionError as error: checks.append(DoctorCheck("runtime", False, _issues(error.issues))) @@ -151,11 +195,14 @@ def diagnose_project( ) ) - audit = audit_graph(graph, lean_root=resolved_lean_root) lean_findings = tuple(finding for finding in audit.findings if finding.code in _LEAN_FINDING_CODES) if lean_root is not None and not lean_root_valid: lean_findings = ( - AuditFinding(".", "invalid-lean-root", "Lean root does not exist or is not a directory"), + AuditFinding( + ".", + "invalid-lean-root", + lean_root_error or "Lean root could not be inspected safely", + ), *lean_findings, ) roadmap_findings = tuple(finding for finding in audit.findings if finding.code not in _LEAN_FINDING_CODES) @@ -164,6 +211,7 @@ def diagnose_project( checks.append(DoctorCheck("lean targets", True, "not checked; no Lean root supplied")) else: checks.append(_finding_check("lean targets", lean_findings, "all asserted local Lean targets resolve")) + paths.verify() return _result(checks) @@ -188,6 +236,12 @@ def _blocked_lean_check(reason: str, lean_root: str | Path | None) -> DoctorChec def _finding_check(name: str, findings: tuple[AuditFinding, ...], clean_detail: str) -> DoctorCheck: if not findings: return DoctorCheck(name, True, clean_detail) + if len(findings) == 1 and findings[0].code == "invalid-lean-root": + return DoctorCheck( + name, + False, + f"1 finding(s): invalid-lean-root: {findings[0].reason}", + ) codes = ", ".join(sorted({finding.code for finding in findings})) return DoctorCheck(name, False, f"{len(findings)} finding(s): {codes}") diff --git a/autoform_cli/execution_input.py b/autoform_cli/execution_input.py new file mode 100644 index 00000000..8c47d6ea --- /dev/null +++ b/autoform_cli/execution_input.py @@ -0,0 +1,1012 @@ +"""Build the immutable input contract for CLI-backed Autoform work.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import tempfile +import unicodedata +from dataclasses import asdict, dataclass, replace +from pathlib import Path, PurePosixPath + +from ._tree_snapshot import ( + TreeSelection, + TreeSnapshot, + TreeSnapshotError, + capture_directory_descriptor, +) + +from .coverage import ( + COVERAGE_V2_SCHEMA, + CoverageSummary, + _artifact_relative_path, + _coverage_frontmatter, + _roadmap_source_provenance, + load_coverage, +) +from .graph import GraphValidationError, load_graph +from .lean import ( + BoundProjectSources, + SourceIndex, + open_project_sources, +) +from .runtime import ( + RuntimeGraph, + RuntimeLeanTarget, + RuntimePaths, + RuntimeProjectionError, + _source_target_relative, + _source_target_walk, + load_runtime_graph, + resolve_runtime_paths, +) +from .workspace import _path_contains_symlink, _path_is_reparse_point +from .workspace_manifest import WorkspaceError + +# V3 replaces V2's whole-manifest workspace digest with a selected-project +# binding. A controller must not reinterpret a V2 digest under the new meaning. +EXECUTION_INPUT_SCHEMA = "autoform-execution-input/v3" +_EXECUTION_INPUT_READ_ATTEMPTS = 3 +_EXECUTION_CONTRACT_ROOTS = frozenset({"coverage", "roadmap"}) + + +def _portable_path_key(path: PurePosixPath) -> tuple[str, ...]: + return tuple(unicodedata.normalize("NFC", part).casefold() for part in path.parts) + + +def _execution_contract_path(path: PurePosixPath) -> bool: + if not path.parts: + return False + root = unicodedata.normalize("NFC", path.parts[0]).casefold() + if root not in _EXECUTION_CONTRACT_ROOTS: + return False + return root != "roadmap" or not any( + part.startswith(".") for part in path.parts[1:] + ) + + +_EXECUTION_CONTRACT_SELECTION = TreeSelection( + include=lambda path, _mode: _execution_contract_path(path), + descend=_execution_contract_path, + record_omitted=False, +) + + +@dataclass(frozen=True, order=True, slots=True) +class ExecutionInputIssue: + """One stable reason a ready-work snapshot could not be built.""" + + code: str + reason: str + + +class ExecutionInputError(ValueError): + """The authored project cannot supply a safe ready-work snapshot.""" + + def __init__(self, issues: tuple[ExecutionInputIssue, ...] | list[ExecutionInputIssue]) -> None: + self.issues = tuple(sorted(set(issues))) + super().__init__("; ".join(f"{issue.code}: {issue.reason}" for issue in self.issues)) + + +@dataclass(frozen=True, slots=True) +class _ExecutionAuthorityRevision: + """Digests needed to prove loader results came from this generation.""" + + sha256: str + runtime_source_revision: str + roadmap_sha256: str + coverage_sha256: str | None + source_sha256s: tuple[tuple[str, str], ...] + lean_source_revision: str | None + + +@dataclass(frozen=True, slots=True) +class ExecutionSourceUnit: + """One source unit copied from the validated v2 coverage contract.""" + + unit: str + area: str + start_line: int + end_line: int + locator: str + unit_sha256: str + disposition: str + evidence: str + roadmap_nodes: tuple[str, ...] + + def as_dict(self) -> dict[str, object]: + result = asdict(self) + result["roadmap_nodes"] = list(self.roadmap_nodes) + return result + + +@dataclass(frozen=True, order=True, slots=True) +class ExecutionNodeBinding: + """One validated reciprocal roadmap binding.""" + + node_id: str + unit: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class ExecutionInput: + """A deterministic snapshot for ready-work discovery and validation.""" + + schema: str + runtime: RuntimeGraph + runtime_sha256: str + coverage_schema: str + coverage_path: str + coverage_sha256: str + artifact_path: str + artifact_sha256: str + units: tuple[ExecutionSourceUnit, ...] + node_bindings: tuple[ExecutionNodeBinding, ...] + authority_sha256: str | None = None + lean_source_revision: str | None = None + workspace_project_id: str | None = None + workspace_project_binding_sha256: str | None = None + + def as_dict(self) -> dict[str, object]: + return { + "artifact": { + "path": self.artifact_path, + "sha256": self.artifact_sha256, + }, + "authority_sha256": self.authority_sha256, + "coverage": { + "path": self.coverage_path, + "schema": self.coverage_schema, + "sha256": self.coverage_sha256, + }, + "node_bindings": [binding.as_dict() for binding in self.node_bindings], + "runtime": self.runtime.as_dict(), + "runtime_sha256": self.runtime_sha256, + "lean_source_revision": self.lean_source_revision, + "schema": self.schema, + "units": [unit.as_dict() for unit in self.units], + "workspace": { + "blueprint_path": self.runtime.blueprint_path, + "project_binding_sha256": self.workspace_project_binding_sha256, + "project_id": self.workspace_project_id, + }, + } + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + @property + def sha256(self) -> str: + return hashlib.sha256(self.to_json().encode("utf-8")).hexdigest() + + @property + def source_contract_sha256(self) -> str: + """Hash the immutable source-coverage contract without progress state.""" + + payload = { + "artifact": {"path": self.artifact_path, "sha256": self.artifact_sha256}, + "coverage": { + "path": self.coverage_path, + "schema": self.coverage_schema, + "sha256": self.coverage_sha256, + }, + "node_bindings": [binding.as_dict() for binding in self.node_bindings], + "units": [unit.as_dict() for unit in self.units], + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def load_execution_input( + project_or_blueprint: str | Path, + *, + lean_root: str | Path | None = None, + project_id: str | None = None, +) -> ExecutionInput: + """Read a stable runtime and exhaustive coverage snapshot, or fail closed.""" + + if lean_root is None: + resolved_lean_root = None + else: + requested_lean_root = Path(os.path.abspath(Path(lean_root).expanduser())) + try: + metadata = requested_lean_root.stat(follow_symlinks=False) + if _path_contains_symlink(requested_lean_root) or _path_is_reparse_point( + requested_lean_root, + metadata, + ): + raise WorkspaceError(["Lean root path contains a symbolic link"]) + resolved_lean_root = requested_lean_root + except (OSError, RuntimeError, ValueError, WorkspaceError): + raise ExecutionInputError( + [ExecutionInputIssue("lean-root-unsafe", "Lean root path cannot be resolved safely")] + ) from None + if not stat.S_ISDIR(metadata.st_mode): + raise ExecutionInputError( + [ExecutionInputIssue("lean-root-unsafe", "Lean root path is not a directory")] + ) + runtime: RuntimeGraph | None = None + coverage: CoverageSummary | None = None + authority: _ExecutionAuthorityRevision | None = None + workspace_project_id: str | None = None + workspace_project_binding_sha256: str | None = None + for _ in range(_EXECUTION_INPUT_READ_ATTEMPTS): + paths = None + final_paths = None + lean_sources: BoundProjectSources | None = None + try: + try: + paths = resolve_runtime_paths( + project_or_blueprint, + project_id=project_id, + _retain_workspace=True, + ) + except (GraphValidationError, RuntimeProjectionError) as error: + raise ExecutionInputError( + [ExecutionInputIssue("runtime-invalid", reason) for reason in error.issues] + ) from error + except OSError: + continue + paths.verify() + if not paths.strongly_bound: + raise ExecutionInputError( + [ + ExecutionInputIssue( + "strong-binding-required", + "ready-work discovery requires descriptor-relative filesystem support", + ) + ] + ) + if paths.workspace_managed and paths.workspace_project_id is None: + raise ExecutionInputError( + [ + ExecutionInputIssue( + "workspace-project-required", + "ready-work discovery requires a registered workspace project", + ) + ] + ) + binding = _runtime_binding(paths) + try: + blueprint_snapshot = _capture_execution_authority(paths) + paths.verify() + except (OSError, TreeSnapshotError): + continue + try: + if resolved_lean_root is not None: + lean_sources = open_project_sources(resolved_lean_root) + lean_snapshot = lean_sources.capture() + else: + lean_snapshot = None + except TreeSnapshotError: + continue + except OSError as error: + raise ExecutionInputError( + [ExecutionInputIssue("lean-source-unsafe", str(error))] + ) from error + authority = _execution_authority_revision( + blueprint_snapshot, + lean_snapshot.revision if lean_snapshot is not None else None, + ) + blueprint_generation = blueprint_snapshot.generation_revision + lean_generation = ( + lean_snapshot.generation_revision if lean_snapshot is not None else None + ) + unsafe_issues = _unsafe_snapshot_issues(blueprint_snapshot) + if unsafe_issues: + if _execution_generation_changed( + paths, + blueprint_generation, + lean_sources, + lean_generation, + ): + continue + raise ExecutionInputError(unsafe_issues) + + deferred_error: ExecutionInputError | None = None + try: + with tempfile.TemporaryDirectory(prefix="autoform-execution-") as temporary: + temporary_project = Path(temporary).resolve() + temporary_blueprint = temporary_project / "blueprint" + blueprint_snapshot.materialize(temporary_blueprint) + runtime = load_runtime_graph( + temporary_blueprint, + lean_root=None, + ) + runtime = _rebase_runtime( + runtime, + paths.blueprint_dir.relative_to(paths.project_root).as_posix(), + ) + runtime = _apply_lean_index( + runtime, + lean_snapshot.index if lean_snapshot is not None else None, + ) + coverage = _require_v2_coverage(temporary_blueprint) + except (GraphValidationError, RuntimeProjectionError) as error: + deferred_error = ExecutionInputError( + [ExecutionInputIssue("runtime-invalid", reason) for reason in error.issues] + ) + except ExecutionInputError as error: + deferred_error = error + except TreeSnapshotError: + deferred_error = ExecutionInputError( + [ + ExecutionInputIssue( + "execution-authority-unsafe", + "execution authority cannot be materialized safely", + ) + ] + ) + except OSError: + deferred_error = ExecutionInputError( + [ + ExecutionInputIssue( + "execution-snapshot-unavailable", + "private execution snapshot could not be materialized safely", + ) + ] + ) + + del blueprint_snapshot + if deferred_error is not None: + if _execution_generation_changed( + paths, + blueprint_generation, + lean_sources, + lean_generation, + ): + continue + raise deferred_error + assert runtime is not None and coverage is not None + if ( + runtime.source_revision != authority.runtime_source_revision + or not _runtime_matches_lean_index( + runtime, + lean_snapshot.index if lean_snapshot is not None else None, + ) + or not _coverage_matches_authority(coverage, authority) + ): + continue + try: + final_paths = resolve_runtime_paths( + project_or_blueprint, + project_id=project_id, + _retain_workspace=True, + ) + except (GraphValidationError, RuntimeProjectionError, OSError): + continue + final_paths.verify() + paths.verify() + final_binding = _runtime_binding(final_paths) + expected_blueprint_path = paths.blueprint_dir.relative_to(paths.project_root).as_posix() + if ( + binding == final_binding + and runtime.blueprint_path == expected_blueprint_path + and not _execution_generation_changed( + paths, + blueprint_generation, + lean_sources, + lean_generation, + ) + ): + workspace_project_id = paths.workspace_project_id + workspace_project_binding_sha256 = paths.workspace_project_binding_sha256 + paths.verify() + final_paths.verify() + break + except (RuntimeProjectionError, WorkspaceError): + continue + finally: + if final_paths is not None: + final_paths.close() + if paths is not None: + paths.close() + if lean_sources is not None: + lean_sources.close() + else: + raise _changed_execution_input() + + assert runtime is not None and coverage is not None and authority is not None + missing_article_ids = tuple(node.id for node in runtime.nodes if node.article_id is None) + if missing_article_ids: + raise ExecutionInputError( + [ + ExecutionInputIssue( + "article-id-required", + "ready-work discovery requires durable article_id frontmatter on every " + f"roadmap article; missing: {', '.join(missing_article_ids)}", + ) + ] + ) + runtime_json = runtime.to_json() + return ExecutionInput( + schema=EXECUTION_INPUT_SCHEMA, + runtime=runtime, + authority_sha256=authority.sha256, + runtime_sha256=hashlib.sha256(runtime_json.encode("utf-8")).hexdigest(), + lean_source_revision=authority.lean_source_revision, + coverage_schema=coverage.schema, + coverage_path=coverage.source_path, + coverage_sha256=coverage.source_sha256, + artifact_path=_required(coverage.artifact_path), + artifact_sha256=_required(coverage.artifact_sha256), + units=tuple( + ExecutionSourceUnit( + unit=unit.unit, + area=unit.area, + start_line=unit.start_line, + end_line=unit.end_line, + locator=unit.locator, + unit_sha256=unit.unit_sha256, + disposition=unit.disposition, + evidence=unit.evidence, + roadmap_nodes=unit.roadmap_nodes, + ) + for unit in coverage.units + ), + node_bindings=tuple( + ExecutionNodeBinding(binding.node_id, binding.unit) + for binding in coverage.node_bindings + ), + workspace_project_id=workspace_project_id, + workspace_project_binding_sha256=workspace_project_binding_sha256, + ) + + +def _changed_execution_input() -> ExecutionInputError: + return ExecutionInputError( + [ + ExecutionInputIssue( + "execution-input-changed", + "runtime or coverage authority kept changing while ready work was read", + ) + ] + ) + + +def _execution_authority_revision( + snapshot: TreeSnapshot, + lean_source_revision: str | None, +) -> _ExecutionAuthorityRevision: + """Identify every runtime and coverage input in one captured generation.""" + + digest = hashlib.sha256(b"autoform-execution-authority/v2\0") + runtime_digest = hashlib.sha256(b"autoform-runtime-source/v1\0") + roadmap_sources: list[tuple[str, str]] = [] + roadmap_bytes: dict[str, bytes] = {} + source_sha256s: list[tuple[str, str]] = [] + coverage_sha256: str | None = None + for relative in snapshot.directories: + if relative: + _update_authority_digest(digest, b"directory", relative, b"") + for relative, file_bytes in snapshot.files: + _update_authority_digest(digest, b"file", relative, file_bytes) + relative_path = PurePosixPath(relative) + portable_path = _portable_path_key(relative_path) + file_sha256 = hashlib.sha256(file_bytes).hexdigest() + if portable_path == ("coverage", "readme.md"): + coverage_sha256 = file_sha256 + if portable_path[:1] == ("sources",): + source_sha256s.append((relative, file_sha256)) + if portable_path[:1] == ("roadmap",) and relative_path.suffix == ".md": + canonical = PurePosixPath("roadmap", *relative_path.parts[1:]).as_posix() + roadmap_sources.append((canonical, file_sha256)) + roadmap_bytes[canonical] = file_bytes + for relative, target in snapshot.symlinks: + _update_authority_digest( + digest, + b"symlink", + relative, + target.encode("utf-8", errors="surrogateescape"), + ) + for relative, mode in snapshot.special: + _update_authority_digest(digest, b"special", relative, str(mode).encode("ascii")) + for relative in snapshot.placeholders: + _update_authority_digest(digest, b"placeholder", relative, b"") + + for relative, _sha256 in sorted( + roadmap_sources, + key=lambda entry: _authority_entry_sort_key(PurePosixPath(entry[0])), + ): + file_bytes = roadmap_bytes[relative] + article_path = os.fsencode(relative) + runtime_digest.update(len(article_path).to_bytes(8, "big")) + runtime_digest.update(article_path) + runtime_digest.update(len(file_bytes).to_bytes(8, "big")) + runtime_digest.update(file_bytes) + if lean_source_revision is not None: + lean_revision = lean_source_revision.encode("ascii") + _update_authority_digest(digest, b"lean", "", lean_revision) + return _ExecutionAuthorityRevision( + sha256=digest.hexdigest(), + runtime_source_revision=runtime_digest.hexdigest(), + roadmap_sha256=_roadmap_source_provenance(roadmap_sources), + coverage_sha256=coverage_sha256, + source_sha256s=tuple(sorted(source_sha256s)), + lean_source_revision=lean_source_revision, + ) + + +def _authority_entry_sort_key(relative: PurePosixPath) -> tuple[int, str]: + """Put roadmap files in the node order used by RuntimeGraph provenance.""" + + if relative.parts[:1] != ("roadmap",) or relative.suffix != ".md": + return 1, relative.as_posix() + article = relative.relative_to("roadmap") + if article.name == "README.md": + node_id = article.parent.as_posix() + if node_id == ".": + node_id = "roadmap" + else: + node_id = article.with_suffix("").as_posix() + return 0, node_id + + +def _capture_execution_authority(paths: RuntimePaths) -> TreeSnapshot: + contract = _capture_execution_tree(paths, _EXECUTION_CONTRACT_SELECTION) + _reject_portable_snapshot_collisions(contract) + artifact = _coverage_artifact_from_snapshot(contract) + source_targets = _local_source_targets_from_snapshot(contract) + bound_artifact = ( + _bind_existing_target_path(paths, artifact) if artifact is not None else None + ) + bound_source_paths = _bind_existing_source_target_paths(paths, source_targets) + selection = _execution_authority_selection(bound_artifact, bound_source_paths) + authority = _capture_execution_tree(paths, selection) + if _execution_contract_projection(authority).generation_revision != ( + contract.generation_revision + ): + raise TreeSnapshotError("execution contract changed while its artifact was selected") + return authority + + +def _capture_execution_tree(paths: RuntimePaths, selection: TreeSelection) -> TreeSnapshot: + descriptor = paths.duplicate_blueprint_descriptor() + try: + return capture_directory_descriptor( + descriptor, + expected_identity=paths.blueprint_identity, + selection=selection, + ) + finally: + os.close(descriptor) + + +def _coverage_artifact_from_snapshot(snapshot: TreeSnapshot) -> PurePosixPath | None: + contracts = [ + data + for relative, data in snapshot.files + if _portable_path_key(PurePosixPath(relative)) == ("coverage", "readme.md") + ] + if len(contracts) > 1: + raise TreeSnapshotError("coverage contract path is portably ambiguous") + if not contracts: + return None + contract = contracts[0] + try: + text = contract.decode("utf-8") + except UnicodeError: + return None + schemas, frontmatter, _end, issues = _coverage_frontmatter(text) + if issues or len(schemas) != 1 or schemas[0][1] != COVERAGE_V2_SCHEMA: + return None + artifact = frontmatter.get("artifact") + if artifact is None: + return None + relative, issue = _artifact_relative_path(artifact[1]) + return relative if issue is None else None + + +def _execution_authority_selection( + artifact: PurePosixPath | None, + source_targets: tuple[PurePosixPath, ...], +) -> TreeSelection: + byte_paths = {artifact} if artifact is not None else set() + validation_paths: set[PurePosixPath] = set() + for path in (*byte_paths, *source_targets): + validation_paths.add(path) + validation_paths.update(parent for parent in path.parents if parent.parts) + + def contract_path(path: PurePosixPath) -> bool: + return _execution_contract_path(path) + + def traversed(path: PurePosixPath) -> bool: + return contract_path(path) or path in validation_paths + + return TreeSelection( + include=lambda path, mode: contract_path(path) + or path in byte_paths + or (path in validation_paths and not stat.S_ISREG(mode)), + descend=traversed, + placeholder=lambda path, mode: path in validation_paths and stat.S_ISREG(mode), + record_omitted=False, + ) + + +def _local_source_targets_from_snapshot( + snapshot: TreeSnapshot, +) -> tuple[PurePosixPath, ...]: + """Return local source paths named by the captured roadmap bytes.""" + + if snapshot.unsupported_entries(): + return () + try: + with tempfile.TemporaryDirectory(prefix="autoform-source-targets-") as temporary: + blueprint = Path(temporary) / "blueprint" + snapshot.materialize(blueprint) + graph = load_graph(blueprint) + targets: set[PurePosixPath] = set() + for node in graph.nodes.values(): + for target in node.sources: + relative = _source_target_relative( + node.path, + target, + graph.blueprint_dir, + ) + walk = _source_target_walk( + node.path, + target, + graph.blueprint_dir, + ) + if isinstance(relative, PurePosixPath): + targets.add(relative) + if isinstance(walk, PurePosixPath): + targets.update(_source_walk_prefixes(walk)) + except (GraphValidationError, OSError, TreeSnapshotError): + return () + return tuple(sorted(targets)) + + +def _source_walk_prefixes(walk: PurePosixPath) -> tuple[PurePosixPath, ...]: + """Retain every path reached before lexical parent components are applied.""" + + parts: list[str] = [] + visited: list[PurePosixPath] = [] + for part in walk.parts: + if part == "..": + if parts: + parts.pop() + continue + parts.append(part) + visited.append(PurePosixPath(*parts)) + return tuple(visited) + + +def _bind_existing_source_target_paths( + paths: RuntimePaths, + targets: tuple[PurePosixPath, ...], +) -> tuple[PurePosixPath, ...]: + """Map local href components to names selected by the host filesystem.""" + + trie: dict[str, dict] = {} + for target in targets: + branch = trie + for part in target.parts: + branch = branch.setdefault(part, {}) + selected: set[PurePosixPath] = set() + root = paths.duplicate_blueprint_descriptor() + + def merge(target: dict[str, dict], source: dict[str, dict]) -> None: + for name, children in source.items(): + merge(target.setdefault(name, {}), children) + + def visit( + parent: int, + requested_children: dict[str, dict], + actual_parts: tuple[str, ...], + ) -> None: + names = tuple(sorted(os.listdir(parent))) + resolved: dict[str, tuple[tuple[int, int, int], dict[str, dict]]] = {} + for requested, descendants in sorted(requested_children.items()): + try: + metadata = os.stat(requested, dir_fd=parent, follow_symlinks=False) + except (FileNotFoundError, NotADirectoryError): + continue + signature = (metadata.st_dev, metadata.st_ino, metadata.st_mode) + actual = _selected_directory_entry(parent, names, requested, signature) + selected.add(PurePosixPath(*actual_parts, actual)) + existing = resolved.get(actual) + if existing is None: + resolved[actual] = (signature, dict(descendants)) + else: + if existing[0] != signature: + raise TreeSnapshotError( + "source target path changed while it was selected" + ) + merge(existing[1], descendants) + if tuple(sorted(os.listdir(parent))) != names: + raise TreeSnapshotError("source target path changed while it was selected") + for actual, (signature, descendants) in sorted(resolved.items()): + current = os.stat(actual, dir_fd=parent, follow_symlinks=False) + if (current.st_dev, current.st_ino, current.st_mode) != signature: + raise TreeSnapshotError("source target path changed while it was selected") + if not descendants or not stat.S_ISDIR(current.st_mode): + continue + descriptor = os.open( + actual, + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent, + ) + try: + opened = os.fstat(descriptor) + if (opened.st_dev, opened.st_ino, opened.st_mode) != signature: + raise TreeSnapshotError( + "source target path changed while it was selected" + ) + visit(descriptor, descendants, (*actual_parts, actual)) + finally: + os.close(descriptor) + final = os.stat(actual, dir_fd=parent, follow_symlinks=False) + if (final.st_dev, final.st_ino, final.st_mode) != signature: + raise TreeSnapshotError("source target path changed while it was selected") + + try: + visit(root, trie, ()) + except OSError as error: + raise TreeSnapshotError( + "source target path changed while it was selected" + ) from error + finally: + os.close(root) + return tuple(sorted(selected)) + + +def _bind_existing_target_path( + paths: RuntimePaths, + target: PurePosixPath, +) -> PurePosixPath | None: + selected = _bind_existing_source_target_paths(paths, (target,)) + matches = [path for path in selected if len(path.parts) == len(target.parts)] + if len(matches) > 1: + raise TreeSnapshotError("selected path has no unique filesystem spelling") + return matches[0] if matches else None + + +def _selected_directory_entry( + descriptor: int, + names: tuple[str, ...], + requested: str, + signature: tuple[int, int, int], +) -> str: + if requested in names: + return requested + folded = unicodedata.normalize("NFC", requested).casefold() + matches: list[str] = [] + for name in names: + if unicodedata.normalize("NFC", name).casefold() != folded: + continue + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if (metadata.st_dev, metadata.st_ino, metadata.st_mode) == signature: + matches.append(name) + if len(matches) != 1: + raise TreeSnapshotError("source target path has no stable directory entry") + return matches[0] + + +def _execution_contract_projection(snapshot: TreeSnapshot) -> TreeSnapshot: + def selected(relative: str) -> bool: + return not relative or _execution_contract_path(PurePosixPath(relative)) + + return TreeSnapshot( + root_identity=snapshot.root_identity, + directories=tuple(path for path in snapshot.directories if selected(path)), + files=tuple(entry for entry in snapshot.files if selected(entry[0])), + symlinks=tuple(entry for entry in snapshot.symlinks if selected(entry[0])), + special=tuple(entry for entry in snapshot.special if selected(entry[0])), + placeholders=tuple(path for path in snapshot.placeholders if selected(path)), + omitted=tuple(entry for entry in snapshot.omitted if selected(entry[0])), + identities=tuple(entry for entry in snapshot.identities if selected(entry[0])), + ) + + +def _reject_portable_snapshot_collisions(snapshot: TreeSnapshot) -> None: + paths = [ + *snapshot.directories, + *(relative for relative, _data in snapshot.files), + *(relative for relative, _target in snapshot.symlinks), + *(relative for relative, _mode in snapshot.special), + *snapshot.placeholders, + ] + seen: dict[tuple[str, ...], str] = {} + for relative in paths: + if not relative: + continue + key = _portable_path_key(PurePosixPath(relative)) + previous = seen.setdefault(key, relative) + if previous != relative: + raise TreeSnapshotError("execution contract contains portably ambiguous paths") + + +def _execution_generation_changed( + paths: RuntimePaths, + expected_generation: str, + lean_sources: BoundProjectSources | None, + expected_lean_generation: str | None, +) -> bool: + try: + paths.verify() + current = _capture_execution_authority(paths) + paths.verify() + if current.generation_revision != expected_generation: + return True + if lean_sources is None: + return expected_lean_generation is not None + if expected_lean_generation is None: + return True + lean_sources.verify() + current_lean = lean_sources.capture() + lean_sources.verify() + return current_lean.generation_revision != expected_lean_generation + except (OSError, RuntimeProjectionError, TreeSnapshotError, WorkspaceError): + return True + + +def _unsafe_snapshot_issues(snapshot: TreeSnapshot) -> list[ExecutionInputIssue]: + issues = [ + ExecutionInputIssue( + "execution-authority-unsafe", + f"execution authority contains a symbolic link: {relative}", + ) + for relative, _target in snapshot.symlinks + ] + issues.extend( + ExecutionInputIssue( + "execution-authority-unsafe", + f"execution authority contains a special file: {relative}", + ) + for relative, _mode in snapshot.special + ) + return issues + + +def _runtime_binding(paths: RuntimePaths) -> tuple[object, ...]: + return ( + paths.project_root, + paths.blueprint_dir, + paths.workspace_project_id, + paths.workspace_project_binding_sha256, + paths.workspace_root_identity, + paths.workspace_manifest_sha256, + paths.blueprint_identity, + paths.roadmap_identity, + ) + + +def _rebase_runtime(runtime: RuntimeGraph, blueprint_path: str) -> RuntimeGraph: + source = PurePosixPath(runtime.blueprint_path) + destination = PurePosixPath(blueprint_path) + nodes = tuple( + replace( + node, + article_path=( + destination / PurePosixPath(node.article_path).relative_to(source) + ).as_posix(), + ) + for node in runtime.nodes + ) + return replace(runtime, blueprint_path=destination.as_posix(), nodes=nodes) + + +def _apply_lean_index(runtime: RuntimeGraph, index: SourceIndex | None) -> RuntimeGraph: + if index is None: + return runtime + nodes = tuple( + replace( + node, + lean_targets=tuple( + RuntimeLeanTarget( + target.declaration, + declaration.path.as_posix() if declaration is not None else None, + ) + for target in node.lean_targets + for declaration in (index.find(target.declaration),) + ), + ) + for node in runtime.nodes + ) + return replace(runtime, nodes=nodes) + + +def _update_authority_digest( + digest, + kind: bytes, + relative: str, + data: bytes, +) -> None: + path = os.fsencode(relative) + for field in (kind, path, data): + digest.update(len(field).to_bytes(8, "big")) + digest.update(field) + + +def _coverage_matches_authority( + coverage: CoverageSummary, + authority: _ExecutionAuthorityRevision, +) -> bool: + artifact_key = ( + _portable_path_key(PurePosixPath(coverage.artifact_path)) + if coverage.artifact_path is not None + else None + ) + artifact_matches = [ + sha256 + for relative, sha256 in authority.source_sha256s + if artifact_key is not None + and _portable_path_key(PurePosixPath(relative)) == artifact_key + ] + return ( + coverage.source_sha256 == authority.coverage_sha256 + and coverage._roadmap_sha256 == authority.roadmap_sha256 + and len(artifact_matches) == 1 + and coverage.artifact_sha256 == artifact_matches[0] + ) + + +def _runtime_matches_lean_index(runtime: RuntimeGraph, index: SourceIndex | None) -> bool: + if index is None: + return True + for node in runtime.nodes: + for target in node.lean_targets: + declaration = index.find(target.declaration) + expected = declaration.path.as_posix() if declaration is not None else None + if target.source_file != expected: + return False + return True + + +def _require_v2_coverage(blueprint: Path) -> CoverageSummary: + coverage, issues = load_coverage(blueprint) + if issues: + raise ExecutionInputError( + [ExecutionInputIssue(issue.code, issue.reason) for issue in issues] + ) + if coverage is None or coverage.schema != COVERAGE_V2_SCHEMA: + raise ExecutionInputError( + [ + ExecutionInputIssue( + "coverage-v2-required", + "ready-work discovery requires an exhaustive autoform-coverage/v2 contract", + ) + ] + ) + if not coverage.complete: + mapped_count = coverage.counts["MAPPED"] + subject = "unit remains" if mapped_count == 1 else "units remain" + raise ExecutionInputError( + [ + ExecutionInputIssue( + "coverage-incomplete", + "ready-work discovery requires a terminal coverage disposition for every " + f"v2 source unit; {mapped_count} {subject} MAPPED", + ) + ] + ) + return coverage + + +def _required(value: str | None) -> str: + if value is None: # Defensive: a valid v2 summary always carries both. + raise ExecutionInputError( + [ExecutionInputIssue("coverage-v2-invalid", "v2 coverage binding is incomplete")] + ) + return value + + +__all__ = [ + "EXECUTION_INPUT_SCHEMA", + "ExecutionInput", + "ExecutionInputError", + "ExecutionInputIssue", + "ExecutionNodeBinding", + "ExecutionSourceUnit", + "load_execution_input", +] diff --git a/autoform_cli/graph.py b/autoform_cli/graph.py index ec98cdd0..ee1999e0 100644 --- a/autoform_cli/graph.py +++ b/autoform_cli/graph.py @@ -10,11 +10,23 @@ from __future__ import annotations import hashlib +import os import re +import stat +from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath +from types import MappingProxyType from urllib.parse import unquote, urlsplit +from .workspace import ( + _DIRECTORY_BINDING_SUPPORTED, + _WorkspaceRootBinding, + _open_workspace_root, + _path_is_reparse_point, +) +from .workspace_manifest import WorkspaceError + _HEADING = re.compile(r"^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$") _FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") @@ -22,6 +34,7 @@ _HTML_COMMENT = re.compile(r"|$)", re.DOTALL) _INLINE_CODE = re.compile(r"(`+).*?\1") ARTICLE_ID_PATTERN = re.compile(r"af_[0-9a-f]{24}\Z") +SOURCE_UNIT_PATTERN = re.compile(r"[a-z][a-z0-9]*(?:-[a-z0-9]+)*\Z") _FRONTMATTER_KEYS = frozenset( { "article_id", @@ -35,11 +48,24 @@ "not_ready", "origin", "discussion", + "source_units", } ) _FORMALIZED = "formalized" _TRUE = frozenset({"true", "yes"}) _FALSE = frozenset({"false", "no"}) +_DIRECTORY_FLAGS = ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) +) +_FILE_FLAGS = ( + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NONBLOCK", 0) +) #: ``## Depends on`` carries the prerequisites needed to *state* a node; #: ``## Proof depends on`` carries the extra prerequisites its *proof* needs. @@ -89,6 +115,7 @@ class Node: depth: int = 0 article_id: str | None = None source_sha256: str | None = None + source_units: tuple[str, ...] = () @property def formalizable(self) -> bool: @@ -96,20 +123,140 @@ def formalizable(self) -> bool: return self.declaration is not None +def _restore_node(node: Node, state: list[object]) -> None: + """Restore both pre-coverage and current slotted ``Node`` pickles.""" + + field_names = tuple(Node.__dataclass_fields__) + if len(state) == len(field_names) - 1: + state = [*state, ()] + if len(state) != len(field_names): + raise ValueError("unsupported Node pickle state") + for name, value in zip(field_names, state): + object.__setattr__(node, name, value) + + +# Python generates its own slotted-frozen dataclass hook. Assign after the +# decorator has run so every supported interpreter uses the compatibility hook. +Node.__setstate__ = _restore_node # type: ignore[attr-defined] + + +class _TrackedNodeDict(dict[str, Node]): + """A normal mutable node dictionary with a cheap structural revision.""" + + __slots__ = ("_revision",) + + def __init__(self, *args, **kwargs) -> None: + self._revision = getattr(self, "_revision", -1) + 1 + super().__init__(*args, **kwargs) + + @property + def revision(self) -> int: + return getattr(self, "_revision", 0) + + def _touch(self) -> None: + self._revision = getattr(self, "_revision", 0) + 1 + + def __setitem__(self, key: str, value: Node) -> None: + self._touch() + super().__setitem__(key, value) + + def __delitem__(self, key: str) -> None: + self._touch() + super().__delitem__(key) + + def clear(self) -> None: + self._touch() + super().clear() + + def pop(self, key, *args): + self._touch() + return super().pop(key, *args) + + def popitem(self): + self._touch() + return super().popitem() + + def setdefault(self, key, default=None): + self._touch() + return super().setdefault(key, default) + + def update(self, *args, **kwargs) -> None: + self._touch() + super().update(*args, **kwargs) + + def __ior__(self, other): + self._touch() + return super().__ior__(other) + + def __getstate__(self) -> int: + return self._revision + + def __setstate__(self, state: int) -> None: + self._revision = max(self.revision, state) + + +class _GraphCache: + __slots__ = ("_children_by_parent", "_children_revision", "_source_bytes") + + _children_by_parent: Mapping[str | None, tuple[str, ...]] + _children_revision: int + + @dataclass(frozen=True, slots=True) -class Graph: +class Graph(_GraphCache): """A validated blueprint graph, keyed by stable node id.""" blueprint_dir: Path nodes: dict[str, Node] + def __post_init__(self) -> None: + if not isinstance(self.nodes, _TrackedNodeDict): + object.__setattr__(self, "nodes", _TrackedNodeDict(self.nodes)) + if not hasattr(self, "_source_bytes"): + object.__setattr__(self, "_source_bytes", {}) + self._refresh_children() + + def _refresh_children(self) -> None: + children: dict[str | None, list[str]] = {} + for node in self.nodes.values(): + children.setdefault(node.parent, []).append(node.id) + object.__setattr__( + self, + "_children_by_parent", + MappingProxyType({parent: tuple(node_ids) for parent, node_ids in children.items()}), + ) + object.__setattr__(self, "_children_revision", self.nodes.revision) + @property def edge_count(self) -> int: return sum(len(node.dependencies) for node in self.nodes.values()) def children(self, node_id: str) -> tuple[str, ...]: """Return the direct contained articles of *node_id*.""" - return tuple(node.id for node in self.nodes.values() if node.parent == node_id) + if getattr(self, "_children_revision", -1) != self.nodes.revision: + self._refresh_children() + return self._children_by_parent.get(node_id, ()) + + def source_bytes(self, node_id: str) -> bytes | None: + """Return immutable source bytes captured with this graph, when available.""" + + return self._source_bytes.get(node_id) + + +def _restore_graph_state(graph: Graph, state: list[object]) -> None: + """Restore legacy slot pickles through the current cache initializer.""" + if len(state) != 2: + raise ValueError("unsupported Graph pickle state") + blueprint_dir, nodes = state + object.__setattr__(graph, "blueprint_dir", blueprint_dir) + object.__setattr__(graph, "nodes", nodes) + graph.__post_init__() + + +# Python 3.10's ``dataclass(slots=True, frozen=True)`` replaces a class-defined +# pickle hook. Installing it after decoration keeps old Graph pickles compatible +# on every supported interpreter. +setattr(Graph, "__setstate__", _restore_graph_state) @dataclass(frozen=True, slots=True) @@ -129,9 +276,38 @@ class _NodeSource: path: Path text: str source_sha256: str + content: bytes + + +@dataclass(frozen=True, slots=True) +class _BoundRoadmapDirectory: + relative: str + identity: tuple[int, ...] + names: tuple[str, ...] -def load_graph(blueprint_dir: str | Path) -> Graph: +@dataclass(frozen=True, slots=True) +class _BoundRoadmapEntry: + relative: str + identity: tuple[int, ...] + ignored: bool = False + + +@dataclass(frozen=True, slots=True) +class _PortableRoadmapSnapshot: + root_identity: tuple[int, ...] + entries: tuple[tuple[str, tuple[int, ...]], ...] + directories: tuple[str, ...] + sources: tuple[_NodeSource, ...] + issues: tuple[str, ...] + + +def load_graph( + blueprint_dir: str | Path, + *, + _expected_blueprint_identity: tuple[int, int] | None = None, + _expected_roadmap_identity: tuple[int, int] | None = None, +) -> Graph: """Load and validate Markdown nodes beneath *blueprint_dir*.""" blueprint = Path(blueprint_dir).expanduser().resolve() @@ -142,13 +318,18 @@ def load_graph(blueprint_dir: str | Path) -> Graph: parsed: list[_ParsedNode] = [] canonical_ids: dict[Path, str] = {} node_ids: dict[str, Path] = {} - sources, discovery_issues = _discover_nodes(blueprint) + sources, discovery_issues = _discover_nodes( + blueprint, + expected_blueprint_identity=_expected_blueprint_identity, + expected_roadmap_identity=_expected_roadmap_identity, + ) issues.extend(discovery_issues) article_ids: dict[str, str] = {} source_hashes = {source.id: source.source_sha256 for source in sources} + source_bytes = {source.id: source.content for source in sources} for source in sources: - canonical = source.path.resolve() + canonical = source.path if canonical in canonical_ids: issues.append(f"{source.id}: duplicates node {canonical_ids[canonical]!r}") continue @@ -220,6 +401,9 @@ def resolve(targets: tuple[str, ...], node: _ParsedNode = parsed_node) -> list[s depth=_article_depth(parsed_node.id, parents), article_id=metadata.get("article_id"), source_sha256=source_hashes[parsed_node.id], + source_units=tuple(metadata.get("source_units", "").split(",")) + if metadata.get("source_units") + else (), ) if not issues: @@ -228,50 +412,474 @@ def resolve(targets: tuple[str, ...], node: _ParsedNode = parsed_node) -> list[s issues.extend(_find_rollup_cycles(nodes)) if issues: raise GraphValidationError(issues) - return Graph(blueprint_dir=blueprint, nodes=nodes) + graph = Graph(blueprint_dir=blueprint, nodes=nodes) + object.__setattr__(graph, "_source_bytes", source_bytes) + return graph -def _discover_nodes(blueprint: Path) -> tuple[list[_NodeSource], list[str]]: +def _discover_nodes( + blueprint: Path, + *, + expected_blueprint_identity: tuple[int, int] | None = None, + expected_roadmap_identity: tuple[int, int] | None = None, +) -> tuple[list[_NodeSource], list[str]]: roadmap_root = blueprint / "roadmap" if not roadmap_root.is_dir(): return [], [f"roadmap directory does not exist: {roadmap_root}"] + if not _DIRECTORY_BINDING_SUPPORTED or os.listdir not in getattr(os, "supports_fd", ()): + return _discover_nodes_portably( + blueprint, + roadmap_root, + expected_blueprint_identity=expected_blueprint_identity, + expected_roadmap_identity=expected_roadmap_identity, + ) + + try: + binding = _open_workspace_root(blueprint) + except WorkspaceError: + return [], ["blueprint directory cannot be inspected safely"] issues: list[str] = [] sources: list[_NodeSource] = [] - roadmap_root = roadmap_root.resolve() - entries = sorted(roadmap_root.rglob("*")) - for path in entries: - if path.is_file() and path.name.casefold() == "readme.md" and path.name != "README.md": - relative = path.relative_to(roadmap_root).as_posix() - issues.append( - f"{relative}: noncanonical README filename; container pages must be named exactly README.md " - "for portable behavior on case-sensitive filesystems" + directories: list[_BoundRoadmapDirectory] = [] + entries: list[_BoundRoadmapEntry] = [] + roadmap_descriptor: int | None = None + try: + if ( + expected_blueprint_identity is not None + and binding.identity != expected_blueprint_identity + ): + return [], ["blueprint changed while the graph was loaded"] + try: + roadmap_identity = os.stat( + "roadmap", + dir_fd=binding.descriptor, + follow_symlinks=False, ) + if not stat.S_ISDIR(roadmap_identity.st_mode): + return [], [f"roadmap directory does not exist: {roadmap_root}"] + roadmap_descriptor = os.open( + "roadmap", + _DIRECTORY_FLAGS, + dir_fd=binding.descriptor, + ) + opened = os.fstat(roadmap_descriptor) + except (OSError, ValueError): + return [], [f"roadmap directory does not exist: {roadmap_root}"] + if _stat_signature(opened) != _stat_signature(roadmap_identity): + return [], ["roadmap changed while the graph was loaded"] + if ( + expected_roadmap_identity is not None + and (opened.st_dev, opened.st_ino) != expected_roadmap_identity + ): + return [], ["roadmap changed while the graph was loaded"] + _scan_bound_roadmap_directory( + roadmap_descriptor, + relative="", + identity=_stat_signature(opened), + roadmap_root=roadmap_root, + directories=directories, + entries=entries, + sources=sources, + issues=issues, + ) + _graph_snapshot_checkpoint("before-final-verification", "") + _verify_roadmap_snapshot(binding, roadmap_descriptor, directories, entries) + except (_RoadmapChanged, WorkspaceError): + return [], ["roadmap changed while the graph was loaded"] + finally: + if roadmap_descriptor is not None: + try: + os.close(roadmap_descriptor) + except OSError: + pass + binding.close() + + sources.sort(key=lambda source: source.path.as_posix()) + issues.extend( + _chapter_issues( + roadmap_root, + [directory.relative for directory in directories], + sources, + ) + ) + return sources, issues + + +class _RoadmapChanged(Exception): + """The bound roadmap tree did not remain one filesystem generation.""" + + +def _graph_snapshot_checkpoint(_event: str, _relative: str) -> None: + """Deterministic roadmap-substitution boundary used by adversarial tests.""" - for path in entries: - if not path.is_file() or path.suffix != ".md": + +def _stat_signature(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _scan_bound_roadmap_directory( + descriptor: int, + *, + relative: str, + identity: tuple[int, ...], + roadmap_root: Path, + directories: list[_BoundRoadmapDirectory], + entries: list[_BoundRoadmapEntry], + sources: list[_NodeSource], + issues: list[str], +) -> None: + """Capture one roadmap subtree while retaining only its ancestor descriptors.""" + + try: + names = tuple(sorted(os.listdir(descriptor))) + except OSError: + raise _RoadmapChanged from None + if any( + not isinstance(name, str) + or not name + or name in {".", ".."} + or "/" in name + or "\\" in name + for name in names + ): + raise _RoadmapChanged + directories.append(_BoundRoadmapDirectory(relative, identity, names)) + _graph_snapshot_checkpoint("after-directory-list", relative) + for name in names: + child_relative = f"{relative}/{name}" if relative else name + try: + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + except OSError: + raise _RoadmapChanged from None + child_identity = _stat_signature(metadata) + if name.startswith("."): + entries.append(_BoundRoadmapEntry(child_relative, child_identity, ignored=True)) + continue + if stat.S_ISDIR(metadata.st_mode): + child_descriptor: int | None = None + try: + child_descriptor = os.open(name, _DIRECTORY_FLAGS, dir_fd=descriptor) + opened = os.fstat(child_descriptor) + if _stat_signature(opened) != child_identity: + raise _RoadmapChanged + _scan_bound_roadmap_directory( + child_descriptor, + relative=child_relative, + identity=child_identity, + roadmap_root=roadmap_root, + directories=directories, + entries=entries, + sources=sources, + issues=issues, + ) + named = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if _stat_signature(named) != child_identity: + raise _RoadmapChanged + except OSError: + raise _RoadmapChanged from None + finally: + if child_descriptor is not None: + try: + os.close(child_descriptor) + except OSError: + pass + continue + entries.append(_BoundRoadmapEntry(child_relative, child_identity)) + if stat.S_ISLNK(metadata.st_mode): + issues.append(f"{child_relative}: roadmap paths must not be symbolic links") + continue + if not stat.S_ISREG(metadata.st_mode): + continue + if name.casefold() == "readme.md" and name != "README.md": + issues.append( + f"{child_relative}: noncanonical README filename; container pages must be named " + "exactly README.md for portable behavior on case-sensitive filesystems" + ) + if Path(name).suffix != ".md": continue try: - content = path.read_bytes() + content = _read_bound_roadmap_file(descriptor, name, child_identity) text = content.decode("utf-8") - except (OSError, UnicodeError) as exc: - relative = path.relative_to(roadmap_root).as_posix() - issues.append(f"{relative}: cannot read roadmap page: {exc}") - continue - node_id = _article_id(path, roadmap_root) - canonical = path.resolve() - if not _is_within(canonical, roadmap_root): - issues.append(f"{node_id}: node file escapes the roadmap directory") + except UnicodeError as error: + issues.append(f"{child_relative}: cannot read roadmap page: {error}") continue + source_path = roadmap_root.joinpath(*PurePosixPath(child_relative).parts) + node_id = _article_id(source_path, roadmap_root) sources.append( - _NodeSource(node_id, canonical, text, hashlib.sha256(content).hexdigest()) + _NodeSource( + node_id, + source_path, + text, + hashlib.sha256(content).hexdigest(), + content, + ) ) + try: + if _stat_signature(os.fstat(descriptor)) != identity: + raise _RoadmapChanged + if tuple(sorted(os.listdir(descriptor))) != names: + raise _RoadmapChanged + except OSError: + raise _RoadmapChanged from None + - issues.extend(_chapter_issues(roadmap_root)) +def _read_bound_roadmap_file( + parent_descriptor: int, + name: str, + expected: tuple[int, ...], +) -> bytes: + descriptor: int | None = None + try: + descriptor = os.open(name, _FILE_FLAGS, dir_fd=parent_descriptor) + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _stat_signature(opened) != expected: + raise _RoadmapChanged + chunks: list[bytes] = [] + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(descriptor) + named = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if _stat_signature(after) != expected or _stat_signature(named) != expected: + raise _RoadmapChanged + return b"".join(chunks) + except (OSError, WorkspaceError): + raise _RoadmapChanged from None + finally: + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + pass + + +def _discover_nodes_portably( + blueprint: Path, + roadmap_root: Path, + *, + expected_blueprint_identity: tuple[int, int] | None = None, + expected_roadmap_identity: tuple[int, int] | None = None, +) -> tuple[list[_NodeSource], list[str]]: + """Keep read-only graph commands usable where descriptor traversal is absent.""" + + try: + blueprint_before = blueprint.stat(follow_symlinks=False) + first = _portable_roadmap_snapshot(roadmap_root) + _graph_snapshot_checkpoint("between-portable-snapshots", "") + second = _portable_roadmap_snapshot(roadmap_root) + blueprint_after = blueprint.stat(follow_symlinks=False) + except (OSError, RuntimeError, ValueError, _RoadmapChanged): + return [], ["roadmap changed while the graph was loaded"] + if ( + first != second + or _stat_signature(blueprint_before) != _stat_signature(blueprint_after) + or ( + expected_blueprint_identity is not None + and (blueprint_after.st_dev, blueprint_after.st_ino) != expected_blueprint_identity + ) + or ( + expected_roadmap_identity is not None + and second.root_identity[:2] != expected_roadmap_identity + ) + ): + return [], ["roadmap changed while the graph was loaded"] + sources = list(second.sources) + issues = list(second.issues) + issues.extend(_chapter_issues(roadmap_root, list(second.directories), sources)) return sources, issues -def _chapter_issues(roadmap_root: Path) -> list[str]: +def _portable_roadmap_snapshot(roadmap_root: Path) -> _PortableRoadmapSnapshot: + root_before = roadmap_root.stat(follow_symlinks=False) + if not stat.S_ISDIR(root_before.st_mode) or _path_is_reparse_point( + roadmap_root, root_before + ): + raise _RoadmapChanged + paths = _portable_roadmap_paths(roadmap_root) + entries: list[tuple[str, tuple[int, ...]]] = [] + directories: list[str] = [] + sources: list[_NodeSource] = [] + issues: list[str] = [] + for path in paths: + relative = path.relative_to(roadmap_root).as_posix() + metadata = path.stat(follow_symlinks=False) + identity = _stat_signature(metadata) + entries.append((relative, identity)) + if _path_is_reparse_point(path, metadata): + issues.append( + f"{relative}: roadmap paths must not be symbolic links or reparse points" + ) + continue + if stat.S_ISDIR(metadata.st_mode): + directories.append(relative) + continue + if not stat.S_ISREG(metadata.st_mode): + continue + if path.name.casefold() == "readme.md" and path.name != "README.md": + issues.append( + f"{relative}: noncanonical README filename; container pages must be named exactly " + "README.md for portable behavior on case-sensitive filesystems" + ) + if path.suffix != ".md": + continue + with path.open("rb") as stream: + opened = os.fstat(stream.fileno()) + content = stream.read() + after = os.fstat(stream.fileno()) + final = path.stat(follow_symlinks=False) + if not stat.S_ISREG(opened.st_mode) or not ( + _stat_signature(opened) + == _stat_signature(after) + == _stat_signature(final) + == identity + ): + raise _RoadmapChanged + try: + text = content.decode("utf-8") + except UnicodeError as error: + issues.append(f"{relative}: cannot read roadmap page: {error}") + continue + source_path = roadmap_root.joinpath(*PurePosixPath(relative).parts) + sources.append( + _NodeSource( + _article_id(source_path, roadmap_root), + source_path, + text, + hashlib.sha256(content).hexdigest(), + content, + ) + ) + root_after = roadmap_root.stat(follow_symlinks=False) + if _stat_signature(root_before) != _stat_signature(root_after): + raise _RoadmapChanged + return _PortableRoadmapSnapshot( + root_identity=_stat_signature(root_after), + entries=tuple(entries), + directories=tuple(directories), + sources=tuple(sources), + issues=tuple(issues), + ) + + +def _portable_roadmap_paths(roadmap_root: Path) -> tuple[Path, ...]: + """Enumerate without traversing links or Windows reparse-point directories.""" + + paths: list[Path] = [] + + def visit(directory: Path) -> None: + before = directory.stat(follow_symlinks=False) + names = tuple(sorted(path.name for path in directory.iterdir())) + for name in names: + path = directory / name + metadata = path.stat(follow_symlinks=False) + if name.startswith("."): + continue + paths.append(path) + if stat.S_ISDIR(metadata.st_mode) and not _path_is_reparse_point(path, metadata): + visit(path) + after = directory.stat(follow_symlinks=False) + final_names = tuple(sorted(path.name for path in directory.iterdir())) + if _stat_signature(before) != _stat_signature(after) or names != final_names: + raise _RoadmapChanged + + visit(roadmap_root) + return tuple(sorted(paths)) + + +def _verify_roadmap_snapshot( + binding: _WorkspaceRootBinding, + roadmap_descriptor: int, + directories: list[_BoundRoadmapDirectory], + entries: list[_BoundRoadmapEntry], +) -> None: + expected_directories = {directory.relative: directory for directory in directories} + expected_entries = {entry.relative: entry for entry in entries} + visited_directories: set[str] = set() + visited_entries: set[str] = set() + + def verify_directory(descriptor: int, relative: str) -> None: + expected = expected_directories.get(relative) + if expected is None: + raise _RoadmapChanged + visited_directories.add(relative) + opened = os.fstat(descriptor) + names = tuple(sorted(os.listdir(descriptor))) + if _stat_signature(opened) != expected.identity or names != expected.names: + raise _RoadmapChanged + for name in names: + child_relative = f"{relative}/{name}" if relative else name + current = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + directory = expected_directories.get(child_relative) + if directory is not None: + if not stat.S_ISDIR(current.st_mode) or _stat_signature(current) != directory.identity: + raise _RoadmapChanged + child_descriptor: int | None = None + try: + child_descriptor = os.open(name, _DIRECTORY_FLAGS, dir_fd=descriptor) + child_opened = os.fstat(child_descriptor) + if _stat_signature(child_opened) != directory.identity: + raise _RoadmapChanged + verify_directory(child_descriptor, child_relative) + named = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if _stat_signature(named) != directory.identity: + raise _RoadmapChanged + finally: + if child_descriptor is not None: + try: + os.close(child_descriptor) + except OSError: + pass + continue + entry = expected_entries.get(child_relative) + if entry is None or ( + not entry.ignored and _stat_signature(current) != entry.identity + ): + raise _RoadmapChanged + if entry.ignored and stat.S_IFMT(current.st_mode) != stat.S_IFMT( + entry.identity[2] + ): + raise _RoadmapChanged + visited_entries.add(child_relative) + after = os.fstat(descriptor) + if ( + _stat_signature(after) != expected.identity + or tuple(sorted(os.listdir(descriptor))) != expected.names + ): + raise _RoadmapChanged + + try: + roadmap = expected_directories.get("") + if roadmap is None: + raise _RoadmapChanged + named = os.stat("roadmap", dir_fd=binding.descriptor, follow_symlinks=False) + if _stat_signature(named) != roadmap.identity: + raise _RoadmapChanged + verify_directory(roadmap_descriptor, "") + if visited_directories != set(expected_directories) or visited_entries != set( + expected_entries + ): + raise _RoadmapChanged + binding.verify() + except OSError: + raise _RoadmapChanged from None + + +def _chapter_issues( + roadmap_root: Path, + directories: list[str], + sources: list[_NodeSource], +) -> list[str]: """Reject a chapter directory that names no chapter. Containment is inferred from nested ``README.md`` articles, so a directory @@ -298,22 +906,32 @@ def _chapter_issues(roadmap_root: Path) -> list[str]: generated nav. """ - try: - chapters = sorted(path for path in roadmap_root.iterdir() if path.is_dir()) - except OSError: - return [] + chapters = sorted( + relative + for relative in directories + if relative and "/" not in relative + ) + article_counts: dict[str, int] = {} + chapter_readmes: set[str] = set() + for source in sources: + relative = source.path.relative_to(roadmap_root) + if len(relative.parts) < 2: + continue + chapter = relative.parts[0] + article_counts[chapter] = article_counts.get(chapter, 0) + 1 + if relative.parts == (chapter, "README.md"): + chapter_readmes.add(chapter) issues = [] for chapter in chapters: - articles = [path for path in chapter.rglob("*.md") if path.is_file()] - if not articles: + article_count = article_counts.get(chapter, 0) + if not article_count: continue - if (chapter / "README.md").is_file(): + if chapter in chapter_readmes: continue - names = articles issues.append( - f"{chapter.name}: chapter directory holds {len(names)} article(s) but no " - f"README.md, so they attach to the roadmap root instead of a chapter; " - f"add {chapter.name}/README.md with the chapter's H1 title" + f"{chapter}: chapter directory holds {article_count} article(s) but no " + "README.md, so they attach to the roadmap root instead of a chapter; " + f"add {chapter}/README.md with the chapter's H1 title" ) return issues @@ -328,7 +946,7 @@ def _article_id(path: Path, roadmap_root: Path) -> str: def _article_parents(parsed: list[_ParsedNode]) -> dict[str, str | None]: """Infer strict single-parent containment from nested README articles.""" - by_path = {node.path.resolve(): node.id for node in parsed} + by_path = {node.path: node.id for node in parsed} parents: dict[str, str | None] = {} for node in parsed: candidate = node.path.parent @@ -336,7 +954,7 @@ def _article_parents(parsed: list[_ParsedNode]) -> dict[str, str | None]: candidate = candidate.parent parent: str | None = None while candidate != candidate.parent: - readme = (candidate / "README.md").resolve() + readme = candidate / "README.md" if readme in by_path: parent = by_path[readme] break @@ -476,6 +1094,21 @@ def _normalize_value(node_id: str, line_number: int, key: str, value: str) -> tu if folded not in {"cited", "bridged", "background"}: return value, f"{location}: 'origin' accepts cited, bridged, or background" return folded, None + if key == "source_units": + if not (value.startswith("[") and value.endswith("]")): + return value, ( + f"{location}: 'source_units' must be an inline list such as " + "[chapter-one, theorem-two]" + ) + items = tuple(item.strip() for item in value[1:-1].split(",")) + if not items or any(not item for item in items): + return value, f"{location}: 'source_units' must contain at least one unit id" + malformed = next((item for item in items if not SOURCE_UNIT_PATTERN.fullmatch(item)), None) + if malformed is not None: + return value, f"{location}: malformed source unit id {malformed!r}" + if len(set(items)) != len(items): + return value, f"{location}: duplicate source unit id in 'source_units'" + return ",".join(items), None return value, None @@ -501,93 +1134,185 @@ def _resolve_target( if relative.is_absolute() or relative.suffix != ".md": return None, f"{node.id}: dependency target must be a relative .md file: {target!r}" - resolved = (node.path.parent / relative).resolve() + resolved = Path(os.path.abspath(node.path.parent / relative)) if not _is_within(resolved, blueprint): return None, f"{node.id}: dependency target escapes the blueprint directory: {target!r}" - if not resolved.is_file(): - return None, f"{node.id}: dependency target does not exist: {target!r}" dependency = canonical_ids.get(resolved) if dependency is None: - return None, f"{node.id}: dependency target is not a node: {target!r}" + return None, f"{node.id}: dependency target does not exist: {target!r}" return dependency, None def _find_cycles(nodes: dict[str, Node]) -> list[str]: state: dict[str, int] = {} stack: list[str] = [] + stack_indexes: dict[str, int] = {} issues: list[str] = [] + seen_issues: set[str] = set() + + for root_id in sorted(nodes): + if state.get(root_id, 0) != 0: + continue + state[root_id] = 1 + stack_indexes[root_id] = len(stack) + stack.append(root_id) + frames = [(root_id, 0)] + while frames: + node_id, dependency_index = frames[-1] + dependencies = nodes[node_id].dependencies + if dependency_index == len(dependencies): + frames.pop() + stack.pop() + stack_indexes.pop(node_id) + state[node_id] = 2 + continue - def visit(node_id: str) -> None: - state[node_id] = 1 - stack.append(node_id) - for dependency in nodes[node_id].dependencies: - if state.get(dependency, 0) == 0: - visit(dependency) - elif state.get(dependency) == 1: - start = stack.index(dependency) - cycle = stack[start:] + [dependency] + dependency = dependencies[dependency_index] + frames[-1] = (node_id, dependency_index + 1) + dependency_state = state.get(dependency, 0) + if dependency_state == 0: + state[dependency] = 1 + stack_indexes[dependency] = len(stack) + stack.append(dependency) + frames.append((dependency, 0)) + elif dependency_state == 1: + cycle = stack[stack_indexes[dependency] :] + [dependency] message = f"dependency cycle: {' -> '.join(cycle)}" - if message not in issues: + if message not in seen_issues: + seen_issues.add(message) issues.append(message) - stack.pop() - state[node_id] = 2 - - for node_id in sorted(nodes): - if state.get(node_id, 0) == 0: - visit(node_id) return issues def _find_rollup_cycles(nodes: dict[str, Node]) -> list[str]: """Reject cycles introduced by contracting articles at any hierarchy level.""" children: dict[str | None, list[str]] = {} + parents: dict[str, str | None] = {} for node in nodes.values(): children.setdefault(node.parent, []).append(node.id) + parents[node.id] = node.parent - def direct_child(scope: str | None, node_id: str) -> str | None: - current = node_id - while nodes[current].parent != scope: - parent = nodes[current].parent - if parent is None: - return None - current = parent - return current + depths: dict[str, int] = {} + roots: dict[str, str] = {} + for node_id in nodes: + if node_id in depths: + continue + trail: list[str] = [] + seen: set[str] = set() + current: str | None = node_id + while current is not None and current not in depths: + if current in seen or current not in parents: + raise ValueError("article containment is not a forest") + seen.add(current) + trail.append(current) + current = parents[current] + depth = depths[current] if current is not None else -1 + root = roots[current] if current is not None else trail[-1] + for descendant in reversed(trail): + depth += 1 + depths[descendant] = depth + roots[descendant] = root + + ancestors: list[dict[str, str | None]] = [parents] + maximum_depth = max(depths.values(), default=0) + while 1 << len(ancestors) <= maximum_depth: + previous = ancestors[-1] + ancestors.append( + {node_id: previous[parent] if parent is not None else None for node_id, parent in previous.items()} + ) + + def lift(node_id: str, distance: int) -> str: + level = 0 + while distance: + if distance & 1: + parent = ancestors[level][node_id] + if parent is None: + raise ValueError("article containment depth is inconsistent") + node_id = parent + distance >>= 1 + level += 1 + return node_id + + def lowest_common_ancestor(first: str, second: str) -> str | None: + if roots[first] != roots[second]: + return None + if depths[first] < depths[second]: + first, second = second, first + first = lift(first, depths[first] - depths[second]) + if first == second: + return first + for level in range(len(ancestors) - 1, -1, -1): + first_parent = ancestors[level][first] + second_parent = ancestors[level][second] + if first_parent != second_parent: + if first_parent is None or second_parent is None: + continue + first = first_parent + second = second_parent + return parents[first] + + def direct_child(scope: str | None, node_id: str) -> str: + scope_depth = depths[scope] if scope is not None else -1 + return lift(node_id, depths[node_id] - scope_depth - 1) + + projections: dict[str | None, dict[str, set[str]]] = {} + for target in nodes.values(): + for dependency in target.dependencies: + scope = lowest_common_ancestor(target.id, dependency) + if scope == target.id or scope == dependency: + continue + target_child = direct_child(scope, target.id) + source_child = direct_child(scope, dependency) + projections.setdefault(scope, {}).setdefault(target_child, set()).add(source_child) issues: list[str] = [] + seen_issues: set[str] = set() for scope, siblings in children.items(): if len(siblings) < 2: continue - dependencies = {sibling: set() for sibling in siblings} - for target in nodes.values(): - target_child = direct_child(scope, target.id) - if target_child not in dependencies: - continue - for dependency in target.dependencies: - source_child = direct_child(scope, dependency) - if source_child in dependencies and source_child != target_child: - dependencies[target_child].add(source_child) + projected = projections.get(scope) + if not projected: + continue + dependencies = {sibling: projected.get(sibling, set()) for sibling in siblings} state: dict[str, int] = {} stack: list[str] = [] + stack_indexes: dict[str, int] = {} + ordered_dependencies = { + article_id: tuple(sorted(prerequisites)) for article_id, prerequisites in dependencies.items() + } - def visit(article_id: str) -> None: - state[article_id] = 1 - stack.append(article_id) - for prerequisite in sorted(dependencies[article_id]): - if state.get(prerequisite, 0) == 0: - visit(prerequisite) - elif state.get(prerequisite) == 1: - start = stack.index(prerequisite) - cycle = stack[start:] + [prerequisite] + for root_id in sorted(dependencies): + if state.get(root_id, 0) != 0: + continue + state[root_id] = 1 + stack_indexes[root_id] = len(stack) + stack.append(root_id) + frames = [(root_id, 0)] + while frames: + article_id, dependency_index = frames[-1] + prerequisites = ordered_dependencies[article_id] + if dependency_index == len(prerequisites): + frames.pop() + stack.pop() + stack_indexes.pop(article_id) + state[article_id] = 2 + continue + + prerequisite = prerequisites[dependency_index] + frames[-1] = (article_id, dependency_index + 1) + prerequisite_state = state.get(prerequisite, 0) + if prerequisite_state == 0: + state[prerequisite] = 1 + stack_indexes[prerequisite] = len(stack) + stack.append(prerequisite) + frames.append((prerequisite, 0)) + elif prerequisite_state == 1: + cycle = stack[stack_indexes[prerequisite] :] + [prerequisite] label = scope or "root" message = f"rolled-up dependency cycle in {label}: {' -> '.join(cycle)}" - if message not in issues: + if message not in seen_issues: + seen_issues.add(message) issues.append(message) - stack.pop() - state[article_id] = 2 - - for article_id in sorted(dependencies): - if state.get(article_id, 0) == 0: - visit(article_id) return issues @@ -603,5 +1328,6 @@ def _is_within(path: Path, directory: Path) -> bool: "Graph", "GraphValidationError", "Node", + "SOURCE_UNIT_PATTERN", "load_graph", ] diff --git a/autoform_cli/graph_views.py b/autoform_cli/graph_views.py index 6b690784..642b01d5 100644 --- a/autoform_cli/graph_views.py +++ b/autoform_cli/graph_views.py @@ -480,10 +480,16 @@ def _direct_child(graph: Graph, scope: str, node_id: str) -> str | None: def _leaf_descendants(graph: Graph, node_id: str) -> tuple[str, ...]: - children = graph.children(node_id) - if not children: - return (node_id,) - return tuple(leaf for child in children for leaf in _leaf_descendants(graph, child)) + leaves: list[str] = [] + pending = [node_id] + while pending: + current = pending.pop() + children = graph.children(current) + if children: + pending.extend(reversed(children)) + else: + leaves.append(current) + return tuple(leaves) __all__ = [ diff --git a/autoform_cli/lean.py b/autoform_cli/lean.py index acf234d7..174590b8 100644 --- a/autoform_cli/lean.py +++ b/autoform_cli/lean.py @@ -12,11 +12,25 @@ from __future__ import annotations +import hashlib +import json import os import re +import stat import subprocess -from dataclasses import dataclass -from pathlib import Path +import unicodedata +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath + +from . import workspace as workspace_module +from ._tree_snapshot import ( + BoundDirectoryTree, + TreeSelection, + TreeSnapshot, + TreeSnapshotError, +) _LINE_COMMENT = re.compile(r"--.*$") _NAMESPACE = re.compile(r"^\s*namespace\s+(\S+)") @@ -28,7 +42,59 @@ r"(theorem|lemma|def|abbrev|instance|structure|class|inductive|opaque|axiom)\s+" r"([^\s:(){}\[\]⦃⦄,]+)" ) -_IGNORED_DIRECTORIES = frozenset({".lake", ".git", "lake-packages", "build"}) +_IGNORED_DIRECTORIES = frozenset( + { + ".direnv", + ".git", + ".lake", + ".obsidian", + ".trash", + ".venv", + "build", + "lake-packages", + } +) +_IGNORED_DIRECTORY_PREFIXES = (".autoform-publication-",) +_PUBLICATION_MANIFEST = "publication.json" +_PUBLICATION_SCHEMAS = frozenset({"autoform-publication/v1", "autoform-publication/v2"}) +_PUBLICATION_MANIFEST_BYTE_LIMIT = 1024 * 1024 +_DIRECTORY_FLAGS = ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) +) + +# Lean erases the source-level distinction between theorem, lemma, corollary, +# and proposition. Keep that normalization in one place so the lexical audit +# and the kernel-backed CI probe enforce the same authored intent. +DECLARATION_KIND_ALIASES = { + "abbrev": "abbrev", + "axiom": "axiom", + "class": "class", + "corollary": "theorem", + "def": "def", + "definition": "def", + "inductive": "inductive", + "instance": "instance", + "lemma": "theorem", + "opaque": "opaque", + "proposition": "theorem", + "structure": "structure", + "theorem": "theorem", +} + +_DECLARATION_KEYWORDS = { + "abbrev": frozenset({"abbrev"}), + "axiom": frozenset({"axiom"}), + "class": frozenset({"class"}), + "def": frozenset({"def"}), + "inductive": frozenset({"inductive"}), + "instance": frozenset({"instance"}), + "opaque": frozenset({"opaque"}), + "structure": frozenset({"structure"}), + "theorem": frozenset({"lemma", "theorem"}), +} @dataclass(frozen=True, slots=True) @@ -47,31 +113,491 @@ class SourceIndex: root: Path declarations: dict[str, Declaration] + line_counts: dict[Path, int] = field(default_factory=dict) def find(self, name: str) -> Declaration | None: return self.declarations.get(name) -def index_project(root: str | Path) -> SourceIndex: +@dataclass(frozen=True, slots=True) +class IndexedSourceSnapshot: + """One source generation used for both declaration links and its digest.""" + + index: SourceIndex + revision: str + generation_revision: str = "" + + +@dataclass(slots=True) +class BoundProjectSources: + """A retained Lean source root whose captures cannot change path generation.""" + + root: Path + tree: BoundDirectoryTree + excluded: tuple[PurePosixPath, ...] + + def capture(self) -> IndexedSourceSnapshot: + snapshot = self.tree.capture() + return _indexed_source_snapshot(self.root, snapshot, self.excluded) + + def verify(self) -> None: + self.tree.verify() + + def close(self) -> None: + self.tree.close() + + +def index_project( + root: str | Path, *, exclude_roots: Iterable[str | Path] = () +) -> SourceIndex: """Scan ``*.lean`` beneath *root* and index declarations by full name.""" - root_path = Path(root).expanduser().resolve() - declarations: dict[str, Declaration] = {} - if not root_path.is_dir(): - return SourceIndex(root=root_path, declarations=declarations) + return snapshot_project_sources(root, exclude_roots=exclude_roots).index + + +def snapshot_project_sources( + root: str | Path, *, exclude_roots: Iterable[str | Path] = () +) -> IndexedSourceSnapshot: + """Read each Lean source once and derive its index and revision together.""" + + with bind_project_sources(root, exclude_roots=exclude_roots) as bound: + try: + return bound.capture() + except TreeSnapshotError as error: + raise OSError(str(error)) from error + + +def project_source_revision( + root: str | Path, *, exclude_roots: Iterable[str | Path] = () +) -> str: + """Hash the exact Lean source set consumed by :func:`index_project`.""" + return snapshot_project_sources(root, exclude_roots=exclude_roots).revision + - for path in sorted(root_path.rglob("*.lean")): - if _IGNORED_DIRECTORIES.intersection(path.relative_to(root_path).parts): +@contextmanager +def bind_project_sources( + root: str | Path, + *, + exclude_roots: Iterable[str | Path] = (), +) -> Iterator[BoundProjectSources]: + """Retain a Lean root while one or more source snapshots are consumed.""" + + bound = open_project_sources(root, exclude_roots=exclude_roots) + try: + yield bound + finally: + bound.close() + + +def open_project_sources( + root: str | Path, + *, + exclude_roots: Iterable[str | Path] = (), +) -> BoundProjectSources: + """Open a retained Lean source root; the caller must close it.""" + + root_path = Path(os.path.abspath(Path(root).expanduser())) + try: + root_metadata = root_path.stat(follow_symlinks=False) + except OSError as error: + raise OSError("Lean source root cannot be inspected safely") from error + if not stat.S_ISDIR(root_metadata.st_mode): + raise OSError("Lean source root is not a directory") + root_identity = (root_metadata.st_dev, root_metadata.st_ino) + excluded = tuple( + candidate + for value in exclude_roots + if ( + candidate := _relative_exclusion( + root_path, + value, + root_identity=root_identity, + ) + ) + is not None + ) + selection = TreeSelection( + include=lambda path, mode: _lean_snapshot_includes(path, mode, excluded), + descend=lambda path: not _lean_path_is_excluded(path, excluded), + byte_limit=lambda path: ( + _PUBLICATION_MANIFEST_BYTE_LIMIT + if _is_publication_manifest_name(path.name) + else None + ), + record_omitted=False, + ) + try: + tree = BoundDirectoryTree( + root_path, + expected_identity=root_identity, + selection=selection, + ) + except TreeSnapshotError as error: + raise OSError(str(error)) from error + return BoundProjectSources(root_path, tree, excluded) + + +def _lean_snapshot_includes( + relative: PurePosixPath, + mode: int, + excluded: tuple[PurePosixPath, ...], +) -> bool: + if _lean_path_is_excluded(relative, excluded): + return False + return ( + stat.S_ISDIR(mode) + or stat.S_ISLNK(mode) + or relative.suffix.casefold() == ".lean" + or _is_publication_manifest_name(relative.name) + ) + + +def _lean_path_is_excluded( + relative: PurePosixPath, + excluded: tuple[PurePosixPath, ...], +) -> bool: + return ( + bool(_IGNORED_DIRECTORIES.intersection(relative.parts)) + or any(part.startswith(_IGNORED_DIRECTORY_PREFIXES) for part in relative.parts) + or any(relative == prefix or relative.is_relative_to(prefix) for prefix in excluded) + ) + + +def _indexed_source_snapshot( + root: Path, + snapshot: TreeSnapshot, + excluded: tuple[PurePosixPath, ...], +) -> IndexedSourceSnapshot: + publication_manifests: dict[ + PurePosixPath, + list[tuple[str, bytes | None]], + ] = {} + + def add_manifest(relative: str, kind: str, data: bytes | None = None) -> None: + path = PurePosixPath(relative) + if _is_publication_manifest_name(path.name): + publication_manifests.setdefault(path.parent, []).append((kind, data)) + + for relative, data in snapshot.files: + add_manifest(relative, "file", data) + for relative, _target in snapshot.symlinks: + add_manifest(relative, "symlink") + for relative, _mode in snapshot.special: + add_manifest(relative, "special") + for relative in snapshot.placeholders: + add_manifest(relative, "placeholder") + for relative in snapshot.directories: + add_manifest(relative, "directory") + + publication_roots: set[PurePosixPath] = set() + for parent, manifests in sorted( + publication_manifests.items(), + key=lambda item: (len(item[0].parts), item[0].as_posix()), + ): + if any(parent == root or parent.is_relative_to(root) for root in publication_roots): + continue + if len(manifests) != 1: + raise OSError(f"ambiguous publication manifests in {parent.as_posix()}") + kind, data = manifests[0] + if kind != "file" or data is None: + raise OSError( + f"publication manifest is not a regular file in {parent.as_posix()}" + ) + if _is_publication_manifest_bytes(data): + publication_roots.add(parent) + + def in_publication(relative_text: str) -> bool: + relative = PurePosixPath(relative_text) + return any( + relative == publication or relative.is_relative_to(publication) + for publication in publication_roots + ) + + unsupported = [ + (relative, reason) + for relative, reason in snapshot.unsupported_entries() + if not in_publication(relative) + and PurePosixPath(relative).suffix.casefold() == ".lean" + ] + if unsupported: + relative, reason = unsupported[0] + raise OSError(f"unsafe Lean source {relative}: {reason}") + + declarations: dict[str, Declaration] = {} + line_counts: dict[Path, int] = {} + digest = hashlib.sha256(b"autoform-lean-source-index/v1\0") + for relative_text, data in snapshot.files: + relative = PurePosixPath(relative_text) + if relative.suffix.casefold() != ".lean" or _lean_path_is_excluded( + relative, + excluded, + ): continue + if any( + relative == publication or relative.is_relative_to(publication) + for publication in publication_roots + ): + continue + relative_path = Path(relative.as_posix()) + _update_source_digest(digest, relative_path, data) try: - text = path.read_text(encoding="utf-8") - except (OSError, UnicodeError): + text = data.decode("utf-8") + except UnicodeError: continue - relative = path.relative_to(root_path) - for declaration in _scan(text, relative): - # First definition wins, so an earlier file is not masked by a later - # one when a name is genuinely duplicated across namespaces. + line_counts[relative_path] = len(text.splitlines()) + for declaration in _scan(text, relative_path): declarations.setdefault(declaration.name, declaration) - return SourceIndex(root=root_path, declarations=declarations) + return IndexedSourceSnapshot( + SourceIndex(root=root, declarations=declarations, line_counts=line_counts), + digest.hexdigest(), + _lean_generation_revision(snapshot, publication_roots), + ) + + +def _lean_generation_revision( + snapshot: TreeSnapshot, + publication_roots: set[PurePosixPath], +) -> str: + """Hash only effective Lean inputs and their ancestor directories.""" + + def retained_entry(relative_text: str) -> bool: + relative = PurePosixPath(relative_text) + return relative.suffix.casefold() == ".lean" and not any( + relative == publication or relative.is_relative_to(publication) + for publication in publication_roots + ) + + files = tuple(entry for entry in snapshot.files if retained_entry(entry[0])) + symlinks = tuple(entry for entry in snapshot.symlinks if retained_entry(entry[0])) + special = tuple(entry for entry in snapshot.special if retained_entry(entry[0])) + placeholders = tuple(path for path in snapshot.placeholders if retained_entry(path)) + omitted = tuple(entry for entry in snapshot.omitted if retained_entry(entry[0])) + retained_paths = { + PurePosixPath(relative) + for relative, _value in (*files, *symlinks, *special, *omitted) + } + retained_paths.update(PurePosixPath(relative) for relative in placeholders) + retained_directories = {PurePosixPath()} + for path in retained_paths: + retained_directories.update(path.parents) + + directories = tuple( + path + for path in snapshot.directories + if PurePosixPath(path) in retained_directories + ) + retained_identity_paths = set(directories) + retained_identity_paths.update(path.as_posix() for path in retained_paths) + filtered = TreeSnapshot( + root_identity=snapshot.root_identity, + directories=directories, + files=files, + symlinks=symlinks, + special=special, + placeholders=placeholders, + omitted=omitted, + identities=tuple( + entry for entry in snapshot.identities if entry[0] in retained_identity_paths + ), + ) + return filtered.generation_revision + + +def _is_publication_manifest_bytes(data: bytes) -> bool: + if len(data) > _PUBLICATION_MANIFEST_BYTE_LIMIT: + return False + try: + value = json.loads(data.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError): + return False + return isinstance(value, dict) and value.get("schema") in _PUBLICATION_SCHEMAS + + +def _is_publication_manifest_name(name: str) -> bool: + return unicodedata.normalize("NFC", name).casefold() == _PUBLICATION_MANIFEST + + +def _update_source_digest(digest, relative: Path, data: bytes) -> None: + encoded = os.fsencode(relative.as_posix()) + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + digest.update(len(data).to_bytes(8, "big")) + digest.update(data) + + +def _relative_exclusion( + root: Path, + value: str | Path, + *, + root_identity: tuple[int, int], +) -> PurePosixPath | None: + candidate = Path(value).expanduser() + if not candidate.is_absolute(): + candidate = root / candidate + cursor = Path(os.path.abspath(candidate)) + tail: list[str] = [] + while True: + try: + metadata = cursor.stat(follow_symlinks=False) + except FileNotFoundError: + pass + except OSError as error: + raise OSError("Lean exclusion path cannot be inspected safely") from error + else: + if stat.S_ISDIR(metadata.st_mode) and ( + metadata.st_dev, + metadata.st_ino, + ) == root_identity: + break + parent = cursor.parent + if parent == cursor: + return None + tail.append(cursor.name) + cursor = parent + result = _canonical_exclusion_tail(root, tuple(reversed(tail)), root_identity) + if any(part in {"", ".", ".."} for part in result.parts): + return None + return result + + +def _canonical_exclusion_tail( + root: Path, + parts: tuple[str, ...], + root_identity: tuple[int, int], +) -> PurePosixPath: + """Use physical names for existing exclusion components on aliasing filesystems.""" + + if not parts: + return PurePosixPath(".") + if not workspace_module._DIRECTORY_BINDING_SUPPORTED: + first = _canonical_exclusion_tail_portably(root, parts, root_identity) + second = _canonical_exclusion_tail_portably(root, parts, root_identity) + if first != second: + raise OSError("Lean exclusion path changed while it was selected") + return first[0] + descriptors: list[int] = [] + descriptor: int | None = None + try: + descriptor = os.open(root, _DIRECTORY_FLAGS) + descriptors.append(descriptor) + opened_root = os.fstat(descriptor) + if (opened_root.st_dev, opened_root.st_ino) != root_identity: + raise OSError("Lean source root changed while exclusions were selected") + selected: list[str] = [] + for index, requested in enumerate(parts): + try: + requested_metadata = os.stat( + requested, + dir_fd=descriptor, + follow_symlinks=False, + ) + except (FileNotFoundError, NotADirectoryError): + selected.extend(parts[index:]) + break + signature = ( + requested_metadata.st_dev, + requested_metadata.st_ino, + requested_metadata.st_mode, + ) + names = tuple(sorted(os.listdir(descriptor))) + folded = unicodedata.normalize("NFC", requested).casefold() + matches = [] + for name in names: + if unicodedata.normalize("NFC", name).casefold() != folded: + continue + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if (metadata.st_dev, metadata.st_ino, metadata.st_mode) == signature: + matches.append(name) + if len(matches) != 1: + raise OSError("Lean exclusion path has no stable directory entry") + actual = matches[0] + selected.append(actual) + if tuple(sorted(os.listdir(descriptor))) != names: + raise OSError("Lean exclusion path changed while it was selected") + current = os.stat(actual, dir_fd=descriptor, follow_symlinks=False) + if (current.st_dev, current.st_ino, current.st_mode) != signature: + raise OSError("Lean exclusion path changed while it was selected") + if index == len(parts) - 1: + continue + if not stat.S_ISDIR(current.st_mode): + selected.extend(parts[index + 1 :]) + break + child = os.open(actual, _DIRECTORY_FLAGS, dir_fd=descriptor) + child_metadata = os.fstat(child) + if ( + child_metadata.st_dev, + child_metadata.st_ino, + child_metadata.st_mode, + ) != signature: + os.close(child) + raise OSError("Lean exclusion path changed while it was selected") + descriptors.append(child) + descriptor = child + return PurePosixPath(*selected) + finally: + for opened in reversed(descriptors): + os.close(opened) + + +def _canonical_exclusion_tail_portably( + root: Path, + parts: tuple[str, ...], + root_identity: tuple[int, int], +) -> tuple[PurePosixPath, tuple[tuple[str, tuple[int, int, int]], ...]]: + root_metadata = root.stat(follow_symlinks=False) + if (root_metadata.st_dev, root_metadata.st_ino) != root_identity: + raise OSError("Lean source root changed while exclusions were selected") + current = root + selected: list[str] = [] + observed: list[tuple[str, tuple[int, int, int]]] = [] + for index, requested in enumerate(parts): + requested_path = current / requested + try: + requested_metadata = requested_path.stat(follow_symlinks=False) + except (FileNotFoundError, NotADirectoryError): + selected.extend(parts[index:]) + break + signature = ( + requested_metadata.st_dev, + requested_metadata.st_ino, + requested_metadata.st_mode, + ) + names = tuple(sorted(entry.name for entry in os.scandir(current))) + folded = unicodedata.normalize("NFC", requested).casefold() + matches = [] + for name in names: + if unicodedata.normalize("NFC", name).casefold() != folded: + continue + metadata = (current / name).stat(follow_symlinks=False) + if (metadata.st_dev, metadata.st_ino, metadata.st_mode) == signature: + matches.append(name) + if len(matches) != 1: + raise OSError("Lean exclusion path has no stable directory entry") + actual = matches[0] + selected.append(actual) + actual_path = current / actual + final = actual_path.stat(follow_symlinks=False) + if ( + tuple(sorted(entry.name for entry in os.scandir(current))) != names + or (final.st_dev, final.st_ino, final.st_mode) != signature + ): + raise OSError("Lean exclusion path changed while it was selected") + observed.append(("/".join(selected), signature)) + if index == len(parts) - 1: + continue + if not stat.S_ISDIR(final.st_mode) or _is_reparse_point(final): + selected.extend(parts[index + 1 :]) + break + current = actual_path + final_root = root.stat(follow_symlinks=False) + if (final_root.st_dev, final_root.st_ino) != root_identity: + raise OSError("Lean source root changed while exclusions were selected") + return PurePosixPath(*selected), tuple(observed) + + +def _is_reparse_point(metadata: os.stat_result) -> bool: + attributes = getattr(metadata, "st_file_attributes", 0) + marker = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(attributes & marker) def _scan(text: str, relative: Path) -> list[Declaration]: @@ -144,6 +670,47 @@ def declaration_names(lean: str) -> list[str]: return [name.strip() for name in lean.replace(",", " ").split() if name.strip()] +def declaration_kind(intent: str | None) -> str | None: + """Return the kernel-checkable kind represented by authored intent.""" + + if intent is None: + return None + return DECLARATION_KIND_ALIASES.get(intent.strip().casefold()) + + +def declaration_keywords(intent: str | None) -> frozenset[str] | None: + """Return source keywords accepted for authored declaration intent.""" + + kind = declaration_kind(intent) + return _DECLARATION_KEYWORDS.get(kind) if kind is not None else None + + +def mathlib_module_name(source_file: str) -> str | None: + """Map a canonical ``Mathlib/**/*.lean`` source path to its module name.""" + + if not source_file or "\\" in source_file: + return None + path = PurePosixPath(source_file) + if path.is_absolute() or path.as_posix() != source_file: + return None + parts = path.parts + if not parts or any(part in {"", ".", ".."} for part in parts): + return None + if parts[0] != "Mathlib" and parts[0] != "Mathlib.lean": + return None + if not parts[-1].endswith(".lean") or parts[-1] == ".lean": + return None + module_parts = [*parts[:-1], parts[-1][: -len(".lean")]] + if not module_parts or module_parts[0] != "Mathlib": + return None + for part in module_parts: + if not part or not (part[0].isalpha() or part[0] == "_"): + return None + if any(not (character.isalnum() or character in "_'") for character in part): + return None + return ".".join(module_parts) + + @dataclass(frozen=True, slots=True) class SourceLinker: """Build permalinks into the project's Lean sources.""" @@ -169,12 +736,24 @@ def build_linker( *, repository_url: str | None = None, ref: str | None = None, + exclude_roots: Iterable[str | Path] = (), + source_index: SourceIndex | None = None, + detect_missing: bool = True, ) -> SourceLinker: """Index *lean_root* and resolve the repository coordinates to link against.""" + resolved_repository_url = repository_url + resolved_ref = ref + if detect_missing: + resolved_repository_url = repository_url or detect_repository_url(lean_root) + resolved_ref = ref or detect_ref(lean_root) return SourceLinker( - index=index_project(lean_root), - repository_url=repository_url or detect_repository_url(lean_root), - ref=ref or detect_ref(lean_root), + index=( + source_index + if source_index is not None + else index_project(lean_root, exclude_roots=exclude_roots) + ), + repository_url=resolved_repository_url, + ref=resolved_ref, ) @@ -224,12 +803,19 @@ def _git(root: str | Path, *arguments: str) -> str | None: __all__ = [ + "DECLARATION_KIND_ALIASES", + "IndexedSourceSnapshot", "Declaration", "SourceIndex", "SourceLinker", "build_linker", + "declaration_kind", + "declaration_keywords", "declaration_names", "detect_ref", "detect_repository_url", "index_project", + "mathlib_module_name", + "project_source_revision", + "snapshot_project_sources", ] diff --git a/autoform_cli/project/__init__.py b/autoform_cli/project/__init__.py new file mode 100644 index 00000000..c030b8ef --- /dev/null +++ b/autoform_cli/project/__init__.py @@ -0,0 +1,37 @@ +"""Offline Lean project inspection and supported release data.""" + +from .catalog import ProjectCatalogError, load_release_catalog, parse_release_catalog +from .create import ProjectCreateError, ProjectCreateResult, create_project +from .inspect import inspect_project +from .repair import ( + PROJECT_REPAIR_SCHEMA, + ProjectRepairConflict, + ProjectRepairError, + ProjectRepairResult, + repair_project, +) +from .model import ( + PROJECT_INSPECTION_SCHEMA, + RELEASE_CATALOG_SCHEMA, + ProjectInspection, + ReleaseCatalog, +) + +__all__ = [ + "PROJECT_INSPECTION_SCHEMA", + "RELEASE_CATALOG_SCHEMA", + "PROJECT_REPAIR_SCHEMA", + "ProjectCatalogError", + "ProjectCreateError", + "ProjectCreateResult", + "ProjectInspection", + "ProjectRepairConflict", + "ProjectRepairError", + "ProjectRepairResult", + "ReleaseCatalog", + "create_project", + "inspect_project", + "load_release_catalog", + "parse_release_catalog", + "repair_project", +] diff --git a/autoform_cli/project/catalog.py b/autoform_cli/project/catalog.py new file mode 100644 index 00000000..b5724536 --- /dev/null +++ b/autoform_cli/project/catalog.py @@ -0,0 +1,83 @@ +"""Load Autoform's bundled known-good Lean and Mathlib releases.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from .model import ( + RELEASE_CATALOG_SCHEMA, + LeanRelease, + MathlibRelease, + ReleaseCatalog, + SupportedRelease, +) + + +class ProjectCatalogError(ValueError): + """The bundled release catalog is missing or invalid.""" + + +def load_release_catalog() -> ReleaseCatalog: + try: + text = files("autoform_cli.project").joinpath("releases.json").read_text(encoding="utf-8") + except (OSError, TypeError, UnicodeError): + raise ProjectCatalogError("bundled project release catalog is unavailable") from None + try: + payload = json.loads(text) + except (TypeError, ValueError, RecursionError, MemoryError): + raise ProjectCatalogError("bundled project release catalog is invalid") from None + return parse_release_catalog(payload) + + +def parse_release_catalog(payload: Any) -> ReleaseCatalog: + if not isinstance(payload, dict) or set(payload) != {"schema", "releases"}: + raise ProjectCatalogError("release catalog has invalid fields") + if payload["schema"] != RELEASE_CATALOG_SCHEMA or not isinstance(payload["releases"], list): + raise ProjectCatalogError("release catalog has an invalid schema") + + releases: list[SupportedRelease] = [] + for entry in payload["releases"]: + releases.append(_parse_release(entry)) + if not releases: + raise ProjectCatalogError("release catalog is empty") + if tuple(release.id for release in releases) != tuple(sorted(release.id for release in releases)): + raise ProjectCatalogError("release catalog is not canonically ordered") + if len({release.id for release in releases}) != len(releases): + raise ProjectCatalogError("release catalog has duplicate release ids") + if sum(release.recommended for release in releases) != 1: + raise ProjectCatalogError("release catalog must have exactly one recommended release") + return ReleaseCatalog(RELEASE_CATALOG_SCHEMA, tuple(releases)) + + +def _parse_release(entry: Any) -> SupportedRelease: + expected = {"id", "channel", "recommended", "lean", "mathlib"} + if not isinstance(entry, dict) or set(entry) != expected: + raise ProjectCatalogError("release entry has invalid fields") + release_id = _string(entry["id"]) + channel = _string(entry["channel"]) + recommended = entry["recommended"] + if not isinstance(recommended, bool): + raise ProjectCatalogError("release recommendation must be boolean") + lean = _object(entry["lean"], {"toolchain", "version"}, "Lean release") + mathlib = _object(entry["mathlib"], {"git", "revision"}, "Mathlib release") + return SupportedRelease( + id=release_id, + channel=channel, + recommended=recommended, + lean=LeanRelease(toolchain=_string(lean["toolchain"]), version=_string(lean["version"])), + mathlib=MathlibRelease(git=_string(mathlib["git"]), revision=_string(mathlib["revision"])), + ) + + +def _object(value: Any, fields: set[str], name: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise ProjectCatalogError(f"{name} has invalid fields") + return value + + +def _string(value: Any) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise ProjectCatalogError("release catalog strings must be nonempty and trimmed") + return value diff --git a/autoform_cli/project/create.py b/autoform_cli/project/create.py new file mode 100644 index 00000000..4bf8d8e4 --- /dev/null +++ b/autoform_cli/project/create.py @@ -0,0 +1,619 @@ +"""Create a complete Autoform Lean project and publish it atomically.""" + +from __future__ import annotations + +import ctypes +import errno +import json +import os +import re +import secrets +import stat +from dataclasses import dataclass +from pathlib import Path + +from ..graph import GraphValidationError, load_graph +from ..scaffold import ScaffoldError, _normalize_autoform_source, scaffold_project +from .catalog import load_release_catalog +from .inplace import InPlaceCreateError, create_in_current_directory +from .inspect import inspect_project +from .model import SupportedRelease + +_PACKAGE_NAME = re.compile(r"[A-Z][A-Za-z0-9]*") +_FULL_SHA = re.compile(r"[0-9a-f]{40}") +_RESERVED_PACKAGE_NAMES = frozenset({"Mathlib", "Prop", "Sort", "Type"}) +_STAGE_ATTEMPTS = 32 + + +class ProjectCreateError(ValueError): + """A new project could not be created without risking existing data.""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(message) + + def as_dict(self) -> dict[str, object]: + return {"error": {"code": self.code, "message": self.message}, "ok": False} + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True, slots=True) +class ProjectCreateResult: + package: str + release: str + target: str + written: tuple[str, ...] + workflows_pinned: bool + + def as_dict(self) -> dict[str, object]: + return { + "ok": True, + "package": self.package, + "release": self.release, + "target": self.target, + "workflows_pinned": self.workflows_pinned, + "written": list(self.written), + } + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + +def create_project( + target: str | Path | None, + *, + package: str | None, + release_id: str | None, + autoform_source: str = "", + autoform_ref: str = "", +) -> ProjectCreateResult: + """Create a project at an absent target or in an empty current directory.""" + + current_directory = _is_current_directory_target(target) + requested = None if current_directory else _validate_target(target) + package_name = _validate_package(package) + release = _find_release(release_id) + workflow_source, workflow_ref = _validate_workflow_pin(autoform_source, autoform_ref) + if current_directory: + return _create_project_in_current_directory( + package_name, + release, + autoform_source=workflow_source, + autoform_ref=workflow_ref, + ) + assert requested is not None + return _create_project_at_absent_target( + requested, + package_name, + release, + autoform_source=workflow_source, + autoform_ref=workflow_ref, + ) + + +def _create_project_at_absent_target( + requested: Path, + package_name: str, + release: SupportedRelease, + *, + autoform_source: str, + autoform_ref: str, +) -> ProjectCreateResult: + """Keep the whole-directory publication used for absent targets.""" + + parent = requested.parent + parent_descriptor = _open_parent(parent) + workspace_name: str | None = None + workspace_path: Path | None = None + workspace_descriptor: int | None = None + workspace_identity: tuple[int, int] | None = None + published = False + try: + _require_absent(parent_descriptor, requested.name) + workspace_name = _create_stage(parent_descriptor, requested.name) + workspace_path = parent / workspace_name + workspace_metadata = os.stat( + workspace_name, dir_fd=parent_descriptor, follow_symlinks=False + ) + if not stat.S_ISDIR(workspace_metadata.st_mode): + raise OSError(errno.ENOTDIR, "staging path is not a directory") + workspace_identity = workspace_metadata.st_dev, workspace_metadata.st_ino + workspace_descriptor = _open_stage(parent_descriptor, workspace_name) + _require_stage_identity(parent_descriptor, workspace_name, workspace_descriptor) + stage_path = workspace_path / "project" + stage_path.mkdir(mode=0o700) + stage_descriptor = _open_stage(workspace_descriptor, "project") + try: + written, workflows_pinned = _build_staged_project( + stage_path, + package_name, + release, + autoform_source=autoform_source, + autoform_ref=autoform_ref, + ) + _require_stage_identity(parent_descriptor, workspace_name, workspace_descriptor) + _validate_staged_project(stage_path, release) + _require_stage_identity(parent_descriptor, workspace_name, workspace_descriptor) + os.fchmod(stage_descriptor, 0o755) + os.fsync(stage_descriptor) + _require_stage_identity(workspace_descriptor, "project", stage_descriptor) + try: + _rename_noreplace( + workspace_descriptor, + "project", + parent_descriptor, + requested.name, + ) + except FileExistsError: + raise ProjectCreateError( + "project-target-exists", + "The target already exists; project new never overwrites it.", + ) from None + published = True + finally: + os.close(stage_descriptor) + try: + os.rmdir(workspace_name, dir_fd=parent_descriptor) + except OSError: + pass + workspace_name = None + workspace_path = None + workspace_identity = None + return ProjectCreateResult( + package=package_name, + release=release.id, + target=requested.name, + written=written, + workflows_pinned=workflows_pinned, + ) + except ProjectCreateError: + raise + except (GraphValidationError, ScaffoldError): + raise ProjectCreateError( + "project-create-validation-failed", + "The staged project did not satisfy Autoform's project contracts.", + ) from None + except OSError: + raise ProjectCreateError( + "project-create-failed", + "Project creation failed; no project was created.", + ) from None + finally: + cleanup_failed = False + if not published and workspace_name is not None: + if workspace_identity is None: + cleanup_failed = True + elif workspace_descriptor is None: + cleanup_failed = not _remove_owned_empty_stage( + parent_descriptor, workspace_name, workspace_identity + ) + else: + cleanup_failed = not _remove_owned_stage( + parent_descriptor, + workspace_name, + workspace_descriptor, + workspace_identity, + ) + if workspace_descriptor is not None: + os.close(workspace_descriptor) + os.close(parent_descriptor) + if cleanup_failed: + raise ProjectCreateError( + "project-cleanup-failed", + "Project creation failed and owned temporary files could not be completely removed.", + ) + + +def _create_project_in_current_directory( + package_name: str, + release: SupportedRelease, + *, + autoform_source: str, + autoform_ref: str, +) -> ProjectCreateResult: + try: + result = create_in_current_directory( + package=package_name, + release=release.id, + autoform_source=autoform_source, + autoform_ref=autoform_ref, + build=lambda stage: _build_staged_project( + stage, + package_name, + release, + autoform_source=autoform_source, + autoform_ref=autoform_ref, + ), + validate=lambda stage: _validate_staged_project(stage, release), + ) + except ProjectCreateError: + raise + except InPlaceCreateError as error: + raise ProjectCreateError(error.code, error.message) from None + except (GraphValidationError, ScaffoldError): + raise ProjectCreateError( + "project-create-validation-failed", + "The staged project did not satisfy Autoform's project contracts.", + ) from None + except OSError: + raise ProjectCreateError( + "project-create-failed", + "Project creation failed; no project was created.", + ) from None + return ProjectCreateResult( + package=package_name, + release=release.id, + target=".", + written=result.written, + workflows_pinned=result.workflows_pinned, + ) + + +def _is_current_directory_target(target: str | Path | None) -> bool: + try: + return os.fspath(target) == "." + except TypeError: + return False + + +def _validate_package(package: str | None) -> str: + if ( + package is None + or _PACKAGE_NAME.fullmatch(package) is None + or package in _RESERVED_PACKAGE_NAMES + ): + raise ProjectCreateError( + "project-name-invalid", + "Project name must be an UpperCamelCase Lean identifier.", + ) + return package + + +def _find_release(release_id: str | None) -> SupportedRelease: + catalog = load_release_catalog() + release = next((item for item in catalog.releases if item.id == release_id), None) + if release is None: + raise ProjectCreateError( + "project-release-unknown", + "The requested release is not in the bundled release catalog.", + ) + return release + + +def _validate_workflow_pin(source: str, ref: str) -> tuple[str, str]: + """Validate explicit provenance before creating filesystem state.""" + + if not isinstance(source, str) or not isinstance(ref, str): + raise ProjectCreateError( + "project-provenance-invalid", + "Autoform workflow provenance must include a safe Git source and full commit SHA.", + ) + if not source and not ref: + return "", "" + safe_source = _normalize_autoform_source(source) + normalized_ref = ref.strip().lower() + if ( + not source + or not ref + or safe_source is None + or _FULL_SHA.fullmatch(normalized_ref) is None + ): + raise ProjectCreateError( + "project-provenance-invalid", + "Autoform workflow provenance must include a safe Git source and full commit SHA.", + ) + return safe_source, normalized_ref + + +def _validate_target(target: str | Path | None) -> Path: + try: + if target is None: + raise ValueError + raw = Path(target).expanduser().absolute() + except (OSError, RuntimeError, ValueError): + raise ProjectCreateError( + "project-target-invalid", "The project target cannot be resolved safely." + ) from None + if raw.name in {"", ".", ".."}: + raise ProjectCreateError( + "project-target-invalid", "The project target must name a new directory." + ) + parent = raw.parent + if not parent.exists(): + raise ProjectCreateError( + "project-parent-missing", "The target parent directory does not exist." + ) + if not parent.is_dir(): + raise ProjectCreateError( + "project-parent-invalid", "The target parent is not a directory." + ) + try: + metadata = parent.stat() + except OSError: + raise ProjectCreateError( + "project-parent-invalid", "The target parent is not a directory." + ) from None + writable_by_others = metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + if writable_by_others and not metadata.st_mode & stat.S_ISVTX: + raise ProjectCreateError( + "project-parent-unsafe", + "The target parent must not be group- or world-writable unless it is sticky.", + ) + try: + canonical_parent = parent.resolve(strict=True) + except (OSError, RuntimeError, ValueError): + raise ProjectCreateError( + "project-parent-invalid", "The target parent is not a directory." + ) from None + return canonical_parent / raw.name + + +def _open_parent(parent: Path) -> int: + if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_DIRECTORY") or os.open not in os.supports_dir_fd: + raise ProjectCreateError( + "project-create-safety-unavailable", + "This platform cannot create the project with the required path safety.", + ) + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + absolute = parent.absolute() + try: + descriptor = os.open(absolute.anchor, flags) + try: + for part in absolute.parts[1:]: + child = os.open(part, flags, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + except BaseException: + os.close(descriptor) + raise + except OSError: + raise ProjectCreateError( + "project-path-is-symlink", "The target path contains a symbolic link." + ) from None + return descriptor + + +def _require_absent(parent_descriptor: int, name: str) -> None: + try: + os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + return + except OSError: + raise ProjectCreateError( + "project-create-failed", "Project creation failed; no project was created." + ) from None + raise ProjectCreateError( + "project-target-exists", "The target already exists; project new never overwrites it." + ) + + +def _create_stage(parent_descriptor: int, target_name: str) -> str: + for _ in range(_STAGE_ATTEMPTS): + name = f".{target_name}.autoform-new-{secrets.token_hex(8)}" + try: + os.mkdir(name, mode=0o700, dir_fd=parent_descriptor) + return name + except FileExistsError: + continue + raise ProjectCreateError( + "project-create-failed", "Project creation failed; no project was created." + ) + + +def _open_stage(parent_descriptor: int, stage_name: str) -> int: + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + return os.open(stage_name, flags, dir_fd=parent_descriptor) + + +def _list_directory(directory_descriptor: int) -> list[str]: + """List through a fresh descriptor so earlier scans cannot leave it at EOF.""" + + fresh = _open_stage(directory_descriptor, ".") + try: + return os.listdir(fresh) + finally: + os.close(fresh) + + +def _descriptor_identity(descriptor: int) -> tuple[int, int]: + metadata = os.fstat(descriptor) + if not stat.S_ISDIR(metadata.st_mode): + raise OSError(errno.ENOTDIR, "staging path is not a directory") + return metadata.st_dev, metadata.st_ino + + +def _require_stage_identity( + workspace_descriptor: int, stage_name: str, stage_descriptor: int +) -> None: + expected = _descriptor_identity(stage_descriptor) + metadata = os.stat(stage_name, dir_fd=workspace_descriptor, follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode) or (metadata.st_dev, metadata.st_ino) != expected: + raise ProjectCreateError( + "project-create-failed", "Project creation failed; no project was created." + ) + + +def _build_staged_project( + stage: Path, + package: str, + release: SupportedRelease, + *, + autoform_source: str, + autoform_ref: str, +) -> tuple[tuple[str, ...], bool]: + source = stage / "src" / f"{package}.lean" + source.parent.mkdir() + files = { + stage / "lean-toolchain": f"{release.lean.toolchain}\n", + stage / "lakefile.toml": ( + f'name = "{package}"\n' + 'version = "0.1.0"\n' + f'defaultTargets = ["{package}"]\n\n' + '[[require]]\n' + 'name = "mathlib"\n' + f'git = "{release.mathlib.git}"\n' + f'rev = "{release.mathlib.revision}"\n\n' + '[[lean_lib]]\n' + f'name = "{package}"\n' + 'srcDir = "src"\n' + ), + source: ( + "import Mathlib\n\n" + f"namespace {package}\n\n" + "/-- Marker declaration for the initial project build. -/\n" + "def autoformProjectInitialized : Bool := true\n\n" + f"end {package}\n" + ), + } + for destination, content in files.items(): + with destination.open("x", encoding="utf-8", newline="\n") as output: + output.write(content) + output.flush() + os.fsync(output.fileno()) + scaffold = scaffold_project( + stage, + title=package, + autoform_source=autoform_source, + autoform_ref=autoform_ref, + discover_plugin_pin=False, + ) + written = tuple(sorted((*scaffold.written, "lakefile.toml", "lean-toolchain", f"src/{package}.lean"))) + return written, not scaffold.unpinned + + +def _validate_staged_project(stage: Path, release: SupportedRelease) -> None: + inspection = inspect_project(stage) + if ( + not inspection.ok + or inspection.compatibility.status != "supported" + or inspection.compatibility.release != release.id + or inspection.lake is None + ): + raise ProjectCreateError( + "project-create-validation-failed", + "The staged project did not satisfy Autoform's project contracts.", + ) + load_graph(stage / "blueprint") + + +def _rename_noreplace( + source_parent_descriptor: int, + source: str, + target_parent_descriptor: int, + target: str, +) -> None: + libc = ctypes.CDLL(None, use_errno=True) + source_bytes = os.fsencode(source) + target_bytes = os.fsencode(target) + if hasattr(libc, "renameatx_np"): + function = libc.renameatx_np + function.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + function.restype = ctypes.c_int + result = function( + source_parent_descriptor, + source_bytes, + target_parent_descriptor, + target_bytes, + 0x00000004, + ) + elif hasattr(libc, "renameat2"): + function = libc.renameat2 + function.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + function.restype = ctypes.c_int + result = function( + source_parent_descriptor, + source_bytes, + target_parent_descriptor, + target_bytes, + 1, + ) + else: + raise ProjectCreateError( + "project-create-safety-unavailable", + "This platform cannot atomically publish a new project without replacement.", + ) + if result == 0: + return + error = ctypes.get_errno() + if error in {errno.EEXIST, errno.ENOTEMPTY}: + raise FileExistsError(error, os.strerror(error), target) + if error in {errno.ENOSYS, errno.ENOTSUP}: + raise ProjectCreateError( + "project-create-safety-unavailable", + "This platform cannot atomically publish a new project without replacement.", + ) + raise OSError(error, os.strerror(error), target) + + +def _remove_owned_stage( + parent_descriptor: int, + stage_name: str, + stage_descriptor: int, + identity: tuple[int, int], +) -> bool: + try: + if _descriptor_identity(stage_descriptor) != identity: + return False + metadata = os.stat(stage_name, dir_fd=parent_descriptor, follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode) or (metadata.st_dev, metadata.st_ino) != identity: + return False + _remove_directory_contents(stage_descriptor) + os.rmdir(stage_name, dir_fd=parent_descriptor) + return True + except FileNotFoundError: + return True + except OSError: + return False + + +def _remove_owned_empty_stage( + parent_descriptor: int, + stage_name: str, + identity: tuple[int, int], +) -> bool: + """Remove a just-created stage that could not be opened, if still empty.""" + + try: + metadata = os.stat(stage_name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + not stat.S_ISDIR(metadata.st_mode) + or (metadata.st_dev, metadata.st_ino) != identity + ): + return False + os.rmdir(stage_name, dir_fd=parent_descriptor) + return True + except FileNotFoundError: + return True + except OSError: + return False + + +def _remove_directory_contents(directory_descriptor: int) -> None: + for name in _list_directory(directory_descriptor): + metadata = os.stat(name, dir_fd=directory_descriptor, follow_symlinks=False) + if stat.S_ISDIR(metadata.st_mode): + child = os.open( + name, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0), + dir_fd=directory_descriptor, + ) + try: + opened = os.fstat(child) + expected = opened.st_dev, opened.st_ino + if expected != (metadata.st_dev, metadata.st_ino): + raise OSError(errno.ESTALE, "directory changed during cleanup") + _remove_directory_contents(child) + current = os.stat(name, dir_fd=directory_descriptor, follow_symlinks=False) + if (current.st_dev, current.st_ino) != expected: + raise OSError(errno.ESTALE, "directory changed during cleanup") + finally: + os.close(child) + os.rmdir(name, dir_fd=directory_descriptor) + else: + os.unlink(name, dir_fd=directory_descriptor) + + +__all__ = ["ProjectCreateError", "ProjectCreateResult", "create_project"] diff --git a/autoform_cli/project/inplace.py b/autoform_cli/project/inplace.py new file mode 100644 index 00000000..9d05e153 --- /dev/null +++ b/autoform_cli/project/inplace.py @@ -0,0 +1,2533 @@ +"""Crash-recoverable population of an existing empty current directory. + +The absent-target creator can publish one directory with one rename. An +existing directory cannot use that trick without changing its inode, so this +module publishes one top-level entry at a time behind a durable write-ahead +journal. Every operation beneath the target is descriptor-relative. +""" + +from __future__ import annotations + +import ctypes +import errno +import hashlib +import json +import os +import secrets +import stat +import sys +import tempfile +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +try: + import fcntl +except ImportError: # pragma: no cover - imported on unsupported Windows only + fcntl = None # type: ignore[assignment] + + +MARKER = ".autoform-project-new" +STAGE = "project" +METADATA = "transaction.json" +MANIFEST = "manifest.json" +JOURNAL = "journal.jsonl" +SCHEMA = 1 +CONTROL_FILE_LIMIT = 32 * 1024 * 1024 +_CHUNK = 1024 * 1024 +_DIRECTORY_MODE = 0o700 +_CONTROL_MODE = 0o600 +_RECOVERY_MESSAGE = ( + "An interrupted or changed project transaction requires recovery; " + "no unverified data was removed." +) +_SAFETY_MESSAGE = ( + "This platform cannot create the project with the required path and " + "durability safety." +) +_LINUX_LOCAL_FILESYSTEMS = { + 0xEF53, # ext2/ext3/ext4 + 0x58465342, # XFS + 0x9123683E, # Btrfs + 0x01021994, # tmpfs, useful for process-crash tests + 0x794C7630, # overlayfs with a local upper layer +} +_DARWIN_LOCAL_FILESYSTEMS = {"apfs", "hfs"} + + +class InPlaceCreateError(ValueError): + """The current directory could not be populated safely.""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(message) + + +@dataclass(frozen=True, slots=True) +class InPlaceResult: + written: tuple[str, ...] + workflows_pinned: bool + + +@dataclass(frozen=True, slots=True) +class _Node: + path: str + kind: str + dev: int + ino: int + mode: int + nlink: int + size: int | None + sha256: str | None + + def as_dict(self) -> dict[str, object]: + return { + "dev": self.dev, + "ino": self.ino, + "kind": self.kind, + "mode": self.mode, + "nlink": self.nlink, + "path": self.path, + "sha256": self.sha256, + "size": self.size, + } + + def content_key(self) -> tuple[object, ...]: + return (self.path, self.kind, self.mode, self.size, self.sha256) + + +@dataclass(frozen=True, slots=True) +class _Control: + name: str + descriptor: int + dev: int + ino: int + + def as_dict(self) -> dict[str, int]: + return {"dev": self.dev, "ino": self.ino, "mode": _CONTROL_MODE} + + +@dataclass(slots=True) +class _Transaction: + transaction_id: str + marker_descriptor: int + marker_identity: tuple[int, int] + stage_descriptor: int | None + stage_identity: tuple[int, int] + metadata: _Control | None + manifest_control: _Control + journal_control: _Control | None + manifest: tuple[_Node, ...] + manifest_document: dict[str, object] + manifest_checksum: str + journal: list[dict[str, object]] + + +@dataclass(frozen=True, slots=True) +class _JournalState: + published: tuple[str, ...] + pending: str | None + committed: bool + rollback_started: bool + + +class _LinuxStatFs(ctypes.Structure): + _fields_ = [ + ("f_type", ctypes.c_long), + ("f_bsize", ctypes.c_long), + ("f_blocks", ctypes.c_ulong), + ("f_bfree", ctypes.c_ulong), + ("f_bavail", ctypes.c_ulong), + ("f_files", ctypes.c_ulong), + ("f_ffree", ctypes.c_ulong), + ("f_fsid", ctypes.c_int * 2), + ("f_namelen", ctypes.c_long), + ("f_frsize", ctypes.c_long), + ("f_flags", ctypes.c_long), + ("f_spare", ctypes.c_long * 4), + ] + + +class _DarwinStatFs(ctypes.Structure): + _fields_ = [ + ("f_bsize", ctypes.c_uint32), + ("f_iosize", ctypes.c_int32), + ("f_blocks", ctypes.c_uint64), + ("f_bfree", ctypes.c_uint64), + ("f_bavail", ctypes.c_uint64), + ("f_files", ctypes.c_uint64), + ("f_ffree", ctypes.c_uint64), + ("f_fsid", ctypes.c_int32 * 2), + ("f_owner", ctypes.c_uint32), + ("f_type", ctypes.c_uint32), + ("f_flags", ctypes.c_uint32), + ("f_fssubtype", ctypes.c_uint32), + ("f_fstypename", ctypes.c_char * 16), + ("f_mntonname", ctypes.c_char * 1024), + ("f_mntfromname", ctypes.c_char * 1024), + ("f_reserved", ctypes.c_uint32 * 8), + ] + + +def create_in_current_directory( + *, + package: str, + release: str, + autoform_source: str, + autoform_ref: str, + build: Callable[[Path], tuple[tuple[str, ...], bool]], + validate: Callable[[Path], None], +) -> InPlaceResult: + """Build outside the target, then populate the existing current directory.""" + + parent_descriptor: int | None = None + target_descriptor: int | None = None + transaction: _Transaction | None = None + try: + ( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) = _open_current_target() + _preflight(target_descriptor) + _lock_target(target_descriptor) + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + with tempfile.TemporaryDirectory(prefix="autoform-project-render-") as scratch: + rendered = Path(scratch).resolve() / STAGE + rendered.mkdir(mode=_DIRECTORY_MODE) + written, workflows_pinned = build(rendered) + source_parent_descriptor = _open_absolute_directory(rendered.parent) + source_descriptor: int | None = None + try: + source_descriptor = _open_directory(source_parent_descriptor, rendered.name) + source_metadata = os.fstat(source_descriptor) + source_identity = _stat_identity(source_metadata) + source_mode = stat.S_IMODE(source_metadata.st_mode) + _require_directory_entry( + source_parent_descriptor, + rendered.name, + source_descriptor, + source_identity, + source_mode, + ) + before_validation = _snapshot_tree(source_descriptor) + _require_expected_files(before_validation, written) + validate(rendered) + _require_directory_entry( + source_parent_descriptor, + rendered.name, + source_descriptor, + source_identity, + source_mode, + ) + if _snapshot_tree(source_descriptor) != before_validation: + raise InPlaceCreateError( + "project-create-validation-failed", + "The staged project changed while it was being validated.", + ) + invocation = { + "autoform_ref": autoform_ref, + "autoform_source": autoform_source, + "package": package, + "release": release, + "workflows_pinned": workflows_pinned, + "written": list(written), + } + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + names = set(_list_directory(target_descriptor)) + if not names: + transaction = _start_transaction( + target_descriptor, + target_identity, + target_mode, + source_descriptor, + before_validation, + invocation, + ) + elif MARKER in names: + transaction, rollback_pending = _load_transaction( + target_descriptor, + target_identity, + target_mode, + before_validation, + invocation, + ) + if rollback_pending: + if transaction.stage_descriptor is None: + _cleanup_marker( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + expected_roots=set(), + ) + elif not _rollback( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + ): + raise _recovery_required() + _close_transaction(transaction) + transaction = None + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + if _list_directory(target_descriptor): + raise _recovery_required() + transaction = _start_transaction( + target_descriptor, + target_identity, + target_mode, + source_descriptor, + before_validation, + invocation, + ) + elif _tree_has_same_content(target_descriptor, before_validation): + return InPlaceResult(written, workflows_pinned) + else: + raise InPlaceCreateError( + "project-target-not-empty", + "The current directory must be completely empty before project creation.", + ) + + assert transaction is not None + try: + _publish( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + ) + _finish( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + ) + except InPlaceCreateError as error: + if error.code in { + "project-recovery-required", + "project-target-changed", + "project-target-not-empty", + }: + raise + if not _rollback( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + ): + raise _recovery_required() from None + raise + except OSError: + if not _rollback( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + ): + raise _recovery_required() from None + raise InPlaceCreateError( + "project-create-failed", + "Project creation failed; no project was created.", + ) from None + return InPlaceResult(written, workflows_pinned) + finally: + if source_descriptor is not None: + os.close(source_descriptor) + os.close(source_parent_descriptor) + finally: + if transaction is not None: + _close_transaction(transaction) + if target_descriptor is not None: + os.close(target_descriptor) + if parent_descriptor is not None: + os.close(parent_descriptor) + + +def _checkpoint(name: str) -> None: + """A no-op boundary used by process-crash tests.""" + + +def _recovery_required() -> InPlaceCreateError: + return InPlaceCreateError("project-recovery-required", _RECOVERY_MESSAGE) + + +def _directory_flags() -> int: + return ( + os.O_RDONLY + | os.O_DIRECTORY + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0) + ) + + +def _open_absolute_directory(path: Path) -> int: + absolute = path.resolve(strict=True) + descriptor = os.open(absolute.anchor, _directory_flags()) + try: + for part in absolute.parts[1:]: + child = os.open(part, _directory_flags(), dir_fd=descriptor) + os.close(descriptor) + descriptor = child + except BaseException: + os.close(descriptor) + raise + return descriptor + + +def _open_directory(parent_descriptor: int, name: str) -> int: + return os.open(name, _directory_flags(), dir_fd=parent_descriptor) + + +def _list_directory(directory_descriptor: int) -> list[str]: + """List through a fresh descriptor so a prior scan cannot leave it at EOF.""" + + fresh = _open_directory(directory_descriptor, ".") + try: + return os.listdir(fresh) + finally: + os.close(fresh) + + +def _open_current_target() -> tuple[int, str, int, tuple[int, int], int]: + try: + target_descriptor = os.open(".", _directory_flags()) + except (AttributeError, OSError): + raise InPlaceCreateError( + "project-create-safety-unavailable", _SAFETY_MESSAGE + ) from None + try: + current = Path.cwd() + if current.parent == current or not current.name: + raise InPlaceCreateError( + "project-target-invalid", + "The filesystem root cannot be used as a project target.", + ) + parent_descriptor = _open_absolute_directory(current.parent) + metadata = os.fstat(target_descriptor) + identity = (metadata.st_dev, metadata.st_ino) + mode = stat.S_IMODE(metadata.st_mode) + try: + _require_target( + parent_descriptor, + current.name, + target_descriptor, + identity, + mode, + ) + except BaseException: + os.close(parent_descriptor) + raise + except BaseException: + os.close(target_descriptor) + raise + return parent_descriptor, current.name, target_descriptor, identity, mode + + +def _require_target( + parent_descriptor: int, + target_name: str, + target_descriptor: int, + identity: tuple[int, int], + mode: int, +) -> None: + try: + opened = os.fstat(target_descriptor) + named = os.stat( + target_name, dir_fd=parent_descriptor, follow_symlinks=False + ) + except OSError: + raise InPlaceCreateError( + "project-target-changed", + "The current directory changed while the project was being created.", + ) from None + if ( + not stat.S_ISDIR(opened.st_mode) + or not stat.S_ISDIR(named.st_mode) + or (opened.st_dev, opened.st_ino) != identity + or (named.st_dev, named.st_ino) != identity + or stat.S_IMODE(opened.st_mode) != mode + or stat.S_IMODE(named.st_mode) != mode + ): + raise InPlaceCreateError( + "project-target-changed", + "The current directory changed while the project was being created.", + ) + + +def _preflight(target_descriptor: int) -> None: + required = ( + hasattr(os, "O_DIRECTORY") + and hasattr(os, "O_NOFOLLOW") + and hasattr(os, "O_NONBLOCK") + and os.open in os.supports_dir_fd + and os.stat in os.supports_dir_fd + and os.stat in os.supports_follow_symlinks + and os.mkdir in os.supports_dir_fd + and os.unlink in os.supports_dir_fd + and os.rmdir in os.supports_dir_fd + and fcntl is not None + ) + if sys.platform not in {"linux", "darwin"} or not required: + raise InPlaceCreateError("project-create-safety-unavailable", _SAFETY_MESSAGE) + if not _filesystem_supported(target_descriptor) or _noreplace_function() is None: + raise InPlaceCreateError("project-create-safety-unavailable", _SAFETY_MESSAGE) + try: + flags = os.fstatvfs(target_descriptor).f_flag + if flags & getattr(os, "ST_RDONLY", 1): + raise OSError(errno.EROFS, "read-only filesystem") + _list_directory(target_descriptor) + os.fsync(target_descriptor) + except OSError: + raise InPlaceCreateError("project-create-safety-unavailable", _SAFETY_MESSAGE) from None + + +def _filesystem_supported(descriptor: int) -> bool: + try: + libc = ctypes.CDLL(None, use_errno=True) + function = libc.fstatfs + except (AttributeError, OSError): + return False + if sys.platform == "linux": + value = _LinuxStatFs() + function.argtypes = [ctypes.c_int, ctypes.POINTER(_LinuxStatFs)] + function.restype = ctypes.c_int + if function(descriptor, ctypes.byref(value)) != 0: + return False + bits = ctypes.sizeof(ctypes.c_long) * 8 + filesystem_type = int(value.f_type) & ((1 << bits) - 1) + return filesystem_type in _LINUX_LOCAL_FILESYSTEMS + if sys.platform == "darwin": + value = _DarwinStatFs() + function.argtypes = [ctypes.c_int, ctypes.POINTER(_DarwinStatFs)] + function.restype = ctypes.c_int + if function(descriptor, ctypes.byref(value)) != 0: + return False + filesystem_type = bytes(value.f_fstypename).split(b"\0", 1)[0] + try: + name = filesystem_type.decode("ascii") + except UnicodeDecodeError: + return False + return name in _DARWIN_LOCAL_FILESYSTEMS + return False + + +def _lock_target(target_descriptor: int) -> None: + assert fcntl is not None + try: + fcntl.flock(target_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise InPlaceCreateError( + "project-target-not-empty", + "Another project creation transaction is active in this directory.", + ) from None + except OSError: + raise InPlaceCreateError("project-create-safety-unavailable", _SAFETY_MESSAGE) from None + + +def _noreplace_function() -> tuple[Any, int] | None: + try: + libc = ctypes.CDLL(None, use_errno=True) + except OSError: + return None + if sys.platform == "darwin" and hasattr(libc, "renameatx_np"): + function = libc.renameatx_np + flag = 0x00000004 + elif sys.platform == "linux" and hasattr(libc, "renameat2"): + function = libc.renameat2 + flag = 1 + else: + return None + function.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + function.restype = ctypes.c_int + return function, flag + + +def _rename_noreplace( + source_parent: int, source: str, target_parent: int, target: str +) -> None: + implementation = _noreplace_function() + if implementation is None: + raise InPlaceCreateError("project-create-safety-unavailable", _SAFETY_MESSAGE) + function, flag = implementation + result = function( + source_parent, + os.fsencode(source), + target_parent, + os.fsencode(target), + flag, + ) + if result == 0: + return + error = ctypes.get_errno() + if error in {errno.EEXIST, errno.ENOTEMPTY}: + raise FileExistsError(error, os.strerror(error), target) + if error in {errno.ENOSYS, errno.ENOTSUP, errno.EINVAL, errno.EXDEV}: + raise InPlaceCreateError("project-create-safety-unavailable", _SAFETY_MESSAGE) + raise OSError(error, os.strerror(error), target) + + +def _snapshot_tree(directory_descriptor: int) -> tuple[_Node, ...]: + nodes: list[_Node] = [] + _snapshot_children(directory_descriptor, "", nodes) + return tuple(nodes) + + +def _snapshot_children( + directory_descriptor: int, prefix: str, nodes: list[_Node] +) -> None: + before_names = sorted(_list_directory(directory_descriptor)) + for name in before_names: + if not name or name in {".", ".."} or "/" in name or "\0" in name: + raise OSError(errno.EINVAL, "unsafe directory entry") + path = f"{prefix}/{name}" if prefix else name + metadata = os.stat(name, dir_fd=directory_descriptor, follow_symlinks=False) + if stat.S_ISDIR(metadata.st_mode): + child = _open_directory(directory_descriptor, name) + try: + opened = os.fstat(child) + if ( + not stat.S_ISDIR(opened.st_mode) + or _stat_identity(opened) != _stat_identity(metadata) + ): + raise OSError(errno.ESTALE, "directory changed during snapshot") + node = _node_from_directory(path, opened) + nodes.append(node) + _snapshot_children(child, path, nodes) + current = os.fstat(child) + if _directory_signature(current) != _directory_signature(opened): + raise OSError(errno.ESTALE, "directory changed during snapshot") + finally: + os.close(child) + elif stat.S_ISREG(metadata.st_mode): + nodes.append(_snapshot_file(directory_descriptor, name, path, metadata)) + else: + raise InPlaceCreateError( + "project-create-validation-failed", + "The staged project contains an unsupported filesystem entry.", + ) + if sorted(_list_directory(directory_descriptor)) != before_names: + raise OSError(errno.ESTALE, "directory changed during snapshot") + + +def _snapshot_file( + parent_descriptor: int, + name: str, + path: str, + metadata: os.stat_result, +) -> _Node: + flags = ( + os.O_RDONLY + | os.O_NOFOLLOW + | os.O_NONBLOCK + | getattr(os, "O_CLOEXEC", 0) + ) + descriptor = os.open(name, flags, dir_fd=parent_descriptor) + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or _file_signature(before) != _file_signature(metadata) + or before.st_nlink != 1 + ): + raise OSError(errno.ESTALE, "file changed during snapshot") + digest = hashlib.sha256() + while chunk := os.read(descriptor, _CHUNK): + digest.update(chunk) + after = os.fstat(descriptor) + if _file_signature(after) != _file_signature(before): + raise OSError(errno.ESTALE, "file changed during snapshot") + return _Node( + path=path, + kind="file", + dev=after.st_dev, + ino=after.st_ino, + mode=stat.S_IMODE(after.st_mode), + nlink=after.st_nlink, + size=after.st_size, + sha256=digest.hexdigest(), + ) + finally: + os.close(descriptor) + + +def _node_from_directory(path: str, metadata: os.stat_result) -> _Node: + return _Node( + path=path, + kind="directory", + dev=metadata.st_dev, + ino=metadata.st_ino, + mode=stat.S_IMODE(metadata.st_mode), + nlink=metadata.st_nlink, + size=None, + sha256=None, + ) + + +def _stat_identity(metadata: os.stat_result) -> tuple[int, int]: + return metadata.st_dev, metadata.st_ino + + +def _directory_signature(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + stat.S_IFMT(metadata.st_mode), + stat.S_IMODE(metadata.st_mode), + metadata.st_nlink, + ) + + +def _file_signature(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + stat.S_IFMT(metadata.st_mode), + stat.S_IMODE(metadata.st_mode), + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _require_expected_files(nodes: Sequence[_Node], written: Sequence[str]) -> None: + actual_files = {node.path for node in nodes if node.kind == "file"} + expected_files = set(written) + if actual_files != expected_files or MARKER in { + node.path.split("/", 1)[0] for node in nodes + }: + raise InPlaceCreateError( + "project-create-validation-failed", + "The staged project did not match the files reported by the project builder.", + ) + + +def _copy_tree( + source_descriptor: int, + destination_descriptor: int, + expected: Sequence[_Node], +) -> None: + expected_by_path = {node.path: node for node in expected} + _copy_children(source_descriptor, destination_descriptor, "", expected_by_path) + + +def _copy_children( + source_descriptor: int, + destination_descriptor: int, + prefix: str, + expected: Mapping[str, _Node], +) -> None: + names = sorted(_list_directory(source_descriptor)) + for name in names: + path = f"{prefix}/{name}" if prefix else name + node = expected.get(path) + if node is None: + raise OSError(errno.ESTALE, "source tree changed during copy") + metadata = os.stat(name, dir_fd=source_descriptor, follow_symlinks=False) + if node.kind == "directory": + if ( + not stat.S_ISDIR(metadata.st_mode) + or _stat_identity(metadata) != (node.dev, node.ino) + ): + raise OSError(errno.ESTALE, "source directory changed during copy") + os.mkdir(name, mode=_DIRECTORY_MODE, dir_fd=destination_descriptor) + source_child = _open_directory(source_descriptor, name) + destination_child = _open_directory(destination_descriptor, name) + try: + if _stat_identity(os.fstat(source_child)) != (node.dev, node.ino): + raise OSError(errno.ESTALE, "source directory changed during copy") + _copy_children(source_child, destination_child, path, expected) + os.fchmod(destination_child, node.mode) + os.fsync(destination_child) + finally: + os.close(destination_child) + os.close(source_child) + elif node.kind == "file": + _copy_file( + source_descriptor, + destination_descriptor, + name, + node, + ) + else: # pragma: no cover - manifest validation rejects this + raise OSError(errno.EINVAL, "unsupported staged entry") + if sorted(_list_directory(source_descriptor)) != names: + raise OSError(errno.ESTALE, "source tree changed during copy") + os.fsync(destination_descriptor) + + +def _copy_file( + source_parent: int, + destination_parent: int, + name: str, + expected: _Node, +) -> None: + source_flags = ( + os.O_RDONLY + | os.O_NOFOLLOW + | os.O_NONBLOCK + | getattr(os, "O_CLOEXEC", 0) + ) + source = os.open(name, source_flags, dir_fd=source_parent) + destination: int | None = None + try: + before = os.fstat(source) + if ( + not stat.S_ISREG(before.st_mode) + or _stat_identity(before) != (expected.dev, expected.ino) + or before.st_nlink != 1 + ): + raise OSError(errno.ESTALE, "source file changed during copy") + destination = os.open( + name, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0), + _CONTROL_MODE, + dir_fd=destination_parent, + ) + digest = hashlib.sha256() + size = 0 + while chunk := os.read(source, _CHUNK): + digest.update(chunk) + size += len(chunk) + _write_all(destination, chunk) + after = os.fstat(source) + if ( + _file_signature(after) != _file_signature(before) + or size != expected.size + or digest.hexdigest() != expected.sha256 + ): + raise OSError(errno.ESTALE, "source file changed during copy") + os.fchmod(destination, expected.mode) + os.fsync(destination) + finally: + if destination is not None: + os.close(destination) + os.close(source) + + +def _write_all(descriptor: int, payload: bytes) -> None: + view = memoryview(payload) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError(errno.EIO, "short filesystem write") + view = view[written:] + + +def _same_content(left: Sequence[_Node], right: Sequence[_Node]) -> bool: + return tuple(node.content_key() for node in left) == tuple( + node.content_key() for node in right + ) + + +def _tree_matches(directory_descriptor: int, expected: Sequence[_Node]) -> bool: + try: + return _snapshot_tree(directory_descriptor) == tuple(expected) + except (InPlaceCreateError, OSError): + return False + + +def _tree_has_same_content( + directory_descriptor: int, expected: Sequence[_Node] +) -> bool: + try: + return _same_content(_snapshot_tree(directory_descriptor), expected) + except (InPlaceCreateError, OSError): + return False + + +def _entry_matches( + parent_descriptor: int, name: str, expected: Sequence[_Node] +) -> bool: + subtree = tuple( + node + for node in expected + if node.path == name or node.path.startswith(f"{name}/") + ) + if not subtree: + return False + adjusted = tuple( + _Node( + path=node.path, + kind=node.kind, + dev=node.dev, + ino=node.ino, + mode=node.mode, + nlink=node.nlink, + size=node.size, + sha256=node.sha256, + ) + for node in subtree + ) + actual: list[_Node] = [] + try: + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if stat.S_ISDIR(metadata.st_mode): + child = _open_directory(parent_descriptor, name) + try: + actual.append(_node_from_directory(name, os.fstat(child))) + _snapshot_children(child, name, actual) + finally: + os.close(child) + elif stat.S_ISREG(metadata.st_mode): + actual.append(_snapshot_file(parent_descriptor, name, name, metadata)) + else: + return False + except (InPlaceCreateError, OSError): + return False + return tuple(actual) == adjusted + + +def _entry_is_owned_remainder( + parent_descriptor: int, name: str, expected: Sequence[_Node] +) -> bool: + index = {node.path: node for node in expected} + try: + return _node_is_owned_remainder(parent_descriptor, name, index) + except (InPlaceCreateError, OSError): + return False + + +def _node_is_owned_remainder( + parent_descriptor: int, path: str, expected: Mapping[str, _Node] +) -> bool: + node = expected.get(path) + if node is None: + return False + name = path.rsplit("/", 1)[-1] + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if node.kind == "file": + return _snapshot_file(parent_descriptor, name, path, metadata) == node + if ( + node.kind != "directory" + or not stat.S_ISDIR(metadata.st_mode) + or _stat_identity(metadata) != (node.dev, node.ino) + or stat.S_IMODE(metadata.st_mode) != node.mode + ): + return False + child = _open_directory(parent_descriptor, name) + try: + opened = os.fstat(child) + if ( + _stat_identity(opened) != (node.dev, node.ino) + or stat.S_IMODE(opened.st_mode) != node.mode + ): + return False + actual_names = set(_list_directory(child)) + expected_names = { + candidate.path.rsplit("/", 1)[-1] + for candidate in expected.values() + if candidate.path.startswith(f"{path}/") + and "/" not in candidate.path[len(path) + 1 :] + } + if not actual_names.issubset(expected_names): + return False + if not all( + _node_is_owned_remainder(child, f"{path}/{child_name}", expected) + for child_name in actual_names + ): + return False + current = os.fstat(child) + named = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + return ( + _stat_identity(current) == (node.dev, node.ino) + and _stat_identity(named) == (node.dev, node.ino) + and stat.S_IMODE(current.st_mode) == node.mode + and stat.S_IMODE(named.st_mode) == node.mode + ) + finally: + os.close(child) + + +def _exists(parent_descriptor: int, name: str) -> bool: + try: + os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + return False + return True + + +def _create_control(marker_descriptor: int, name: str) -> _Control: + descriptor = os.open( + name, + os.O_RDWR + | os.O_CREAT + | os.O_EXCL + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0), + _CONTROL_MODE, + dir_fd=marker_descriptor, + ) + os.fchmod(descriptor, _CONTROL_MODE) + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + os.close(descriptor) + raise OSError(errno.ESTALE, "control file changed") + return _Control(name, descriptor, metadata.st_dev, metadata.st_ino) + + +def _open_control( + marker_descriptor: int, + name: str, + expected: Mapping[str, object] | None = None, +) -> _Control: + descriptor = os.open( + name, + os.O_RDWR + | os.O_NOFOLLOW + | os.O_NONBLOCK + | getattr(os, "O_CLOEXEC", 0), + dir_fd=marker_descriptor, + ) + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != _CONTROL_MODE + ): + os.close(descriptor) + raise _recovery_required() + if expected is not None and ( + metadata.st_dev != expected.get("dev") + or metadata.st_ino != expected.get("ino") + or stat.S_IMODE(metadata.st_mode) != expected.get("mode") + ): + os.close(descriptor) + raise _recovery_required() + return _Control(name, descriptor, metadata.st_dev, metadata.st_ino) + + +def _control_matches(marker_descriptor: int, control: _Control) -> bool: + try: + opened = os.fstat(control.descriptor) + named = os.stat( + control.name, dir_fd=marker_descriptor, follow_symlinks=False + ) + except OSError: + return False + return ( + stat.S_ISREG(opened.st_mode) + and stat.S_ISREG(named.st_mode) + and opened.st_nlink == 1 + and named.st_nlink == 1 + and stat.S_IMODE(opened.st_mode) == _CONTROL_MODE + and stat.S_IMODE(named.st_mode) == _CONTROL_MODE + and _stat_identity(opened) == (control.dev, control.ino) + and _stat_identity(named) == (control.dev, control.ino) + ) + + +def _canonical_json(document: object) -> bytes: + return ( + json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + "\n" + ).encode("ascii") + + +def _write_control(control: _Control, document: object) -> None: + payload = _canonical_json(document) + if len(payload) > CONTROL_FILE_LIMIT: + raise OSError(errno.EFBIG, "control file too large") + os.ftruncate(control.descriptor, 0) + os.lseek(control.descriptor, 0, os.SEEK_SET) + _write_all(control.descriptor, payload) + os.fsync(control.descriptor) + + +def _strict_json(payload: bytes) -> object: + if not _json_depth_within_limit(payload): + raise ValueError("JSON nesting is too deep") + + def object_pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + return json.loads( + payload, + object_pairs_hook=object_pairs, + parse_constant=lambda value: (_ for _ in ()).throw( + ValueError(f"invalid JSON constant {value}") + ), + ) + + +def _json_depth_within_limit(payload: bytes, *, limit: int = 64) -> bool: + depth = 0 + in_string = False + escaped = False + for byte in payload: + if in_string: + if escaped: + escaped = False + elif byte == ord("\\"): + escaped = True + elif byte == ord('"'): + in_string = False + continue + if byte == ord('"'): + in_string = True + elif byte in {ord("["), ord("{")}: + depth += 1 + if depth > limit: + return False + elif byte in {ord("]"), ord("}")}: + depth -= 1 + if depth < 0: + return False + return depth == 0 and not in_string + + +def _read_control_bytes(control: _Control) -> bytes: + before = os.fstat(control.descriptor) + if before.st_size > CONTROL_FILE_LIMIT: + raise _recovery_required() + os.lseek(control.descriptor, 0, os.SEEK_SET) + chunks: list[bytes] = [] + remaining = CONTROL_FILE_LIMIT + 1 + while remaining: + chunk = os.read(control.descriptor, min(_CHUNK, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + payload = b"".join(chunks) + after = os.fstat(control.descriptor) + if ( + len(payload) > CONTROL_FILE_LIMIT + or _file_signature(after) != _file_signature(before) + or len(payload) != after.st_size + ): + raise _recovery_required() + return payload + + +def _start_transaction( + target_descriptor: int, + target_identity: tuple[int, int], + target_mode: int, + source_descriptor: int, + source_manifest: Sequence[_Node], + invocation: dict[str, object], +) -> _Transaction: + marker_descriptor: int | None = None + stage_descriptor: int | None = None + controls: list[_Control] = [] + try: + if _list_directory(target_descriptor): + raise InPlaceCreateError( + "project-target-not-empty", + "The current directory must be completely empty before project creation.", + ) + try: + os.mkdir(MARKER, mode=_DIRECTORY_MODE, dir_fd=target_descriptor) + except FileExistsError: + raise InPlaceCreateError( + "project-target-not-empty", + "Another project creation transaction is active in this directory.", + ) from None + os.fsync(target_descriptor) + marker_descriptor = _open_directory(target_descriptor, MARKER) + os.fchmod(marker_descriptor, _DIRECTORY_MODE) + marker_metadata = os.fstat(marker_descriptor) + marker_identity = _stat_identity(marker_metadata) + _require_directory_entry( + target_descriptor, + MARKER, + marker_descriptor, + marker_identity, + _DIRECTORY_MODE, + ) + + os.mkdir(STAGE, mode=_DIRECTORY_MODE, dir_fd=marker_descriptor) + stage_descriptor = _open_directory(marker_descriptor, STAGE) + os.fchmod(stage_descriptor, _DIRECTORY_MODE) + stage_metadata = os.fstat(stage_descriptor) + stage_identity = _stat_identity(stage_metadata) + _require_directory_entry( + marker_descriptor, + STAGE, + stage_descriptor, + stage_identity, + _DIRECTORY_MODE, + ) + os.fsync(marker_descriptor) + if set(_list_directory(target_descriptor)) != {MARKER}: + raise _recovery_required() + _copy_tree(source_descriptor, stage_descriptor, source_manifest) + if _snapshot_tree(source_descriptor) != tuple(source_manifest): + raise OSError(errno.ESTALE, "rendered project changed during copy") + staged_manifest = _snapshot_tree(stage_descriptor) + if not _same_content(source_manifest, staged_manifest): + raise OSError(errno.ESTALE, "copied project differs from rendered project") + if any(node.dev != target_identity[0] for node in staged_manifest): + raise OSError(errno.EXDEV, "staged project crossed a filesystem boundary") + + manifest_control = _create_control(marker_descriptor, MANIFEST) + controls.append(manifest_control) + journal_control = _create_control(marker_descriptor, JOURNAL) + controls.append(journal_control) + metadata_control = _create_control(marker_descriptor, METADATA) + controls.append(metadata_control) + transaction_id = secrets.token_hex(16) + control_identities = { + control.name: control.as_dict() for control in controls + } + manifest_document: dict[str, object] = { + "controls": control_identities, + "invocation": invocation, + "marker": { + "dev": marker_identity[0], + "ino": marker_identity[1], + "mode": _DIRECTORY_MODE, + }, + "nodes": [node.as_dict() for node in staged_manifest], + "schema": SCHEMA, + "stage": { + "dev": stage_identity[0], + "ino": stage_identity[1], + "mode": _DIRECTORY_MODE, + }, + "target": { + "dev": target_identity[0], + "ino": target_identity[1], + "mode": target_mode, + }, + "transaction_id": transaction_id, + } + manifest_payload = _canonical_json(manifest_document) + manifest_checksum = hashlib.sha256(manifest_payload).hexdigest() + _write_control(manifest_control, manifest_document) + metadata_document = { + "controls": control_identities, + "invocation": invocation, + "manifest_sha256": manifest_checksum, + "schema": SCHEMA, + "transaction_id": transaction_id, + } + _write_control(metadata_control, metadata_document) + transaction = _Transaction( + transaction_id=transaction_id, + marker_descriptor=marker_descriptor, + marker_identity=marker_identity, + stage_descriptor=stage_descriptor, + stage_identity=stage_identity, + metadata=metadata_control, + manifest_control=manifest_control, + journal_control=journal_control, + manifest=staged_manifest, + manifest_document=manifest_document, + manifest_checksum=manifest_checksum, + journal=[], + ) + begin = _journal_record( + transaction, + "begin", + manifest_sha256=manifest_checksum, + ) + _append_journal(transaction, begin) + os.fsync(marker_descriptor) + os.fsync(stage_descriptor) + os.fsync(target_descriptor) + return transaction + except InPlaceCreateError: + for control in controls: + _safe_close(control.descriptor) + if stage_descriptor is not None: + _safe_close(stage_descriptor) + if marker_descriptor is not None: + _safe_close(marker_descriptor) + raise + except OSError: + for control in controls: + _safe_close(control.descriptor) + if stage_descriptor is not None: + _safe_close(stage_descriptor) + if marker_descriptor is not None: + _safe_close(marker_descriptor) + raise _recovery_required() from None + + +def _load_transaction( + target_descriptor: int, + target_identity: tuple[int, int], + target_mode: int, + source_manifest: Sequence[_Node], + invocation: dict[str, object], +) -> tuple[_Transaction | None, bool]: + transaction: _Transaction | None = None + try: + marker_descriptor = _open_directory(target_descriptor, MARKER) + except OSError: + raise _recovery_required() from None + marker_metadata = os.fstat(marker_descriptor) + marker_identity = _stat_identity(marker_metadata) + controls: list[_Control] = [] + stage_descriptor: int | None = None + try: + _require_directory_entry( + target_descriptor, + MARKER, + marker_descriptor, + marker_identity, + _DIRECTORY_MODE, + ) + marker_names = set(_list_directory(marker_descriptor)) + if not marker_names: + raise _recovery_required() + if MANIFEST not in marker_names: + raise _recovery_required() + + manifest_control = _open_control(marker_descriptor, MANIFEST) + controls.append(manifest_control) + manifest_payload = _read_control_bytes(manifest_control) + try: + manifest_raw = _strict_json(manifest_payload) + except (UnicodeDecodeError, ValueError): + raise _recovery_required() from None + manifest_document, manifest = _validate_manifest_document( + manifest_raw, + target_identity, + target_mode, + invocation, + source_manifest, + ) + if manifest_payload != _canonical_json(manifest_document): + raise _recovery_required() + expected_controls = _mapping(manifest_document.get("controls")) + _require_control_identity(manifest_control, expected_controls, MANIFEST) + manifest_checksum = hashlib.sha256( + _canonical_json(manifest_document) + ).hexdigest() + transaction_id = _string(manifest_document.get("transaction_id")) + marker_identity_document = _mapping(manifest_document.get("marker")) + if ( + _integer(marker_identity_document.get("dev")), + _integer(marker_identity_document.get("ino")), + ) != marker_identity: + raise _recovery_required() + stage_identity_document = _mapping(manifest_document.get("stage")) + stage_identity = ( + _integer(stage_identity_document.get("dev")), + _integer(stage_identity_document.get("ino")), + ) + + if STAGE in marker_names: + stage_descriptor = _open_directory(marker_descriptor, STAGE) + _require_directory_entry( + marker_descriptor, + STAGE, + stage_descriptor, + stage_identity, + _DIRECTORY_MODE, + ) + + metadata_control: _Control | None = None + journal_control: _Control | None = None + if METADATA in marker_names: + metadata_control = _open_control( + marker_descriptor, + METADATA, + _mapping(expected_controls.get(METADATA)), + ) + controls.append(metadata_control) + if JOURNAL in marker_names: + journal_control = _open_control( + marker_descriptor, + JOURNAL, + _mapping(expected_controls.get(JOURNAL)), + ) + controls.append(journal_control) + + transaction = _Transaction( + transaction_id=transaction_id, + marker_descriptor=marker_descriptor, + marker_identity=marker_identity, + stage_descriptor=stage_descriptor, + stage_identity=stage_identity, + metadata=metadata_control, + manifest_control=manifest_control, + journal_control=journal_control, + manifest=manifest, + manifest_document=manifest_document, + manifest_checksum=manifest_checksum, + journal=[], + ) + marker_descriptor = -1 + stage_descriptor = None + controls.clear() + + root_names = _root_names(manifest) + complete = _all_published(target_descriptor, transaction) + if transaction.stage_descriptor is None: + allowed_cleanup_names = ( + {MANIFEST, METADATA, JOURNAL}, + {MANIFEST, JOURNAL}, + {MANIFEST}, + ) + if marker_names not in allowed_cleanup_names: + raise _recovery_required() + if transaction.metadata is not None: + _validate_metadata(transaction) + if transaction.journal_control is not None: + transaction.journal = _read_journal(transaction) + state = _journal_state(transaction, root_names) + if complete and not state.committed: + raise _recovery_required() + if not complete and not state.rollback_started: + raise _recovery_required() + elif marker_names != {MANIFEST}: + raise _recovery_required() + if complete: + return transaction, False + if set(_list_directory(target_descriptor)) == {MARKER}: + return transaction, True + raise _recovery_required() + + if marker_names != {STAGE, METADATA, MANIFEST, JOURNAL}: + raise _recovery_required() + if transaction.stage_descriptor is None: + raise _recovery_required() + _validate_metadata(transaction) + if transaction.journal_control is None: + raise _recovery_required() + transaction.journal = _read_journal(transaction) + state = _require_transaction_namespace(target_descriptor, transaction) + return transaction, state.rollback_started + except BaseException as error: + if transaction is not None: + _close_transaction(transaction) + transaction = None + for control in controls: + _safe_close(control.descriptor) + if stage_descriptor is not None: + _safe_close(stage_descriptor) + if marker_descriptor >= 0: + _safe_close(marker_descriptor) + if isinstance(error, OSError): + raise _recovery_required() from None + raise + + +def _validate_manifest_document( + raw: object, + target_identity: tuple[int, int], + target_mode: int, + invocation: dict[str, object], + source_manifest: Sequence[_Node], +) -> tuple[dict[str, object], tuple[_Node, ...]]: + document = _mapping(raw) + if set(document) != { + "controls", + "invocation", + "marker", + "nodes", + "schema", + "stage", + "target", + "transaction_id", + }: + raise _recovery_required() + if _integer(document.get("schema")) != SCHEMA: + raise _recovery_required() + transaction_id = _string(document.get("transaction_id")) + if len(transaction_id) != 32 or any( + character not in "0123456789abcdef" for character in transaction_id + ): + raise _recovery_required() + if _mapping(document.get("invocation")) != invocation: + raise _recovery_required() + target = _mapping(document.get("target")) + if ( + _integer(target.get("dev")), + _integer(target.get("ino")), + _integer(target.get("mode")), + ) != (*target_identity, target_mode): + raise _recovery_required() + marker = _mapping(document.get("marker")) + stage = _mapping(document.get("stage")) + for identity_document in (marker, stage): + if set(identity_document) != {"dev", "ino", "mode"}: + raise _recovery_required() + _integer(identity_document.get("dev")) + _integer(identity_document.get("ino")) + if _integer(identity_document.get("mode")) != _DIRECTORY_MODE: + raise _recovery_required() + controls = _mapping(document.get("controls")) + if set(controls) != {MANIFEST, JOURNAL, METADATA}: + raise _recovery_required() + for name in (MANIFEST, JOURNAL, METADATA): + value = _mapping(controls.get(name)) + if set(value) != {"dev", "ino", "mode"}: + raise _recovery_required() + _integer(value.get("dev")) + _integer(value.get("ino")) + if _integer(value.get("mode")) != _CONTROL_MODE: + raise _recovery_required() + raw_nodes = document.get("nodes") + if not isinstance(raw_nodes, list): + raise _recovery_required() + nodes = tuple(_parse_node(value) for value in raw_nodes) + _validate_nodes(nodes, target_identity[0]) + if not _same_content(nodes, source_manifest): + raise _recovery_required() + return document, nodes + + +def _parse_node(raw: object) -> _Node: + value = _mapping(raw) + if set(value) != { + "dev", + "ino", + "kind", + "mode", + "nlink", + "path", + "sha256", + "size", + }: + raise _recovery_required() + path = _string(value.get("path")) + kind = _string(value.get("kind")) + digest = value.get("sha256") + size = value.get("size") + if kind == "file": + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise _recovery_required() + parsed_size: int | None = _integer(size) + elif kind == "directory": + if digest is not None or size is not None: + raise _recovery_required() + parsed_size = None + digest = None + else: + raise _recovery_required() + return _Node( + path=path, + kind=kind, + dev=_integer(value.get("dev")), + ino=_integer(value.get("ino")), + mode=_integer(value.get("mode")), + nlink=_integer(value.get("nlink")), + size=parsed_size, + sha256=digest, + ) + + +def _validate_nodes(nodes: Sequence[_Node], target_device: int) -> None: + if not nodes or tuple(node.path for node in nodes) != tuple( + sorted(node.path for node in nodes) + ): + raise _recovery_required() + paths: set[str] = set() + identities: set[tuple[int, int]] = set() + for node in nodes: + parts = node.path.split("/") + if ( + not parts + or any(not part or part in {".", ".."} for part in parts) + or any("\0" in part for part in parts) + or parts[0] == MARKER + or node.path in paths + or node.dev != target_device + or node.mode < 0 + or node.mode > 0o777 + or node.nlink < 1 + or (node.dev, node.ino) in identities + ): + raise _recovery_required() + if len(parts) > 1 and "/".join(parts[:-1]) not in paths: + raise _recovery_required() + if node.kind == "file" and node.nlink != 1: + raise _recovery_required() + paths.add(node.path) + identities.add((node.dev, node.ino)) + + +def _mapping(value: object) -> dict[str, object]: + if not isinstance(value, dict) or not all( + isinstance(key, str) for key in value + ): + raise _recovery_required() + return value + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise _recovery_required() + return value + + +def _integer(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise _recovery_required() + return value + + +def _require_control_identity( + control: _Control, controls: Mapping[str, object], name: str +) -> None: + expected = _mapping(controls.get(name)) + if ( + control.dev != expected.get("dev") + or control.ino != expected.get("ino") + or expected.get("mode") != _CONTROL_MODE + ): + raise _recovery_required() + + +def _validate_metadata(transaction: _Transaction) -> None: + if transaction.metadata is None: + raise _recovery_required() + payload = _read_control_bytes(transaction.metadata) + try: + raw = _strict_json(payload) + except (UnicodeDecodeError, ValueError): + raise _recovery_required() from None + expected_controls = transaction.manifest_document["controls"] + expected = { + "controls": expected_controls, + "invocation": transaction.manifest_document["invocation"], + "manifest_sha256": transaction.manifest_checksum, + "schema": SCHEMA, + "transaction_id": transaction.transaction_id, + } + if raw != expected or payload != _canonical_json(expected): + raise _recovery_required() + + +def _journal_record( + transaction: _Transaction, kind: str, **fields: object +) -> dict[str, object]: + previous = transaction.journal[-1]["hash"] if transaction.journal else "" + record: dict[str, object] = { + "kind": kind, + "previous": previous, + "seq": len(transaction.journal), + "transaction_id": transaction.transaction_id, + **fields, + } + record["hash"] = hashlib.sha256(_canonical_json(record)).hexdigest() + return record + + +def _append_journal( + transaction: _Transaction, record: dict[str, object] +) -> None: + control = transaction.journal_control + if control is None or not _control_matches(transaction.marker_descriptor, control): + raise _recovery_required() + payload = _canonical_json(record) + current_size = os.fstat(control.descriptor).st_size + if current_size + len(payload) > CONTROL_FILE_LIMIT: + raise _recovery_required() + os.lseek(control.descriptor, 0, os.SEEK_END) + _write_all(control.descriptor, payload) + os.fsync(control.descriptor) + os.fsync(transaction.marker_descriptor) + transaction.journal.append(record) + + +def _read_journal(transaction: _Transaction) -> list[dict[str, object]]: + control = transaction.journal_control + if control is None: + raise _recovery_required() + payload = _read_control_bytes(control) + if not payload or len(payload) > CONTROL_FILE_LIMIT: + raise _recovery_required() + complete_payload = payload + incomplete_tail = b"" + if not payload.endswith(b"\n"): + boundary = payload.rfind(b"\n") + complete_payload = payload[: boundary + 1] + incomplete_tail = payload[boundary + 1 :] + records: list[dict[str, object]] = [] + previous = "" + for sequence, line in enumerate(complete_payload.splitlines()): + try: + parsed = _strict_json(line) + except (UnicodeDecodeError, ValueError): + raise _recovery_required() from None + record = _mapping(parsed) + if line + b"\n" != _canonical_json(record): + raise _recovery_required() + digest = record.get("hash") + unhashed = dict(record) + unhashed.pop("hash", None) + expected_hash = hashlib.sha256(_canonical_json(unhashed)).hexdigest() + if ( + not isinstance(digest, str) + or digest != expected_hash + or record.get("previous") != previous + or record.get("seq") != sequence + or record.get("transaction_id") != transaction.transaction_id + ): + raise _recovery_required() + previous = digest + records.append(record) + if incomplete_tail: + if not records: + begin = _journal_record( + transaction, + "begin", + manifest_sha256=transaction.manifest_checksum, + ) + successors = (begin,) + else: + successors = _legal_journal_successors(transaction, records) + if not any( + _canonical_json(candidate).startswith(incomplete_tail) + for candidate in successors + ): + raise _recovery_required() + if not _control_matches(transaction.marker_descriptor, control): + raise _recovery_required() + if not records: + repaired = _canonical_json(begin) + os.lseek(control.descriptor, 0, os.SEEK_SET) + _write_all(control.descriptor, repaired) + os.ftruncate(control.descriptor, len(repaired)) + else: + os.ftruncate(control.descriptor, len(complete_payload)) + os.fsync(control.descriptor) + os.fsync(transaction.marker_descriptor) + _checkpoint("journal-tail-truncated") + if not records: + return [begin] + return records + + +def _legal_journal_successors( + transaction: _Transaction, records: list[dict[str, object]] +) -> tuple[dict[str, object], ...]: + previous = transaction.journal + transaction.journal = records + try: + roots = _root_names(transaction.manifest) + state = _journal_state(transaction, roots) + if state.committed or state.rollback_started: + return () + candidates = [_journal_record(transaction, "rollback-started")] + if state.pending is not None: + candidates.append( + _journal_record( + transaction, + "published", + destination=state.pending, + ) + ) + elif len(state.published) < len(roots): + root = roots[len(state.published)] + candidates.append( + _journal_record( + transaction, + "prepared", + destination=root, + nodes=[ + node.as_dict() + for node in _subtree(transaction.manifest, root) + ], + ) + ) + else: + candidates.append(_journal_record(transaction, "committed")) + return tuple(candidates) + finally: + transaction.journal = previous + + +def _journal_state( + transaction: _Transaction, roots: Sequence[str] +) -> _JournalState: + records = transaction.journal + if not records: + raise _recovery_required() + begin = records[0] + if ( + begin.get("kind") != "begin" + or begin.get("manifest_sha256") != transaction.manifest_checksum + or set(begin) + != { + "hash", + "kind", + "manifest_sha256", + "previous", + "seq", + "transaction_id", + } + ): + raise _recovery_required() + published: list[str] = [] + pending: str | None = None + committed = False + rollback_started = False + for record in records[1:]: + kind = record.get("kind") + if committed or rollback_started: + raise _recovery_required() + if kind == "prepared": + if pending is not None or len(published) >= len(roots): + raise _recovery_required() + destination = roots[len(published)] + if ( + record.get("destination") != destination + or record.get("nodes") + != [node.as_dict() for node in _subtree(transaction.manifest, destination)] + or set(record) + != { + "destination", + "hash", + "kind", + "nodes", + "previous", + "seq", + "transaction_id", + } + ): + raise _recovery_required() + pending = destination + elif kind == "published": + if pending is None or record.get("destination") != pending: + raise _recovery_required() + if set(record) != { + "destination", + "hash", + "kind", + "previous", + "seq", + "transaction_id", + }: + raise _recovery_required() + published.append(pending) + pending = None + elif kind == "committed": + if pending is not None or published != list(roots): + raise _recovery_required() + if set(record) != { + "hash", + "kind", + "previous", + "seq", + "transaction_id", + }: + raise _recovery_required() + committed = True + elif kind == "rollback-started": + if set(record) != { + "hash", + "kind", + "previous", + "seq", + "transaction_id", + }: + raise _recovery_required() + rollback_started = True + else: + raise _recovery_required() + return _JournalState(tuple(published), pending, committed, rollback_started) + + +def _root_names(nodes: Sequence[_Node]) -> tuple[str, ...]: + return tuple(node.path for node in nodes if "/" not in node.path) + + +def _subtree(nodes: Sequence[_Node], root: str) -> tuple[_Node, ...]: + return tuple( + node for node in nodes if node.path == root or node.path.startswith(f"{root}/") + ) + + +def _require_directory_entry( + parent_descriptor: int, + name: str, + descriptor: int, + identity: tuple[int, int], + mode: int, +) -> None: + try: + opened = os.fstat(descriptor) + named = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except OSError: + raise _recovery_required() from None + if ( + not stat.S_ISDIR(opened.st_mode) + or not stat.S_ISDIR(named.st_mode) + or _stat_identity(opened) != identity + or _stat_identity(named) != identity + or stat.S_IMODE(opened.st_mode) != mode + or stat.S_IMODE(named.st_mode) != mode + ): + raise _recovery_required() + + +def _require_transaction_controls( + target_descriptor: int, transaction: _Transaction +) -> None: + _require_directory_entry( + target_descriptor, + MARKER, + transaction.marker_descriptor, + transaction.marker_identity, + _DIRECTORY_MODE, + ) + expected_marker_names = {MANIFEST} + if transaction.stage_descriptor is not None: + expected_marker_names.add(STAGE) + if transaction.metadata is not None: + expected_marker_names.add(METADATA) + if transaction.journal_control is not None: + expected_marker_names.add(JOURNAL) + if set(_list_directory(transaction.marker_descriptor)) != expected_marker_names: + raise _recovery_required() + if transaction.stage_descriptor is not None: + _require_directory_entry( + transaction.marker_descriptor, + STAGE, + transaction.stage_descriptor, + transaction.stage_identity, + _DIRECTORY_MODE, + ) + if not _control_matches( + transaction.marker_descriptor, transaction.manifest_control + ): + raise _recovery_required() + if _read_control_bytes(transaction.manifest_control) != _canonical_json( + transaction.manifest_document + ): + raise _recovery_required() + if transaction.metadata is not None: + if not _control_matches(transaction.marker_descriptor, transaction.metadata): + raise _recovery_required() + _validate_metadata(transaction) + if transaction.journal_control is not None: + if not _control_matches( + transaction.marker_descriptor, transaction.journal_control + ): + raise _recovery_required() + if _read_journal(transaction) != transaction.journal: + raise _recovery_required() + + +def _entry_state( + target_descriptor: int, + stage_descriptor: int, + root: str, + manifest: Sequence[_Node], +) -> tuple[bool, bool, bool, bool]: + source_exists = _exists(stage_descriptor, root) + target_exists = _exists(target_descriptor, root) + source_matches = source_exists and _entry_matches(stage_descriptor, root, manifest) + target_matches = target_exists and _entry_matches(target_descriptor, root, manifest) + return source_exists, source_matches, target_exists, target_matches + + +def _require_transaction_namespace( + target_descriptor: int, transaction: _Transaction +) -> _JournalState: + _require_transaction_controls(target_descriptor, transaction) + if transaction.stage_descriptor is None: + raise _recovery_required() + roots = _root_names(transaction.manifest) + state = _journal_state(transaction, roots) + if state.committed or state.rollback_started: + return state + target_roots: set[str] = set() + source_roots: set[str] = set() + for index, root in enumerate(roots): + source_exists, source_matches, target_exists, target_matches = _entry_state( + target_descriptor, + transaction.stage_descriptor, + root, + transaction.manifest, + ) + if root in state.published: + if source_exists or not target_matches: + raise _recovery_required() + target_roots.add(root) + elif root == state.pending: + if source_matches and not target_exists: + source_roots.add(root) + elif target_matches and not source_exists: + target_roots.add(root) + else: + raise _recovery_required() + else: + if index < len(state.published) or not source_matches or target_exists: + raise _recovery_required() + source_roots.add(root) + if set(_list_directory(target_descriptor)) != {MARKER, *target_roots}: + raise _recovery_required() + if set(_list_directory(transaction.stage_descriptor)) != source_roots: + raise _recovery_required() + if set(_list_directory(transaction.marker_descriptor)) != { + STAGE, + METADATA, + MANIFEST, + JOURNAL, + }: + raise _recovery_required() + return state + + +def _publish( + parent_descriptor: int, + target_name: str, + target_descriptor: int, + target_identity: tuple[int, int], + target_mode: int, + transaction: _Transaction, +) -> None: + if transaction.stage_descriptor is None: + if not _all_published(target_descriptor, transaction): + raise _recovery_required() + return + roots = _root_names(transaction.manifest) + while True: + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + state = _require_transaction_namespace(target_descriptor, transaction) + if state.rollback_started: + raise _recovery_required() + if state.committed or len(state.published) == len(roots): + return + root = roots[len(state.published)] + if state.pending is None: + record = _journal_record( + transaction, + "prepared", + destination=root, + nodes=[node.as_dict() for node in _subtree(transaction.manifest, root)], + ) + _append_journal(transaction, record) + _checkpoint(f"prepared:{root}") + state = _require_transaction_namespace(target_descriptor, transaction) + if state.pending != root: + raise _recovery_required() + source_exists, source_matches, target_exists, target_matches = _entry_state( + target_descriptor, + transaction.stage_descriptor, + root, + transaction.manifest, + ) + if source_matches and not target_exists: + try: + _rename_noreplace( + transaction.stage_descriptor, + root, + target_descriptor, + root, + ) + except FileExistsError: + raise _recovery_required() from None + _checkpoint(f"renamed:{root}") + source_exists, source_matches, target_exists, target_matches = _entry_state( + target_descriptor, + transaction.stage_descriptor, + root, + transaction.manifest, + ) + if source_exists or not target_matches: + raise _recovery_required() + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + os.fsync(transaction.stage_descriptor) + os.fsync(target_descriptor) + record = _journal_record( + transaction, + "published", + destination=root, + ) + _append_journal(transaction, record) + os.fsync(target_descriptor) + _checkpoint(f"published:{root}") + + +def _all_published(target_descriptor: int, transaction: _Transaction) -> bool: + roots = _root_names(transaction.manifest) + if set(_list_directory(target_descriptor)) != {MARKER, *roots}: + return False + return all( + _entry_matches(target_descriptor, root, transaction.manifest) + for root in roots + ) + + +def _finish( + parent_descriptor: int, + target_name: str, + target_descriptor: int, + target_identity: tuple[int, int], + target_mode: int, + transaction: _Transaction, +) -> None: + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + if not _all_published(target_descriptor, transaction): + raise _recovery_required() + if transaction.journal_control is not None: + state = _journal_state( + transaction, _root_names(transaction.manifest) + ) + if state.rollback_started: + raise _recovery_required() + if not state.committed: + if state.pending is not None or len(state.published) != len( + _root_names(transaction.manifest) + ): + raise _recovery_required() + _append_journal(transaction, _journal_record(transaction, "committed")) + os.fsync(target_descriptor) + _checkpoint("committed") + _cleanup_marker( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + expected_roots=set(_root_names(transaction.manifest)), + ) + + +def _cleanup_marker( + parent_descriptor: int, + target_name: str, + target_descriptor: int, + target_identity: tuple[int, int], + target_mode: int, + transaction: _Transaction, + *, + expected_roots: set[str], +) -> None: + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + if set(_list_directory(target_descriptor)) != {MARKER, *expected_roots}: + raise _recovery_required() + _require_directory_entry( + target_descriptor, + MARKER, + transaction.marker_descriptor, + transaction.marker_identity, + _DIRECTORY_MODE, + ) + expected_marker_names = {MANIFEST} + if transaction.stage_descriptor is not None: + expected_marker_names.add(STAGE) + if transaction.metadata is not None: + expected_marker_names.add(METADATA) + if transaction.journal_control is not None: + expected_marker_names.add(JOURNAL) + if set(_list_directory(transaction.marker_descriptor)) != expected_marker_names: + raise _recovery_required() + if transaction.stage_descriptor is not None: + _require_directory_entry( + transaction.marker_descriptor, + STAGE, + transaction.stage_descriptor, + transaction.stage_identity, + _DIRECTORY_MODE, + ) + if _list_directory(transaction.stage_descriptor): + raise _recovery_required() + os.rmdir(STAGE, dir_fd=transaction.marker_descriptor) + os.fsync(transaction.marker_descriptor) + os.close(transaction.stage_descriptor) + transaction.stage_descriptor = None + _checkpoint("stage-removed") + if transaction.metadata is not None: + _require_directory_entry( + target_descriptor, + MARKER, + transaction.marker_descriptor, + transaction.marker_identity, + _DIRECTORY_MODE, + ) + _unlink_control(transaction, transaction.metadata) + transaction.metadata = None + _checkpoint("metadata-removed") + if transaction.journal_control is not None: + _require_directory_entry( + target_descriptor, + MARKER, + transaction.marker_descriptor, + transaction.marker_identity, + _DIRECTORY_MODE, + ) + _unlink_control(transaction, transaction.journal_control) + transaction.journal_control = None + _checkpoint("journal-removed") + _require_directory_entry( + target_descriptor, + MARKER, + transaction.marker_descriptor, + transaction.marker_identity, + _DIRECTORY_MODE, + ) + if not _control_matches( + transaction.marker_descriptor, transaction.manifest_control + ): + raise _recovery_required() + os.unlink(MANIFEST, dir_fd=transaction.marker_descriptor) + os.fsync(transaction.marker_descriptor) + os.close(transaction.manifest_control.descriptor) + transaction.manifest_control = _Control(MANIFEST, -1, -1, -1) + _checkpoint("manifest-removed") + if _list_directory(transaction.marker_descriptor): + raise _recovery_required() + _require_directory_entry( + target_descriptor, + MARKER, + transaction.marker_descriptor, + transaction.marker_identity, + _DIRECTORY_MODE, + ) + os.rmdir(MARKER, dir_fd=target_descriptor) + os.fsync(target_descriptor) + _checkpoint("marker-removed") + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + if set(_list_directory(target_descriptor)) != expected_roots: + raise _recovery_required() + if expected_roots and not all( + _entry_matches(target_descriptor, root, transaction.manifest) + for root in expected_roots + ): + raise _recovery_required() + + +def _unlink_control(transaction: _Transaction, control: _Control) -> None: + if not _control_matches(transaction.marker_descriptor, control): + raise _recovery_required() + os.unlink(control.name, dir_fd=transaction.marker_descriptor) + os.fsync(transaction.marker_descriptor) + os.close(control.descriptor) + + +def _rollback( + parent_descriptor: int, + target_name: str, + target_descriptor: int, + target_identity: tuple[int, int], + target_mode: int, + transaction: _Transaction, +) -> bool: + try: + _require_target( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + ) + if transaction.stage_descriptor is None or transaction.journal_control is None: + return False + state = _require_transaction_namespace(target_descriptor, transaction) + if state.committed: + return False + locations: dict[str, str] = {} + source_roots: set[str] = set() + target_roots: set[str] = set() + for root in _root_names(transaction.manifest): + source_exists, source_matches, target_exists, target_matches = _entry_state( + target_descriptor, + transaction.stage_descriptor, + root, + transaction.manifest, + ) + if ( + source_exists + and not target_exists + and ( + source_matches + or ( + state.rollback_started + and _entry_is_owned_remainder( + transaction.stage_descriptor, + root, + transaction.manifest, + ) + ) + ) + ): + locations[root] = "stage" + source_roots.add(root) + elif ( + target_exists + and not source_exists + and ( + target_matches + or ( + state.rollback_started + and _entry_is_owned_remainder( + target_descriptor, + root, + transaction.manifest, + ) + ) + ) + ): + locations[root] = "target" + target_roots.add(root) + elif ( + state.rollback_started + and not source_exists + and not target_exists + ): + locations[root] = "missing" + else: + return False + if set(_list_directory(target_descriptor)) != {MARKER, *target_roots}: + return False + if set(_list_directory(transaction.stage_descriptor)) != source_roots: + return False + if set(_list_directory(transaction.marker_descriptor)) != { + STAGE, + METADATA, + MANIFEST, + JOURNAL, + }: + return False + if not state.rollback_started: + _append_journal( + transaction, _journal_record(transaction, "rollback-started") + ) + _checkpoint("rollback-started") + _require_directory_entry( + target_descriptor, + MARKER, + transaction.marker_descriptor, + transaction.marker_identity, + _DIRECTORY_MODE, + ) + for root, location in locations.items(): + if location != "target": + continue + if not _entry_is_owned_remainder( + target_descriptor, root, transaction.manifest + ): + return False + _rename_noreplace( + target_descriptor, + root, + transaction.stage_descriptor, + root, + ) + os.fsync(target_descriptor) + os.fsync(transaction.stage_descriptor) + _checkpoint(f"rollback-restored:{root}") + if _exists(target_descriptor, root) or not _entry_is_owned_remainder( + transaction.stage_descriptor, root, transaction.manifest + ): + if not _exists(target_descriptor, root) and _exists( + transaction.stage_descriptor, root + ): + _rename_noreplace( + transaction.stage_descriptor, + root, + target_descriptor, + root, + ) + os.fsync(transaction.stage_descriptor) + os.fsync(target_descriptor) + return False + source_roots.add(root) + target_roots.discard(root) + if set(_list_directory(target_descriptor)) != {MARKER}: + return False + if set(_list_directory(transaction.stage_descriptor)) != source_roots: + return False + for root in _root_names(transaction.manifest): + if not _exists(transaction.stage_descriptor, root): + continue + if not _entry_is_owned_remainder( + transaction.stage_descriptor, root, transaction.manifest + ): + return False + _remove_manifest_entry( + transaction.stage_descriptor, root, transaction.manifest + ) + os.fsync(transaction.stage_descriptor) + _checkpoint(f"rollback-removed:{root}") + if _list_directory(transaction.stage_descriptor): + return False + _cleanup_marker( + parent_descriptor, + target_name, + target_descriptor, + target_identity, + target_mode, + transaction, + expected_roots=set(), + ) + return True + except (InPlaceCreateError, OSError): + return False + + +def _remove_manifest_entry( + parent_descriptor: int, + root: str, + manifest: Sequence[_Node], +) -> None: + index = {node.path: node for node in manifest} + _remove_manifest_node(parent_descriptor, root, index) + + +def _remove_manifest_node( + parent_descriptor: int, path: str, index: Mapping[str, _Node] +) -> None: + node = index[path] + name = path.rsplit("/", 1)[-1] + try: + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + return + if node.kind == "file": + actual = _snapshot_file(parent_descriptor, name, path, metadata) + if actual != node: + raise OSError(errno.ESTALE, "owned file changed during rollback") + os.unlink(name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + _checkpoint(f"rollback-node-removed:{path}") + return + child = _open_directory(parent_descriptor, name) + try: + opened = os.fstat(child) + if ( + not stat.S_ISDIR(opened.st_mode) + or _stat_identity(opened) != (node.dev, node.ino) + or stat.S_IMODE(opened.st_mode) != node.mode + ): + raise OSError(errno.ESTALE, "owned directory changed during rollback") + expected_children = { + candidate.path.rsplit("/", 1)[-1]: candidate.path + for candidate in index.values() + if candidate.path.startswith(f"{path}/") + and "/" not in candidate.path[len(path) + 1 :] + } + actual_children = sorted(_list_directory(child)) + if not set(actual_children).issubset(expected_children): + raise OSError(errno.ESTALE, "owned directory changed during rollback") + for child_name in actual_children: + _remove_manifest_node(child, expected_children[child_name], index) + os.fsync(child) + finally: + os.close(child) + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + not stat.S_ISDIR(current.st_mode) + or _stat_identity(current) != (node.dev, node.ino) + or stat.S_IMODE(current.st_mode) != node.mode + ): + raise OSError(errno.ESTALE, "owned directory changed during rollback") + os.rmdir(name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + _checkpoint(f"rollback-node-removed:{path}") + + +def _close_transaction(transaction: _Transaction) -> None: + descriptors = [ + transaction.stage_descriptor, + transaction.metadata.descriptor if transaction.metadata is not None else None, + ( + transaction.manifest_control.descriptor + if transaction.manifest_control.descriptor >= 0 + else None + ), + ( + transaction.journal_control.descriptor + if transaction.journal_control is not None + else None + ), + transaction.marker_descriptor, + ] + for descriptor in descriptors: + if descriptor is not None: + _safe_close(descriptor) + + +def _safe_close(descriptor: int) -> None: + try: + os.close(descriptor) + except OSError: + pass + + +__all__ = ["InPlaceCreateError", "InPlaceResult", "create_in_current_directory"] diff --git a/autoform_cli/project/inspect.py b/autoform_cli/project/inspect.py new file mode 100644 index 00000000..9dcf3a0a --- /dev/null +++ b/autoform_cli/project/inspect.py @@ -0,0 +1,1265 @@ +"""Deterministically inspect local Lean project configuration without executing it.""" + +from __future__ import annotations + +import errno +import hashlib +import json +import os +import re +import stat +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any +from urllib.parse import urlsplit + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib + +from .catalog import load_release_catalog +from .model import ( + PROJECT_INSPECTION_SCHEMA, + AutoformProject, + LakeProject, + LakeTarget, + LeanProject, + MathlibProject, + ProjectCompatibility, + ProjectDiagnostic, + ProjectInspection, + ReleaseCatalog, +) +from ..workspace_manifest import WORKSPACE_FILE, WorkspaceError, parse_workspace + +_MAX_CONFIG_BYTES = 2 * 1024 * 1024 +_MAX_STRUCTURAL_DEPTH = 128 +_PROJECT_MARKERS = ( + "lakefile.toml", + "lakefile.lean", + "lean-toolchain", + "blueprint", + WORKSPACE_FILE, +) +_TOOLCHAIN = re.compile(r"leanprover/lean4:(?Pv[0-9]+\.[0-9]+\.[0-9]+)") +_RESERVOIR_SCOPE = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?") +# Lake's StdVer: a major.minor.patch triple with an optional `-` suffix that +# runs to the end of the string. +_LAKE_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[^ \t\r\n]+)?") +_SEVERITY_ORDER = {"error": 0, "warning": 1, "info": 2} +_LEAN_ID_BEGIN_ESCAPE = "«" +_LEAN_ID_END_ESCAPE = "»" + + +class _InvalidLakeField(ValueError): + pass + + +class _NonportableLakePath(ValueError): + pass + + +class _DuplicateMathlibRequirement(ValueError): + pass + + +def inspect_project(target: str | Path, *, catalog: ReleaseCatalog | None = None) -> ProjectInspection: + release_catalog = catalog or load_release_catalog() + diagnostics: list[ProjectDiagnostic] = [] + root_descriptor = _discover_root(target, diagnostics) + if root_descriptor is None: + return _inspection(diagnostics, release_catalog) + try: + lake, mathlib = _inspect_lake(root_descriptor, diagnostics) + lean = _inspect_toolchain(root_descriptor, diagnostics) + manifest_path, manifest_digest = _optional_digest( + root_descriptor, "lake-manifest.json", diagnostics + ) + autoform = _inspect_autoform(root_descriptor, diagnostics) + git_path = _inspect_git(root_descriptor, diagnostics) + finally: + os.close(root_descriptor) + compatibility = _compatibility(release_catalog, lean, mathlib, diagnostics) + return ProjectInspection( + schema=PROJECT_INSPECTION_SCHEMA, + project_root=".", + git_path=git_path, + lake=lake, + lake_manifest_path=manifest_path, + lake_manifest_sha256=manifest_digest, + lean=lean, + mathlib=mathlib, + autoform=autoform, + compatibility=compatibility, + diagnostics=_ordered(diagnostics), + ) + + +def _inspection( + diagnostics: list[ProjectDiagnostic], + catalog: ReleaseCatalog, +) -> ProjectInspection: + return ProjectInspection( + schema=PROJECT_INSPECTION_SCHEMA, + project_root=None, + git_path=None, + lake=None, + lake_manifest_path=None, + lake_manifest_sha256=None, + lean=None, + mathlib=None, + autoform=AutoformProject(False, None, None, None, None), + compatibility=ProjectCompatibility( + catalog=catalog.schema, + status="indeterminate", + release=None, + recommended_release=catalog.recommended.id, + ), + diagnostics=_ordered(diagnostics), + ) + + +def _discover_root(target: str | Path, diagnostics: list[ProjectDiagnostic]) -> int | None: + """Return a no-follow descriptor for the nearest enclosing project root. + + Every ancestor is opened with O_NOFOLLOW as it is traversed and the chosen + descriptor is retained, so no pathname is ever re-resolved after being + checked. Replacing a directory with a symlink mid-walk fails the open + instead of redirecting inspection to another project. + """ + if not _secure_inspection_available(diagnostics): + return None + try: + candidate = Path(target).expanduser().absolute() + except (OSError, RuntimeError, ValueError): + _issue(diagnostics, "error", "target-unreadable", "The inspection target cannot be resolved.") + return None + + descriptors: list[int] = [] + chosen: int | None = None + try: + try: + descriptors.append(_open_directory(candidate.anchor, None)) + except OSError: + _issue( + diagnostics, + "error", + "project-root-unreadable", + "The project root cannot be opened safely.", + ) + return None + parts = candidate.parts[1:] + for index, part in enumerate(parts): + last = index == len(parts) - 1 + if part == ".": + continue + if part == "..": + if len(descriptors) > 1: + os.close(descriptors.pop()) + continue + status = _entry_status(descriptors[-1], part) + if status == "missing": + _issue( + diagnostics, + "error", + "target-does-not-exist", + "The inspection target does not exist.", + ) + return None + if status == "symlink": + if last: + _issue( + diagnostics, "error", "target-is-symlink", "The inspection target is a symlink." + ) + else: + _issue( + diagnostics, + "error", + "project-path-is-symlink", + "The target path contains a symlink.", + ) + return None + if status == "directory": + try: + descriptors.append(_open_directory(part, descriptors[-1])) + except OSError: + _issue( + diagnostics, + "error", + "project-root-unreadable", + "The project root cannot be opened safely.", + ) + return None + continue + if last and status == "file": + break + if last and status == "other": + _issue( + diagnostics, + "error", + "target-not-file-or-directory", + "The inspection target is unsupported.", + ) + else: + _issue( + diagnostics, + "error", + "project-root-unreadable", + "The project root cannot be opened safely.", + ) + return None + + for descriptor in reversed(descriptors): + if any( + _relative_status(descriptor, marker) != "missing" for marker in _PROJECT_MARKERS + ): + chosen = descriptor + return chosen + _issue(diagnostics, "error", "project-not-found", "No enclosing Lean or Autoform project was found.") + return None + finally: + for descriptor in descriptors: + if descriptor != chosen: + os.close(descriptor) + + +def _secure_inspection_available(diagnostics: list[ProjectDiagnostic]) -> bool: + if ( + hasattr(os, "O_NOFOLLOW") + and hasattr(os, "O_DIRECTORY") + and os.open in os.supports_dir_fd + and os.stat in os.supports_dir_fd + ): + return True + _issue( + diagnostics, + "error", + "secure-file-inspection-unavailable", + "This platform cannot safely inspect project files without following links.", + ) + return False + + +def _open_directory(name: str, parent_descriptor: int | None) -> int: + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + if parent_descriptor is None: + return os.open(name, flags) + return os.open(name, flags, dir_fd=parent_descriptor) + + +def _entry_status(parent_descriptor: int, name: str) -> str: + try: + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + return "missing" + except OSError: + return "unreadable" + if stat.S_ISLNK(metadata.st_mode): + return "symlink" + if stat.S_ISDIR(metadata.st_mode): + return "directory" + if stat.S_ISREG(metadata.st_mode): + return "file" + return "other" + + +def _inspect_lake( + root_descriptor: int, diagnostics: list[ProjectDiagnostic] +) -> tuple[LakeProject | None, MathlibProject | None]: + toml_status = _relative_status(root_descriptor, "lakefile.toml") + lean_status = _relative_status(root_descriptor, "lakefile.lean") + if toml_status != "missing" and lean_status != "missing": + _issue( + diagnostics, + "error", + "conflicting-lake-configs", + "Both lakefile.toml and lakefile.lean exist.", + ) + return None, None + if toml_status != "missing": + content = _read_file(root_descriptor, "lakefile.toml", "lake-config", diagnostics) + if content is None: + return None, None + try: + text = content.decode("utf-8") + if _toml_nesting_exceeds(text, _MAX_STRUCTURAL_DEPTH): + raise ValueError("TOML nesting limit exceeded") + payload = tomllib.loads(text) + if _semantic_nesting_exceeds(payload, _MAX_STRUCTURAL_DEPTH): + raise ValueError("TOML nesting limit exceeded") + except (UnicodeError, ValueError, tomllib.TOMLDecodeError, RecursionError, MemoryError): + _issue(diagnostics, "error", "invalid-lake-toml", "lakefile.toml is not valid UTF-8 TOML.", "lakefile.toml") + return None, None + lake = _parse_lake_toml(payload, content, diagnostics) + return lake, _parse_mathlib(payload, diagnostics) if lake is not None else None + if lean_status != "missing": + content = _read_file(root_descriptor, "lakefile.lean", "lake-config", diagnostics) + if content is None: + return None, None + _issue( + diagnostics, + "warning", + "lakefile-lean-not-evaluated", + "lakefile.lean was detected but is not executed by offline inspection.", + "lakefile.lean", + ) + return LakeProject( + format="lean", + path="lakefile.lean", + sha256=hashlib.sha256(content).hexdigest(), + name=None, + version=None, + default_targets=(), + package_src_dir=None, + targets=(), + ), None + _issue(diagnostics, "error", "missing-lake-config", "The project has no Lake source configuration.") + return None, None + + +def _parse_lake_toml( + payload: dict[str, Any], content: bytes, diagnostics: list[ProjectDiagnostic] +) -> LakeProject | None: + try: + name = _required_string(payload.get("name"), "name") + version = _lake_version(payload.get("version")) + default_targets = _string_list(payload.get("defaultTargets", []), "defaultTargets") + package_src_dir = _portable_path(payload.get("srcDir"), "srcDir") + targets: list[LakeTarget] = [] + canonical_target_names: list[str] = [] + for kind in ("lean_lib", "lean_exe"): + entries = payload.get(kind, []) + if not isinstance(entries, list): + raise _InvalidLakeField(kind) + for entry in entries: + if not isinstance(entry, dict): + raise _InvalidLakeField(kind) + target_name = _required_string(entry.get("name"), f"{kind}.name") + canonical_name = _canonical_target_name(target_name) + if kind == "lean_lib" and "root" in entry: + raise _InvalidLakeField("lean_lib.root") + if kind == "lean_exe" and "roots" in entry: + raise _InvalidLakeField("lean_exe.roots") + if kind == "lean_lib": + roots = ( + _module_list(entry["roots"], "lean_lib.roots") + if "roots" in entry + else (canonical_name,) + ) + else: + roots = () + targets.append( + LakeTarget( + kind=kind, + name=target_name, + root=( + ( + _module(entry["root"], "lean_exe.root") + if "root" in entry + else canonical_name + ) + if kind == "lean_exe" + else None + ), + roots=roots, + src_dir=_portable_path(entry.get("srcDir"), f"{kind}.srcDir"), + ) + ) + canonical_target_names.append(canonical_name) + if len(set(canonical_target_names)) != len(canonical_target_names): + raise _InvalidLakeField("duplicate target name") + exe_roots = [target.root for target in targets if target.kind == "lean_exe" and target.root] + if len(set(exe_roots)) != len(exe_roots): + raise _InvalidLakeField("duplicate executable root") + _validate_mathlib_requirements(payload) + except _NonportableLakePath: + _issue( + diagnostics, + "error", + "nonportable-lake-path", + "lakefile.toml contains an absolute or parent-relative path.", + "lakefile.toml", + ) + return None + except _DuplicateMathlibRequirement: + _issue( + diagnostics, + "error", + "duplicate-mathlib-requirement", + "lakefile.toml contains multiple direct Mathlib requirements.", + "lakefile.toml", + ) + return None + except _InvalidLakeField: + _issue( + diagnostics, + "error", + "invalid-lake-field", + "lakefile.toml contains an invalid field used by Autoform.", + "lakefile.toml", + ) + return None + return LakeProject( + format="toml", + path="lakefile.toml", + sha256=hashlib.sha256(content).hexdigest(), + name=name, + version=version, + default_targets=default_targets, + package_src_dir=package_src_dir, + targets=tuple(targets), + ) + + +def _inspect_toolchain(root_descriptor: int, diagnostics: list[ProjectDiagnostic]) -> LeanProject | None: + if _relative_status(root_descriptor, "lean-toolchain") == "missing": + _issue(diagnostics, "error", "missing-lean-toolchain", "The project has no lean-toolchain file.") + return None + content = _read_file(root_descriptor, "lean-toolchain", "lean-toolchain", diagnostics) + if content is None: + return None + try: + text = content.decode("utf-8").strip() + except UnicodeError: + text = "" + if not text or "\n" in text or "\r" in text: + _issue(diagnostics, "error", "invalid-lean-toolchain", "lean-toolchain must contain one UTF-8 value.", "lean-toolchain") + return None + match = _TOOLCHAIN.fullmatch(text) + if match is None: + _issue( + diagnostics, + "warning", + "unrecognized-lean-toolchain", + "The Lean toolchain is outside Autoform's recognized stable form.", + "lean-toolchain", + ) + return LeanProject( + path="lean-toolchain", + sha256=hashlib.sha256(content).hexdigest(), + toolchain=text, + version=match.group("version") if match is not None else None, + ) + + +def _open_parent_descriptor(root_descriptor: int, relative: str) -> tuple[int, str]: + parts = PurePosixPath(relative).parts + if not parts or any(part in {"", ".", ".."} for part in parts): + raise OSError(errno.EINVAL, "invalid relative path") + current = os.dup(root_descriptor) + try: + for part in parts[:-1]: + next_descriptor = _open_directory(part, current) + os.close(current) + current = next_descriptor + return current, parts[-1] + except BaseException: + os.close(current) + raise + + +def _relative_status(root_descriptor: int, relative: str) -> str: + if relative == ".": + try: + metadata = os.fstat(root_descriptor) + except OSError: + return "unsafe" + return "directory" if stat.S_ISDIR(metadata.st_mode) else "unsafe" + try: + parent, name = _open_parent_descriptor(root_descriptor, relative) + except FileNotFoundError: + return "missing" + except OSError: + return "unsafe" + try: + metadata = os.stat(name, dir_fd=parent, follow_symlinks=False) + except FileNotFoundError: + return "missing" + except OSError: + return "unsafe" + finally: + os.close(parent) + if stat.S_ISLNK(metadata.st_mode): + return "unsafe" + if stat.S_ISDIR(metadata.st_mode): + return "directory" + if stat.S_ISREG(metadata.st_mode): + return "file" + return "unsafe" + + +def _inspect_git(root_descriptor: int, diagnostics: list[ProjectDiagnostic]) -> str | None: + status = _relative_status(root_descriptor, ".git") + if status == "unsafe": + _issue( + diagnostics, + "error", + "git-path-is-symlink", + "The project's .git metadata path cannot be inspected safely.", + ".git", + ) + return None + return ".git" if status in {"file", "directory"} else None + + +def _inspect_autoform(root_descriptor: int, diagnostics: list[ProjectDiagnostic]) -> AutoformProject: + paths = { + "mkdocs_path": ("mkdocs.yml", "file"), + "verification_workflow_path": (".github/workflows/autoform-verify.yml", "file"), + "pages_workflow_path": (".github/workflows/blueprint-pages.yml", "file"), + } + values: dict[str, str | None] = {"blueprint_path": None} + for field, (relative, expected) in paths.items(): + status = _relative_status(root_descriptor, relative) + if status == "unsafe": + _issue( + diagnostics, + "error", + "scaffold-path-is-symlink", + "An Autoform scaffold path cannot be inspected safely.", + relative, + ) + values[field] = None + elif status == "missing": + values[field] = None + elif status != expected: + _issue( + diagnostics, + "error", + "scaffold-path-unexpected-type", + "An Autoform scaffold path is not the expected file or directory.", + relative, + ) + values[field] = None + else: + values[field] = relative + manifest_path: str | None = None + manifest_sha256: str | None = None + blueprint_paths: tuple[str, ...] = () + manifest_status = _relative_status(root_descriptor, WORKSPACE_FILE) + if manifest_status == "unsafe": + _issue( + diagnostics, + "error", + "autoform-manifest-is-symlink", + "The Autoform workspace manifest cannot be inspected safely.", + WORKSPACE_FILE, + ) + elif manifest_status not in {"missing", "file"}: + _issue( + diagnostics, + "error", + "autoform-manifest-unexpected-type", + "The Autoform workspace manifest is not a regular file.", + WORKSPACE_FILE, + ) + elif manifest_status == "file": + # A root workspace manifest owns blueprint selection. On a + # case-insensitive filesystem, probing the legacy lowercase path can + # otherwise alias a registered location such as ``Blueprint``. + values["blueprint_path"] = None + content = _read_file(root_descriptor, WORKSPACE_FILE, "autoform-manifest", diagnostics) + if content is not None: + manifest_sha256 = hashlib.sha256(content).hexdigest() + try: + manifest = parse_workspace(content.decode("utf-8")) + except (UnicodeDecodeError, WorkspaceError) as error: + detail = ( + "; ".join(error.issues) + if isinstance(error, WorkspaceError) + else f"{WORKSPACE_FILE} is not valid UTF-8 TOML" + ) + _issue( + diagnostics, + "error", + "autoform-manifest-invalid", + detail, + WORKSPACE_FILE, + ) + else: + manifest_path = WORKSPACE_FILE + locations = {location.id: location for location in manifest.locations} + for location in manifest.locations: + status = _relative_status(root_descriptor, location.path) + if status == "unsafe": + _issue( + diagnostics, + "error", + "autoform-location-is-symlink", + "A declared Autoform location cannot be inspected safely.", + location.path, + ) + elif status == "missing": + _issue( + diagnostics, + "warning", + "autoform-location-missing", + "A declared Autoform location does not exist.", + location.path, + ) + elif status != "directory": + _issue( + diagnostics, + "error", + "autoform-location-unexpected-type", + "A declared Autoform location is not a directory.", + location.path, + ) + resolved: list[str] = [] + for project in manifest.projects: + location = locations[project.blueprint_location] + relative = PurePosixPath(location.path, project.blueprint_path).as_posix() + resolved.append(relative) + status = _relative_status(root_descriptor, relative) + if status == "unsafe": + _issue( + diagnostics, + "error", + "autoform-blueprint-is-symlink", + "A registered blueprint cannot be inspected safely.", + relative, + ) + elif status == "missing": + _issue( + diagnostics, + "error", + "autoform-blueprint-missing", + "A registered blueprint directory is missing.", + relative, + ) + elif status != "directory": + _issue( + diagnostics, + "error", + "autoform-blueprint-unexpected-type", + "A registered blueprint path is not a directory.", + relative, + ) + blueprint_paths = tuple(sorted(resolved)) + + if manifest_status == "missing": + blueprint_status = _relative_status(root_descriptor, "blueprint") + if blueprint_status == "unsafe": + _issue( + diagnostics, + "error", + "scaffold-path-is-symlink", + "An Autoform scaffold path cannot be inspected safely.", + "blueprint", + ) + elif blueprint_status == "directory": + values["blueprint_path"] = "blueprint" + elif blueprint_status != "missing": + _issue( + diagnostics, + "error", + "scaffold-path-unexpected-type", + "An Autoform scaffold path is not the expected file or directory.", + "blueprint", + ) + + workflow_count = sum( + values[field] is not None + for field in ("verification_workflow_path", "pages_workflow_path") + ) + if values["blueprint_path"] is not None and values["mkdocs_path"] is None: + _issue(diagnostics, "warning", "autoform-mkdocs-missing", "The blueprint has no mkdocs.yml.") + if workflow_count == 1: + _issue(diagnostics, "warning", "autoform-workflows-partial", "Only one standard Autoform workflow exists.") + return AutoformProject( + detected=values["blueprint_path"] is not None or manifest_path is not None, + blueprint_path=values["blueprint_path"], + blueprint_paths=blueprint_paths, + manifest_path=manifest_path, + manifest_sha256=manifest_sha256, + mkdocs_path=values["mkdocs_path"], + verification_workflow_path=values["verification_workflow_path"], + pages_workflow_path=values["pages_workflow_path"], + ) + + +def _compatibility( + catalog: ReleaseCatalog, + lean: LeanProject | None, + mathlib: MathlibProject | None, + diagnostics: list[ProjectDiagnostic], +) -> ProjectCompatibility: + matched = None + if lean is not None and mathlib is not None: + matched = next( + ( + release + for release in catalog.releases + if release.lean.toolchain == lean.toolchain + and release.mathlib.git == mathlib.git + and release.mathlib.revision == mathlib.revision + ), + None, + ) + if matched is not None: + status = "supported" + release_id = matched.id + elif lean is not None and mathlib is not None: + status = "unlisted" + release_id = None + _issue( + diagnostics, + "warning", + "release-unlisted", + "The configured Lean and Mathlib revisions are not in the bundled release catalog.", + ) + else: + status = "indeterminate" + release_id = None + _issue( + diagnostics, + "warning", + "release-indeterminate", + "Offline inspection cannot determine a Lean and Mathlib release pair.", + ) + return ProjectCompatibility(catalog.schema, status, release_id, catalog.recommended.id) + + +def _optional_digest( + root_descriptor: int, relative: str, diagnostics: list[ProjectDiagnostic] +) -> tuple[str | None, str | None]: + if _relative_status(root_descriptor, relative) == "missing": + return None, None + content = _read_file( + root_descriptor, relative, "lake-manifest", diagnostics, severity="warning" + ) + if content is None: + return None, None + try: + text = content.decode("utf-8") + if _json_nesting_exceeds(text, _MAX_STRUCTURAL_DEPTH): + raise ValueError("JSON nesting limit exceeded") + json.loads(text) + except (UnicodeError, ValueError, RecursionError, MemoryError): + _issue(diagnostics, "warning", "invalid-lake-manifest", "lake-manifest.json is not valid UTF-8 JSON.", relative) + return relative, hashlib.sha256(content).hexdigest() + return relative, hashlib.sha256(content).hexdigest() + + +def _read_file( + root_descriptor: int, + relative: str, + kind: str, + diagnostics: list[ProjectDiagnostic], + *, + severity: str = "error", +) -> bytes | None: + try: + parent, name = _open_parent_descriptor(root_descriptor, relative) + except OSError: + _issue( + diagnostics, + severity, + f"{kind}-is-symlink", + "A decision-bearing project path cannot be traversed safely.", + relative, + ) + return None + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, "O_CLOEXEC", 0) + try: + descriptor = os.open(name, flags, dir_fd=parent) + except OSError as error: + code = ( + f"{kind}-is-symlink" + if error.errno in {errno.ELOOP, errno.ENOTDIR} + else f"{kind}-unreadable" + ) + message = ( + "A decision-bearing project file cannot be opened without following links." + if code.endswith("-is-symlink") + else "A project configuration file cannot be read." + ) + _issue(diagnostics, severity, code, message, relative) + os.close(parent) + return None + os.close(parent) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + _issue( + diagnostics, + severity, + f"{kind}-not-regular", + "A decision-bearing project path is not a regular file.", + relative, + ) + return None + with os.fdopen(descriptor, "rb", closefd=False) as stream: + content = stream.read(_MAX_CONFIG_BYTES + 1) + if len(content) > _MAX_CONFIG_BYTES: + _issue( + diagnostics, + severity, + f"{kind}-too-large", + "A project configuration file exceeds the inspection limit.", + relative, + ) + return None + return content + except OSError: + _issue( + diagnostics, + severity, + f"{kind}-unreadable", + "A project configuration file cannot be read.", + relative, + ) + return None + finally: + os.close(descriptor) + + +def _validate_mathlib_requirements(payload: dict[str, Any]) -> None: + requirements = payload.get("require", []) + if not isinstance(requirements, list): + raise _InvalidLakeField("require") + canonical_names: list[str] = [] + for entry in requirements: + if not isinstance(entry, dict): + raise _InvalidLakeField("require") + canonical_names.append( + _canonical_target_name(_required_string(entry.get("name"), "require.name")) + ) + matches = [ + entry + for entry, canonical_name in zip(requirements, canonical_names, strict=True) + if canonical_name == "mathlib" + ] + if len(matches) > 1: + raise _DuplicateMathlibRequirement("duplicate mathlib") + for entry in matches: + if "scope" in entry: + _required_string(entry["scope"], "mathlib.scope") + if "rev" in entry: + _required_string(entry["rev"], "mathlib.rev") + if "path" in entry: + _portable_path(entry["path"], "mathlib.path") + elif "git" in entry: + _git_url(entry["git"], "mathlib.git") + if "subDir" in entry: + _portable_path(entry["subDir"], "mathlib.subDir") + elif "source" in entry: + _validate_dependency_source(entry["source"]) + + +def _parse_mathlib( + payload: dict[str, Any], diagnostics: list[ProjectDiagnostic] +) -> MathlibProject | None: + requirements = payload.get("require", []) + if not isinstance(requirements, list): + return None + matches = [ + entry + for entry in requirements + if isinstance(entry, dict) + and isinstance(entry.get("name"), str) + and _canonical_target_name(entry["name"]) == "mathlib" + ] + if len(matches) != 1: + return None + entry = matches[0] + if ( + "path" in entry + or "source" in entry + or entry.get("subDir") not in {None, "", "."} + ): + return None + revision = entry.get("rev") + if not isinstance(revision, str): + return None + git = _mathlib_git_source(entry) + if git is None: + return None + try: + parsed = urlsplit(git) + port = parsed.port + except ValueError: + _issue( + diagnostics, + "error", + "invalid-mathlib-url", + "The direct Mathlib Git URL is invalid.", + "lakefile.toml", + ) + return None + if parsed.username is not None or parsed.password is not None: + _issue( + diagnostics, + "error", + "credentialed-mathlib-url", + "The direct Mathlib Git URL must not contain credentials.", + "lakefile.toml", + ) + return None + if ( + parsed.scheme != "https" + or not parsed.hostname + or port is not None + or parsed.query + or parsed.fragment + or parsed.netloc.lower() != parsed.hostname.lower() + ): + _issue( + diagnostics, + "error", + "invalid-mathlib-url", + "The direct Mathlib Git URL must be credential-free HTTPS.", + "lakefile.toml", + ) + return None + return MathlibProject(git=git, revision=revision, source="lakefile.toml") + + +def _validate_dependency_source(value: Any) -> None: + if not isinstance(value, dict): + raise _InvalidLakeField("mathlib.source") + source_type = _required_string(value.get("type"), "mathlib.source.type") + if source_type == "path": + if set(value) != {"type", "dir"}: + raise _InvalidLakeField("mathlib.source") + _portable_path(value["dir"], "mathlib.source.dir") + elif source_type == "git": + if not set(value) <= {"type", "url", "rev", "subDir"} or "url" not in value: + raise _InvalidLakeField("mathlib.source") + _required_string(value["url"], "mathlib.source.url") + if "rev" in value: + _required_string(value["rev"], "mathlib.source.rev") + if "subDir" in value: + _portable_path(value["subDir"], "mathlib.source.subDir") + else: + raise _InvalidLakeField("mathlib.source.type") + + +def _mathlib_git_source(entry: dict[str, Any]) -> str | None: + git = entry.get("git") + if isinstance(git, dict): + git = git.get("url") + if isinstance(git, str): + return git + # `lake new math` emits a scope-only Reservoir requirement with no + # `git` field; Reservoir serves that scope from GitHub. + scope = entry.get("scope") + if isinstance(scope, str) and _RESERVOIR_SCOPE.fullmatch(scope): + return f"https://github.com/{scope}/mathlib4.git" + return None + + +def _git_url(value: Any, field: str) -> str: + if isinstance(value, dict): + if set(value) != {"url"}: + raise _InvalidLakeField(field) + value = value["url"] + return _required_string(value, field) + + +def _required_string(value: Any, field: str) -> str: + if ( + not isinstance(value, str) + or not value + or value != value.strip() + or any(ord(character) < 32 or ord(character) == 127 for character in value) + ): + raise _InvalidLakeField(field) + return value + + +def _lake_version(value: Any) -> str | None: + if value is None: + return None + text = _required_string(value, "version") + if _LAKE_VERSION.fullmatch(text) is None: + raise _InvalidLakeField("version") + return text + + +def _string_list(value: Any, field: str) -> tuple[str, ...]: + if not isinstance(value, list): + raise _InvalidLakeField(field) + return tuple(_required_string(item, field) for item in value) + + +def _module_list(value: Any, field: str) -> tuple[str, ...]: + if not isinstance(value, list): + raise _InvalidLakeField(field) + return tuple(_module(item, field) for item in value) + + +def _module(value: Any, field: str) -> str: + canonical = _canonical_module_name(_required_string(value, field)) + if canonical is None: + raise _InvalidLakeField(field) + return canonical + + +def _canonical_module_name(value: str) -> str | None: + """Render a Lake name the way Lean's `String.toName` reads it. + + Numeric components are decoded as naturals, so `01` and `1` name the same + module, and letter-like Unicode components such as `Ω` are accepted. + Returns None when Lean would reject the string outright. + """ + components = _split_lean_name(value) + if components is None: + return None + root_kind, root_text = components[0] + escape = not ( + root_kind == "str" + and (root_text.startswith("#") or root_text.startswith("?")) + ) + return ".".join( + _render_lean_component(kind, text, escape=escape) + for kind, text in components + ) + + +def _canonical_target_name(value: str) -> str: + """Apply Lake's `stringToLegalOrSimpleName` fallback for target names.""" + canonical = _canonical_module_name(value) + if canonical is not None: + return canonical + escape = not (value.startswith("#") or value.startswith("?")) + return _render_lean_component("str", value, escape=escape) + + +def _split_lean_name(value: str) -> list[tuple[str, str]] | None: + components: list[tuple[str, str]] = [] + index = 0 + while True: + if index >= len(value): + return None + character = value[index] + if character == _LEAN_ID_BEGIN_ESCAPE: + end = value.find(_LEAN_ID_END_ESCAPE, index + 1) + if end < 0: + return None + components.append(("str", value[index + 1 : end])) + index = end + 1 + elif _lean_is_id_first(character): + start = index + index += 1 + while index < len(value) and _lean_is_id_rest(value[index]): + index += 1 + components.append(("str", value[start:index])) + elif _lean_is_digit(character): + start = index + while index < len(value) and _lean_is_digit(value[index]): + index += 1 + digits = value[start:index] + components.append(("num", digits.lstrip("0") or "0")) + else: + return None + if index == len(value): + return components + if value[index] != ".": + return None + index += 1 + + +def _render_lean_component(kind: str, text: str, *, escape: bool = True) -> str: + if kind == "num": + return text + if not escape: + return text + # Lean's `Name.escapePart` cannot round-trip a closing guillemet, so it + # leaves the complete simple component unescaped in that case. + if _LEAN_ID_END_ESCAPE in text: + return text + if text and _lean_is_id_first(text[0]) and all(_lean_is_id_rest(c) for c in text[1:]): + return text + return f"{_LEAN_ID_BEGIN_ESCAPE}{text}{_LEAN_ID_END_ESCAPE}" + + +def _lean_is_digit(character: str) -> bool: + return "0" <= character <= "9" + + +def _lean_is_alpha(character: str) -> bool: + return "a" <= character <= "z" or "A" <= character <= "Z" + + +def _lean_is_id_first(character: str) -> bool: + return _lean_is_alpha(character) or character == "_" or _lean_is_letter_like(character) + + +def _lean_is_id_rest(character: str) -> bool: + return ( + _lean_is_alpha(character) + or _lean_is_digit(character) + or character in "_'!?" + or _lean_is_letter_like(character) + or _lean_is_subscript_alnum(character) + ) + + +def _lean_is_letter_like(character: str) -> bool: + code = ord(character) + return ( + (0x3B1 <= code <= 0x3C9 and code != 0x3BB) + or (0x391 <= code <= 0x3A9 and code not in {0x3A0, 0x3A3}) + or 0x3CA <= code <= 0x3FB + or 0x1F00 <= code <= 0x1FFE + or 0x2100 <= code <= 0x214F + or 0x1D49C <= code <= 0x1D59F + or (0xC0 <= code <= 0xFF and code not in {0xD7, 0xF7}) + or 0x100 <= code <= 0x17F + ) + + +def _lean_is_subscript_alnum(character: str) -> bool: + code = ord(character) + return ( + 0x2080 <= code <= 0x2089 + or 0x2090 <= code <= 0x209C + or 0x1D62 <= code <= 0x1D6A + or code == 0x2C7C + ) + + +def _portable_path(value: Any, field: str) -> str | None: + if value is None: + return None + text = _required_string(value, field) + posix = PurePosixPath(text) + windows = PureWindowsPath(text) + if ( + posix.is_absolute() + or windows.is_absolute() + or windows.drive + or windows.root + or ".." in posix.parts + or "." in posix.parts + or ".." in windows.parts + or "." in windows.parts + ): + raise _NonportableLakePath(field) + return posix.as_posix() + + +def _toml_nesting_exceeds(text: str, limit: int) -> bool: + """Bound both bracket nesting and dotted-key nesting before parsing. + + A table header or dotted key names one table per component, so a flat + document such as `[a.b.c...]` nests as deeply as `[[[...]]]` would while + using only one bracket pair. + """ + depth = 0 + key_components = 0 + in_key = True + quote: str | None = None + escaped = False + index = 0 + while index < len(text): + character = text[index] + if quote is not None: + if escaped: + escaped = False + elif quote[0] == '"' and character == "\\": + escaped = True + elif len(quote) == 3 and character == quote[0]: + run_end = index + while run_end < len(text) and text[run_end] == quote[0]: + run_end += 1 + if run_end - index >= 3: + index = run_end - 1 + quote = None + elif len(quote) == 1 and character == quote: + quote = None + elif character == "#": + newline = text.find("\n", index) + # Stop before the newline so it still resets the key context. + index = len(text) if newline < 0 else newline + continue + elif text.startswith("'''", index) or text.startswith('\"\"\"', index): + quote = text[index : index + 3] + index += 2 + elif character in "'\"": + quote = character + elif character in "[{": + depth += 1 + if depth > limit: + return True + if character == "{": + in_key, key_components = True, 0 + elif character in "]}": + depth = max(0, depth - 1) + elif character in "\n,": + in_key, key_components = True, 0 + elif character == "=": + in_key = False + elif character == "." and in_key: + key_components += 1 + if depth + key_components > limit: + return True + index += 1 + return False + + +def _semantic_nesting_exceeds(value: Any, limit: int) -> bool: + """Check parsed container depth iteratively as a second depth boundary.""" + + stack: list[tuple[Any, int]] = [(value, 1)] + while stack: + current, depth = stack.pop() + if isinstance(current, dict): + if depth > limit: + return True + stack.extend((child, depth + 1) for child in current.values()) + elif isinstance(current, list): + if depth > limit: + return True + stack.extend((child, depth + 1) for child in current) + return False + + +def _json_nesting_exceeds(text: str, limit: int) -> bool: + depth = 0 + in_string = False + escaped = False + for character in text: + if in_string: + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + in_string = False + continue + if character == '"': + in_string = True + elif character in "[{": + depth += 1 + if depth > limit: + return True + elif character in "]}": + depth = max(0, depth - 1) + return False + + +def _issue( + diagnostics: list[ProjectDiagnostic], + severity: str, + code: str, + message: str, + path: str | None = None, +) -> None: + diagnostics.append(ProjectDiagnostic(severity, code, message, path)) + + +def _ordered(diagnostics: list[ProjectDiagnostic]) -> tuple[ProjectDiagnostic, ...]: + unique = set(diagnostics) + return tuple( + sorted( + unique, + key=lambda diagnostic: ( + _SEVERITY_ORDER[diagnostic.severity], + diagnostic.code, + diagnostic.path or "", + diagnostic.message, + ), + ) + ) diff --git a/autoform_cli/project/model.py b/autoform_cli/project/model.py new file mode 100644 index 00000000..ed2db6bc --- /dev/null +++ b/autoform_cli/project/model.py @@ -0,0 +1,211 @@ +"""Immutable schemas for offline Autoform project inspection.""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass + +PROJECT_INSPECTION_SCHEMA = "autoform-project-inspection/v2" +RELEASE_CATALOG_SCHEMA = "autoform-project-release-catalog/v1" + + +@dataclass(frozen=True, order=True, slots=True) +class ProjectDiagnostic: + severity: str + code: str + message: str + path: str | None = None + + def as_dict(self) -> dict[str, str | None]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class LeanRelease: + toolchain: str + version: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class MathlibRelease: + git: str + revision: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class SupportedRelease: + id: str + channel: str + recommended: bool + lean: LeanRelease + mathlib: MathlibRelease + + def as_dict(self) -> dict[str, object]: + return { + "channel": self.channel, + "id": self.id, + "lean": self.lean.as_dict(), + "mathlib": self.mathlib.as_dict(), + "recommended": self.recommended, + } + + +@dataclass(frozen=True, slots=True) +class ReleaseCatalog: + schema: str + releases: tuple[SupportedRelease, ...] + + def as_dict(self) -> dict[str, object]: + return { + "releases": [release.as_dict() for release in self.releases], + "schema": self.schema, + } + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + @property + def recommended(self) -> SupportedRelease: + return next(release for release in self.releases if release.recommended) + + +@dataclass(frozen=True, slots=True) +class LakeTarget: + kind: str + name: str + root: str | None + roots: tuple[str, ...] | None + src_dir: str | None + + def as_dict(self) -> dict[str, object]: + return { + "kind": self.kind, + "name": self.name, + "root": self.root, + "roots": list(self.roots) if self.roots is not None else None, + "src_dir": self.src_dir, + } + + +@dataclass(frozen=True, slots=True) +class LakeProject: + format: str + path: str + sha256: str + name: str | None + version: str | None + default_targets: tuple[str, ...] + package_src_dir: str | None + targets: tuple[LakeTarget, ...] + + def as_dict(self) -> dict[str, object]: + return { + "default_targets": list(self.default_targets), + "format": self.format, + "name": self.name, + "package_src_dir": self.package_src_dir, + "path": self.path, + "sha256": self.sha256, + "targets": [target.as_dict() for target in self.targets], + "version": self.version, + } + + +@dataclass(frozen=True, slots=True) +class LeanProject: + path: str + sha256: str + toolchain: str + version: str | None + + def as_dict(self) -> dict[str, str | None]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class MathlibProject: + git: str | None + revision: str | None + source: str + + def as_dict(self) -> dict[str, str | None]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class AutoformProject: + detected: bool + blueprint_path: str | None + mkdocs_path: str | None + verification_workflow_path: str | None + pages_workflow_path: str | None + blueprint_paths: tuple[str, ...] = () + manifest_path: str | None = None + manifest_sha256: str | None = None + + def as_dict(self) -> dict[str, object]: + return { + "blueprint_path": self.blueprint_path, + "blueprint_paths": list(self.blueprint_paths), + "detected": self.detected, + "manifest_path": self.manifest_path, + "manifest_sha256": self.manifest_sha256, + "mkdocs_path": self.mkdocs_path, + "pages_workflow_path": self.pages_workflow_path, + "verification_workflow_path": self.verification_workflow_path, + } + + +@dataclass(frozen=True, slots=True) +class ProjectCompatibility: + catalog: str + status: str + release: str | None + recommended_release: str + + def as_dict(self) -> dict[str, str | None]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class ProjectInspection: + schema: str + project_root: str | None + git_path: str | None + lake: LakeProject | None + lake_manifest_path: str | None + lake_manifest_sha256: str | None + lean: LeanProject | None + mathlib: MathlibProject | None + autoform: AutoformProject + compatibility: ProjectCompatibility + diagnostics: tuple[ProjectDiagnostic, ...] + + @property + def ok(self) -> bool: + return not any(diagnostic.severity == "error" for diagnostic in self.diagnostics) + + def as_dict(self) -> dict[str, object]: + return { + "autoform": self.autoform.as_dict(), + "compatibility": self.compatibility.as_dict(), + "diagnostics": [diagnostic.as_dict() for diagnostic in self.diagnostics], + "git_path": self.git_path, + "lake": self.lake.as_dict() if self.lake is not None else None, + "lake_manifest_path": self.lake_manifest_path, + "lake_manifest_sha256": self.lake_manifest_sha256, + "lean": self.lean.as_dict() if self.lean is not None else None, + "mathlib": self.mathlib.as_dict() if self.mathlib is not None else None, + "ok": self.ok, + "project_root": self.project_root, + "schema": self.schema, + } + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) diff --git a/autoform_cli/project/releases.json b/autoform_cli/project/releases.json new file mode 100644 index 00000000..41ec5f73 --- /dev/null +++ b/autoform_cli/project/releases.json @@ -0,0 +1,44 @@ +{ + "schema": "autoform-project-release-catalog/v1", + "releases": [ + { + "id": "lean-v4.32.2-mathlib-v4.32.2", + "channel": "stable", + "recommended": false, + "lean": { + "toolchain": "leanprover/lean4:v4.32.2", + "version": "v4.32.2" + }, + "mathlib": { + "git": "https://github.com/leanprover-community/mathlib4.git", + "revision": "v4.32.2" + } + }, + { + "id": "lean-v4.33.0-mathlib-v4.33.0", + "channel": "stable", + "recommended": false, + "lean": { + "toolchain": "leanprover/lean4:v4.33.0", + "version": "v4.33.0" + }, + "mathlib": { + "git": "https://github.com/leanprover-community/mathlib4.git", + "revision": "v4.33.0" + } + }, + { + "id": "lean-v4.33.1-mathlib-v4.33.1", + "channel": "stable", + "recommended": true, + "lean": { + "toolchain": "leanprover/lean4:v4.33.1", + "version": "v4.33.1" + }, + "mathlib": { + "git": "https://github.com/leanprover-community/mathlib4.git", + "revision": "v4.33.1" + } + } + ] +} diff --git a/autoform_cli/project/repair.py b/autoform_cli/project/repair.py new file mode 100644 index 00000000..88b29b39 --- /dev/null +++ b/autoform_cli/project/repair.py @@ -0,0 +1,1275 @@ +"""Conservatively add unambiguous missing files to an existing project.""" + +from __future__ import annotations + +import errno +import hashlib +import json +import os +import re +import secrets +import stat +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +from ..scaffold import ScaffoldError, scaffold_project +from .create import ProjectCreateError, _open_parent, _rename_noreplace +from .inplace import _filesystem_supported, _noreplace_function +from .inspect import inspect_project + +try: + import fcntl +except ImportError: # pragma: no cover - Windows import compatibility + fcntl = None # type: ignore[assignment] + +PROJECT_REPAIR_SCHEMA = "autoform-project-repair/v1" +_RENDER_SOURCE = "https://github.com/facebookresearch/autoform-bot.git" +_RENDER_REF = "0" * 40 +_REQUIRED_INPUTS = { + "README.md": ("title",), + "blueprint/README.md": ("title",), + "blueprint/roadmap/README.md": ("title",), + "mkdocs.yml": ("title", "repository-url"), + ".github/workflows/autoform-verify.yml": ("autoform-source", "autoform-ref"), + ".github/workflows/blueprint-pages.yml": ("autoform-source", "autoform-ref"), +} +_WORKFLOW_PATHS = ( + ".github/workflows/autoform-verify.yml", + ".github/workflows/blueprint-pages.yml", +) + + +@dataclass(frozen=True, order=True, slots=True) +class ProjectRepairConflict: + code: str + message: str + path: str | None = None + + def as_dict(self) -> dict[str, str | None]: + return {"code": self.code, "message": self.message, "path": self.path} + + +class ProjectRepairError(ValueError): + """Repair would require changing or guessing existing project content.""" + + def __init__( + self, + conflicts: tuple[ProjectRepairConflict, ...], + *, + code: str = "project-repair-conflict", + written: tuple[str, ...] = (), + ) -> None: + self.code = code + self.conflicts = conflicts + self.written = written + self.message = "The project cannot be repaired without changing or guessing existing content." + super().__init__(self.message) + + def as_dict(self) -> dict[str, object]: + return { + "error": { + "code": self.code, + "conflicts": [conflict.as_dict() for conflict in self.conflicts], + "message": self.message, + }, + "ok": False, + "schema": PROJECT_REPAIR_SCHEMA, + "written": list(self.written), + } + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True, slots=True) +class ProjectRepairResult: + dry_run: bool + package: str + release: str + planned: tuple[str, ...] + written: tuple[str, ...] + converged: tuple[str, ...] + preserved: tuple[str, ...] + + def as_dict(self) -> dict[str, object]: + return { + "converged": list(self.converged), + "dry_run": self.dry_run, + "ok": True, + "package": self.package, + "planned": list(self.planned), + "preserved": list(self.preserved), + "release": self.release, + "schema": PROJECT_REPAIR_SCHEMA, + "target": ".", + "written": list(self.written), + } + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True, slots=True) +class _PlannedFile: + path: str + content: bytes + mode: int + required_inputs: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class _ParentIdentity: + name: str + path: str + identity: tuple[int, int] + + +def repair_project( + target: str | Path, + *, + dry_run: bool = False, + title: str | None = None, + repository_url: str | None = None, + autoform_source: str | None = None, + autoform_ref: str | None = None, +) -> ProjectRepairResult: + """Add only absent, canonically generated Autoform overlay files.""" + + root = _project_root(target) + root_descriptor = _open_root(root) + written: list[str] = [] + converged: list[str] = [] + try: + try: + if fcntl is None: + raise OSError(errno.ENOSYS, "advisory locks are unavailable") + fcntl.flock(root_descriptor, fcntl.LOCK_EX) + except (AttributeError, OSError): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-safety-unavailable", + "The project root cannot be locked for conservative repair.", + ".", + ), + ), + code="project-repair-safety-unavailable", + ) from None + if not dry_run and ( + not _filesystem_supported(root_descriptor) + or _noreplace_function() is None + ): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-safety-unavailable", + "The project filesystem cannot publish repair files atomically.", + ".", + ), + ), + code="project-repair-safety-unavailable", + ) + root_identity = _descriptor_identity(root_descriptor) + _require_root_identity(root_descriptor, root_identity, root) + _require_private_directory(root_descriptor, ".") + inspection = inspect_project(root) + _require_root_identity(root_descriptor, root_identity, root) + conflicts = _inspection_conflicts(inspection) + if conflicts: + raise ProjectRepairError(tuple(sorted(conflicts))) + assert inspection.lake is not None + assert inspection.lake.name is not None + assert inspection.compatibility.release is not None + _require_config_identity(root_descriptor, inspection) + + try: + desired = _render_overlay( + title=title, + repository_url=repository_url, + autoform_source=autoform_source, + autoform_ref=autoform_ref, + ) + except ProjectRepairError: + raise + except (OSError, ValueError): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-render-failed", + "The canonical repair overlay could not be rendered.", + ), + ), + code="project-repair-failed", + ) from None + _require_root_identity(root_descriptor, root_identity, root) + _require_config_identity(root_descriptor, inspection) + provided_inputs = frozenset( + name + for name, value in ( + ("title", title), + ("repository-url", repository_url), + ("autoform-source", autoform_source), + ("autoform-ref", autoform_ref), + ) + if value is not None + ) + recovery_conflicts = _find_recovery_conflicts(root_descriptor, desired) + if recovery_conflicts: + raise ProjectRepairError(tuple(sorted(recovery_conflicts))) + desired = _scope_workflow_files( + root_descriptor, desired, provided_inputs + ) + planned, preserved, path_conflicts = _plan( + root_descriptor, desired, provided_inputs + ) + if path_conflicts: + raise ProjectRepairError(tuple(sorted(path_conflicts))) + planned_paths = tuple(item.path for item in planned) + for item in planned: + _validate_parent_chain(root_descriptor, item.path) + if dry_run or not planned: + return ProjectRepairResult( + dry_run=dry_run, + package=inspection.lake.name, + release=inspection.compatibility.release, + planned=planned_paths, + written=(), + converged=(), + preserved=preserved, + ) + for item in planned: + try: + _require_root_identity(root_descriptor, root_identity, root) + _require_config_identity(root_descriptor, inspection) + outcome = _publish( + root, + root_descriptor, + root_identity, + item, + inspection, + ) + except OSError: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-write-failed", + "A managed path could not be traversed or published safely.", + item.path, + ), + ), + code="project-repair-failed", + written=tuple(written), + ) from None + except ProjectRepairError as error: + published = tuple((*written, *error.written)) + raise ProjectRepairError( + error.conflicts, + code=error.code, + written=published, + ) from None + (written if outcome == "written" else converged).append(item.path) + return ProjectRepairResult( + dry_run=False, + package=inspection.lake.name, + release=inspection.compatibility.release, + planned=planned_paths, + written=tuple(written), + converged=tuple(converged), + preserved=preserved, + ) + except OSError: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-io-failed", + "A project path could not be inspected or repaired safely.", + ".", + ), + ), + code="project-repair-failed", + written=tuple(written), + ) from None + finally: + pending_error = sys.exc_info()[1] + try: + os.close(root_descriptor) + except OSError: + conflict = ProjectRepairConflict( + "project-repair-close-failed", + "The project root descriptor could not be closed.", + ".", + ) + if isinstance(pending_error, ProjectRepairError): + raise ProjectRepairError( + (*pending_error.conflicts, conflict), + code=pending_error.code, + written=pending_error.written, + ) from None + raise ProjectRepairError( + (conflict,), + code="project-repair-failed", + written=tuple(written), + ) from None + + +def _path_identity(path: Path) -> tuple[int, int]: + metadata = path.stat(follow_symlinks=False) + return metadata.st_dev, metadata.st_ino + + +def _descriptor_identity(descriptor: int) -> tuple[int, int]: + metadata = os.fstat(descriptor) + return metadata.st_dev, metadata.st_ino + + +def _open_root(root: Path) -> int: + try: + return _open_parent(root) + except ProjectCreateError as error: + if error.code == "project-create-safety-unavailable": + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-safety-unavailable", + "The platform cannot traverse the project with the required path safety.", + ".", + ), + ), + code="project-repair-safety-unavailable", + ) from None + if error.code == "project-path-is-symlink": + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-target-invalid", + "The repair target path must not contain a symbolic link.", + ".", + ), + ) + ) from None + raise _race_conflict(".", "The project root changed during repair.") from None + + +def _require_root_identity( + descriptor: int, expected: tuple[int, int], path: Path +) -> None: + metadata = os.fstat(descriptor) + try: + named = path.stat(follow_symlinks=False) + except OSError: + raise _race_conflict(".", "The project root changed during repair.") from None + if ( + not stat.S_ISDIR(metadata.st_mode) + or not stat.S_ISDIR(named.st_mode) + or (metadata.st_dev, metadata.st_ino) != expected + or (named.st_dev, named.st_ino) != expected + ): + raise _race_conflict(".", "The project root changed during repair.") + _require_private_directory(descriptor, ".") + + +def _project_root(target: str | Path) -> Path: + try: + requested = Path(target).expanduser() + if requested.is_symlink(): + raise OSError + root = requested.absolute() + except (OSError, RuntimeError, ValueError): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-target-invalid", + "The repair target must be an existing project directory.", + ), + ) + ) from None + if not root.is_dir() or not (root / "lakefile.toml").is_file(): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-target-invalid", + "The repair target must be the existing project root.", + "lakefile.toml", + ), + ) + ) + return root + + +def _require_config_identity(root_descriptor: int, inspection) -> None: + assert inspection.lake is not None + assert inspection.lean is not None + expected = { + inspection.lake.path: inspection.lake.sha256, + inspection.lean.path: inspection.lean.sha256, + } + for relative, digest in expected.items(): + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, "O_CLOEXEC", 0) + try: + descriptor = os.open(relative, flags, dir_fd=root_descriptor) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise OSError(errno.EINVAL, "project configuration is not a regular file") + content = os.read(descriptor, 2 * 1024 * 1024 + 1) + finally: + os.close(descriptor) + except OSError: + raise _race_conflict(relative, "Project configuration changed during repair.") from None + if not stat.S_ISREG(metadata.st_mode) or hashlib.sha256(content).hexdigest() != digest: + raise _race_conflict(relative, "Project configuration changed during repair.") + + +def _inspection_conflicts(inspection) -> list[ProjectRepairConflict]: + conflicts = [ + ProjectRepairConflict( + "project-repair-inspection-failed", + diagnostic.message, + diagnostic.path, + ) + for diagnostic in inspection.diagnostics + if diagnostic.severity == "error" + ] + if inspection.autoform.manifest_path is not None: + conflicts.append( + ProjectRepairConflict( + "project-repair-workspace-unsupported", + "Use workspace and blueprint commands for a manifest-managed repository; legacy project repair would create an unrelated blueprint/ vault.", + inspection.autoform.manifest_path, + ) + ) + if inspection.lake is None or inspection.lake.name is None: + conflicts.append( + ProjectRepairConflict( + "project-repair-package-indeterminate", + "The existing Lake package name is required for repair.", + "lakefile.toml", + ) + ) + if inspection.lean is None: + conflicts.append( + ProjectRepairConflict( + "project-repair-toolchain-indeterminate", + "An existing lean-toolchain is required for repair.", + "lean-toolchain", + ) + ) + if inspection.compatibility.status != "supported" or inspection.compatibility.release is None: + conflicts.append( + ProjectRepairConflict( + "project-repair-release-indeterminate", + "Repair requires an existing Lean/Mathlib pair from the bundled release catalog.", + ) + ) + return conflicts + + +def _render_overlay( + *, + title: str | None, + repository_url: str | None, + autoform_source: str | None, + autoform_ref: str | None, +) -> tuple[_PlannedFile, ...]: + if (autoform_source is None) != (autoform_ref is None): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-input-invalid", + "--autoform-source and --autoform-ref must be supplied together.", + ), + ), + code="project-repair-input-invalid", + ) + if autoform_source is not None and ( + not autoform_source.strip() or not autoform_ref or not autoform_ref.strip() + ): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-input-invalid", + "--autoform-source and --autoform-ref must both be nonempty.", + ), + ), + code="project-repair-input-invalid", + ) + with tempfile.TemporaryDirectory(prefix="autoform-project-repair-") as temporary: + root = Path(temporary) + try: + result = scaffold_project( + root, + title=title if title is not None else "Autoform repair placeholder", + repository_url=repository_url if repository_url is not None else "", + autoform_source=( + autoform_source if autoform_source is not None else _RENDER_SOURCE + ), + autoform_ref=autoform_ref if autoform_ref is not None else _RENDER_REF, + discover_plugin_pin=False, + ) + except ScaffoldError as error: + raise ProjectRepairError( + tuple( + ProjectRepairConflict("project-repair-input-invalid", issue) + for issue in error.issues + ), + code="project-repair-input-invalid", + ) from None + if result.unpinned: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-input-invalid", + "The supplied workflow provenance did not produce pinned workflows.", + ), + ), + code="project-repair-input-invalid", + ) + rendered = [] + for relative in sorted(result.written): + path = root / relative + rendered.append( + _PlannedFile( + relative, + path.read_bytes(), + stat.S_IMODE(path.stat().st_mode), + _REQUIRED_INPUTS.get(relative, ()), + ) + ) + return tuple(rendered) + + +def _managed_path_state(root_descriptor: int, path: str) -> str: + root_device = os.fstat(root_descriptor).st_dev + descriptor = os.dup(root_descriptor) + try: + for part in PurePosixPath(path).parts[:-1]: + flags = ( + os.O_RDONLY + | os.O_DIRECTORY + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0) + ) + try: + child = os.open(part, flags, dir_fd=descriptor) + except FileNotFoundError: + return "absent" + except OSError: + return "unsafe" + try: + child_device = os.fstat(child).st_dev + except BaseException: + os.close(child) + raise + if child_device != root_device: + os.close(child) + return "unsafe" + os.close(descriptor) + descriptor = child + try: + os.stat( + PurePosixPath(path).name, + dir_fd=descriptor, + follow_symlinks=False, + ) + except FileNotFoundError: + return "absent" + except OSError: + return "unsafe" + return "exists" + finally: + os.close(descriptor) + + +def _scope_workflow_files( + root_descriptor: int, + desired: tuple[_PlannedFile, ...], + provided_inputs: frozenset[str], +) -> tuple[_PlannedFile, ...]: + if {"autoform-source", "autoform-ref"} <= provided_inputs: + return desired + states = tuple( + _managed_path_state(root_descriptor, path) for path in _WORKFLOW_PATHS + ) + if states != ("absent", "absent"): + return desired + return tuple(item for item in desired if not item.path.startswith(".github/")) + + +def _find_recovery_conflicts( + root_descriptor: int, desired: tuple[_PlannedFile, ...] +) -> list[ProjectRepairConflict]: + conflicts: list[ProjectRepairConflict] = [] + root_device = os.fstat(root_descriptor).st_dev + for item in desired: + descriptor = os.dup(root_descriptor) + try: + safe_parent = True + for part in PurePosixPath(item.path).parts[:-1]: + flags = ( + os.O_RDONLY + | os.O_DIRECTORY + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0) + ) + try: + child = os.open(part, flags, dir_fd=descriptor) + except OSError: + safe_parent = False + break + try: + child_device = os.fstat(child).st_dev + except BaseException: + os.close(child) + raise + if child_device != root_device: + os.close(child) + safe_parent = False + break + os.close(descriptor) + descriptor = child + if not safe_parent: + continue + name = PurePosixPath(item.path).name + try: + entries = os.listdir(descriptor) + except OSError: + continue + parent = PurePosixPath(item.path).parent + for entry in sorted(entries): + if not re.fullmatch( + rf"\.{re.escape(name)}\.autoform-repair-[0-9a-f]{{16}}", + entry, + ): + continue + orphan_path = ( + entry if parent == PurePosixPath(".") else f"{parent}/{entry}" + ) + conflicts.append( + ProjectRepairConflict( + "project-repair-recovery-required", + "An unverified repair temporary file requires manual recovery.", + orphan_path, + ) + ) + finally: + os.close(descriptor) + return conflicts + + +def _plan( + root_descriptor: int, + desired: tuple[_PlannedFile, ...], + provided_inputs: frozenset[str], +) -> tuple[tuple[_PlannedFile, ...], tuple[str, ...], list[ProjectRepairConflict]]: + planned: list[_PlannedFile] = [] + preserved: list[str] = [] + conflicts: list[ProjectRepairConflict] = [] + root_device = os.fstat(root_descriptor).st_dev + for item in desired: + parent_descriptor = os.dup(root_descriptor) + try: + walked: list[str] = [] + blocked = False + for part in PurePosixPath(item.path).parts[:-1]: + walked.append(part) + path = "/".join(walked) + try: + parent_descriptor = _open_existing_directory( + parent_descriptor, + part, + path, + expected_device=root_device, + ) + except ProjectRepairError as error: + conflicts.extend(error.conflicts) + blocked = True + break + if blocked: + continue + name = PurePosixPath(item.path).name + try: + orphaned = sorted( + entry + for entry in os.listdir(parent_descriptor) + if re.fullmatch( + rf"\.{re.escape(name)}\.autoform-repair-[0-9a-f]{{16}}", + entry, + ) + ) + except OSError: + conflicts.append( + ProjectRepairConflict( + "project-repair-destination-invalid", + "A managed destination could not be inspected safely.", + item.path, + ) + ) + continue + for orphan in orphaned: + parent = PurePosixPath(item.path).parent + orphan_path = ( + orphan if parent == PurePosixPath(".") else f"{parent}/{orphan}" + ) + conflicts.append( + ProjectRepairConflict( + "project-repair-recovery-required", + "An unverified repair temporary file requires manual recovery.", + orphan_path, + ) + ) + try: + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + missing_inputs = tuple( + value for value in item.required_inputs if value not in provided_inputs + ) + if missing_inputs: + flags = ", ".join(f"--{value}" for value in missing_inputs) + conflicts.append( + ProjectRepairConflict( + "project-repair-input-required", + f"Repair requires explicit {flags} input to reconstruct this file.", + item.path, + ) + ) + else: + planned.append(item) + continue + except OSError: + conflicts.append( + ProjectRepairConflict( + "project-repair-destination-invalid", + "A managed destination could not be inspected safely.", + item.path, + ) + ) + continue + if stat.S_ISLNK(metadata.st_mode): + conflicts.append( + ProjectRepairConflict( + "project-repair-destination-symlink", + "A managed destination is a symbolic link.", + item.path, + ) + ) + elif not stat.S_ISREG(metadata.st_mode): + conflicts.append( + ProjectRepairConflict( + "project-repair-destination-not-file", + "A managed destination exists and is not a regular file.", + item.path, + ) + ) + else: + preserved.append(item.path) + finally: + os.close(parent_descriptor) + return tuple(planned), tuple(sorted(preserved)), conflicts + + +def _require_private_directory(descriptor: int, path: str) -> None: + mode = os.fstat(descriptor).st_mode + if mode & (stat.S_IWGRP | stat.S_IWOTH): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-parent-unsafe", + "A managed parent directory is group- or world-writable.", + path, + ), + ) + ) + + +def _validate_parent_chain(root_descriptor: int, path: str) -> None: + root_device = os.fstat(root_descriptor).st_dev + parent_descriptor = os.dup(root_descriptor) + try: + walked: list[str] = [] + for part in PurePosixPath(path).parts[:-1]: + walked.append(part) + parent_descriptor = _open_existing_directory( + parent_descriptor, + part, + "/".join(walked), + expected_device=root_device, + ) + finally: + os.close(parent_descriptor) + + +def _publish( + root: Path, + root_descriptor: int, + root_identity: tuple[int, int], + item: _PlannedFile, + inspection, +) -> str: + root_device = os.fstat(root_descriptor).st_dev + parent_descriptor = os.dup(root_descriptor) + outcome: str | None = None + try: + parts = PurePosixPath(item.path).parts + walked: list[str] = [] + parent_chain: list[_ParentIdentity] = [] + for part in parts[:-1]: + walked.append(part) + parent_descriptor = _open_existing_directory( + parent_descriptor, + part, + "/".join(walked), + expected_device=root_device, + ) + parent_chain.append( + _ParentIdentity( + name=part, + path="/".join(walked), + identity=_descriptor_identity(parent_descriptor), + ) + ) + outcome = _publish_file( + root_descriptor, + root, + root_identity, + parent_descriptor, + tuple(parent_chain), + parts[-1], + item, + inspection, + ) + return outcome + finally: + pending_error = sys.exc_info()[1] + try: + os.close(parent_descriptor) + except OSError: + conflict = ProjectRepairConflict( + "project-repair-close-failed", + "A managed parent descriptor could not be closed.", + item.path, + ) + if isinstance(pending_error, ProjectRepairError): + raise ProjectRepairError( + (*pending_error.conflicts, conflict), + code=pending_error.code, + written=pending_error.written, + ) from None + raise ProjectRepairError( + (conflict,), + code="project-repair-failed", + written=(item.path,) if outcome == "written" else (), + ) from None + + +def _require_parent_chain( + root_descriptor: int, expected: tuple[_ParentIdentity, ...] +) -> None: + root_device = os.fstat(root_descriptor).st_dev + descriptor = os.dup(root_descriptor) + try: + for link in expected: + descriptor = _open_existing_directory( + descriptor, + link.name, + link.path, + expected_device=root_device, + ) + if _descriptor_identity(descriptor) != link.identity: + raise _race_conflict( + link.path, "A managed parent directory changed during repair." + ) + finally: + os.close(descriptor) + + +def _open_existing_directory( + parent_descriptor: int, + name: str, + path: str, + *, + expected_device: int, +) -> int: + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + try: + child = os.open(name, flags, dir_fd=parent_descriptor) + except FileNotFoundError: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-parent-missing", + "A required managed parent directory is missing.", + path, + ), + ) + ) from None + except OSError: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-parent-not-directory", + "A required parent path is not a safe directory.", + path, + ), + ) + ) from None + try: + _require_private_directory(child, path) + if os.fstat(child).st_dev != expected_device: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-parent-filesystem", + "A managed parent directory is on a different filesystem.", + path, + ), + ) + ) + except BaseException: + os.close(child) + raise + os.close(parent_descriptor) + return child + + +def _require_temporary_identity( + parent_descriptor: int, + name: str, + descriptor: int, + expected: tuple[int, int], +) -> None: + opened = os.fstat(descriptor) + named = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + not stat.S_ISREG(opened.st_mode) + or not stat.S_ISREG(named.st_mode) + or opened.st_nlink != 1 + or named.st_nlink != 1 + or (opened.st_dev, opened.st_ino) != expected + or (named.st_dev, named.st_ino) != expected + ): + raise OSError(errno.ESTALE, "temporary file changed during repair") + + +def _require_file_manifest( + parent_descriptor: int, + name: str, + descriptor: int, + expected_identity: tuple[int, int], + item: _PlannedFile, +) -> None: + _require_temporary_identity( + parent_descriptor, name, descriptor, expected_identity + ) + metadata = os.fstat(descriptor) + if metadata.st_size != len(item.content) or stat.S_IMODE(metadata.st_mode) != item.mode: + raise OSError(errno.ESTALE, "repair file metadata changed") + offset = os.lseek(descriptor, 0, os.SEEK_CUR) + try: + os.lseek(descriptor, 0, os.SEEK_SET) + chunks: list[bytes] = [] + remaining = len(item.content) + 1 + while remaining: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + finally: + os.lseek(descriptor, offset, os.SEEK_SET) + if b"".join(chunks) != item.content: + raise OSError(errno.ESTALE, "repair file content changed") + + +def _publish_file( + root_descriptor: int, + root: Path, + root_identity: tuple[int, int], + parent_descriptor: int, + parent_chain: tuple[_ParentIdentity, ...], + name: str, + item: _PlannedFile, + inspection, +) -> str: + temporary = f".{name}.autoform-repair-{secrets.token_hex(8)}" + parent = PurePosixPath(item.path).parent + temporary_path = ( + temporary if parent == PurePosixPath(".") else f"{parent}/{temporary}" + ) + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + descriptor: int | None = None + temporary_identity: tuple[int, int] | None = None + temporary_created = False + published = False + try: + descriptor = os.open(temporary, flags, 0o600, dir_fd=parent_descriptor) + temporary_created = True + temporary_metadata = os.fstat(descriptor) + temporary_identity = temporary_metadata.st_dev, temporary_metadata.st_ino + view = memoryview(item.content) + while view: + count = os.write(descriptor, view) + if count == 0: + raise OSError(errno.EIO, "short write") + view = view[count:] + os.fchmod(descriptor, item.mode) + os.fsync(descriptor) + _require_root_identity(root_descriptor, root_identity, root) + _require_config_identity(root_descriptor, inspection) + _require_parent_chain(root_descriptor, parent_chain) + _require_root_identity(root_descriptor, root_identity, root) + _require_file_manifest( + parent_descriptor, temporary, descriptor, temporary_identity, item + ) + try: + _rename_noreplace(parent_descriptor, temporary, parent_descriptor, name) + except FileExistsError: + winner_descriptor, winner_identity = _concurrent_result( + parent_descriptor, name, item + ) + try: + _require_root_identity(root_descriptor, root_identity, root) + _require_config_identity(root_descriptor, inspection) + _require_parent_chain(root_descriptor, parent_chain) + _require_root_identity(root_descriptor, root_identity, root) + _require_file_manifest( + parent_descriptor, + name, + winner_descriptor, + winner_identity, + item, + ) + finally: + pending_winner_error = sys.exc_info()[1] + try: + os.close(winner_descriptor) + except OSError: + conflict = ProjectRepairConflict( + "project-repair-close-failed", + "A concurrent destination descriptor could not be closed.", + item.path, + ) + if isinstance(pending_winner_error, ProjectRepairError): + raise ProjectRepairError( + (*pending_winner_error.conflicts, conflict), + code=pending_winner_error.code, + written=pending_winner_error.written, + ) from None + if isinstance(pending_winner_error, OSError): + validation_error = _race_conflict( + item.path, + "A concurrent destination changed during repair.", + ) + raise ProjectRepairError( + (*validation_error.conflicts, conflict), + code=validation_error.code, + ) from None + raise ProjectRepairError( + (conflict,), + code="project-repair-failed", + ) from None + raise _temporary_recovery_error(temporary_path) + except ProjectCreateError: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-safety-unavailable", + "Atomic no-replace publication is unavailable.", + item.path, + ), + ), + code="project-repair-safety-unavailable", + ) from None + published = True + try: + _require_file_manifest( + parent_descriptor, name, descriptor, temporary_identity, item + ) + _require_root_identity(root_descriptor, root_identity, root) + _require_config_identity(root_descriptor, inspection) + _require_parent_chain(root_descriptor, parent_chain) + except (OSError, ProjectRepairError): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-recovery-required", + "A published file was retained after it or its parent changed; inspect it before retrying.", + item.path, + ), + ), + code="project-repair-recovery-required", + written=(item.path,), + ) from None + try: + os.fsync(parent_descriptor) + except OSError: + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-durability-failed", + "A managed file was published but its directory could not be synchronized.", + item.path, + ), + ), + code="project-repair-failed", + written=(item.path,), + ) from None + try: + _require_root_identity(root_descriptor, root_identity, root) + _require_config_identity(root_descriptor, inspection) + _require_parent_chain(root_descriptor, parent_chain) + _require_root_identity(root_descriptor, root_identity, root) + _require_file_manifest( + parent_descriptor, name, descriptor, temporary_identity, item + ) + except (OSError, ProjectRepairError): + raise ProjectRepairError( + ( + ProjectRepairConflict( + "project-repair-recovery-required", + "A published file was retained after the project changed; inspect it before retrying.", + item.path, + ), + ), + code="project-repair-recovery-required", + written=(item.path,), + ) from None + return "written" + except ProjectRepairError as error: + if temporary_created and not published: + if error.code == "project-repair-recovery-required": + raise + raise _temporary_recovery_error( + temporary_path, + conflicts=error.conflicts, + written=error.written, + ) from None + raise + except OSError: + conflicts = ( + ProjectRepairConflict( + "project-repair-write-failed", + "A managed file could not be published safely.", + item.path, + ), + ) + if temporary_created and not published: + raise _temporary_recovery_error( + temporary_path, + conflicts=conflicts, + ) from None + raise ProjectRepairError(conflicts, code="project-repair-failed") from None + finally: + pending_error = sys.exc_info()[1] + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + conflict = ProjectRepairConflict( + "project-repair-close-failed", + "A staged file descriptor could not be closed.", + item.path, + ) + if isinstance(pending_error, ProjectRepairError): + raise ProjectRepairError( + (*pending_error.conflicts, conflict), + code=pending_error.code, + written=pending_error.written, + ) from None + if temporary_created and not published: + raise _temporary_recovery_error( + temporary_path, + conflicts=(conflict,), + ) from None + raise ProjectRepairError( + (conflict,), + code="project-repair-failed", + written=(item.path,) if published else (), + ) from None + + +def _temporary_recovery_error( + path: str, + *, + conflicts: tuple[ProjectRepairConflict, ...] = (), + written: tuple[str, ...] = (), +) -> ProjectRepairError: + recovery = ProjectRepairConflict( + "project-repair-recovery-required", + "A repair temporary was retained after publication did not complete; inspect it before retrying.", + path, + ) + return ProjectRepairError( + (*conflicts, recovery), + code="project-repair-recovery-required", + written=written, + ) + + +def _concurrent_result( + parent_descriptor: int, name: str, item: _PlannedFile +) -> tuple[int, tuple[int, int]]: + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, "O_CLOEXEC", 0) + descriptor: int | None = None + try: + descriptor = os.open(name, flags, dir_fd=parent_descriptor) + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise OSError(errno.EINVAL, "managed destination is not a regular file") + content = os.read(descriptor, len(item.content) + 1) + except OSError: + error = _race_conflict(item.path, "A managed destination changed during repair.") + if descriptor is not None: + error = _close_concurrent_descriptor(descriptor, item, error) + raise error from None + except BaseException: + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + pass + raise + if content == item.content: + assert descriptor is not None + return descriptor, (metadata.st_dev, metadata.st_ino) + assert descriptor is not None + error = _race_conflict(item.path, "A different managed file appeared during repair.") + raise _close_concurrent_descriptor(descriptor, item, error) + + +def _close_concurrent_descriptor( + descriptor: int, + item: _PlannedFile, + error: ProjectRepairError, +) -> ProjectRepairError: + try: + os.close(descriptor) + except OSError: + conflict = ProjectRepairConflict( + "project-repair-close-failed", + "A concurrent destination descriptor could not be closed.", + item.path, + ) + return ProjectRepairError( + (*error.conflicts, conflict), + code=error.code, + written=error.written, + ) + return error + + +def _race_conflict(path: str, message: str) -> ProjectRepairError: + return ProjectRepairError( + (ProjectRepairConflict("project-repair-race-conflict", message, path),), + code="project-repair-race-conflict", + ) + + +__all__ = [ + "PROJECT_REPAIR_SCHEMA", + "ProjectRepairConflict", + "ProjectRepairError", + "ProjectRepairResult", + "repair_project", +] diff --git a/autoform_cli/provenance.py b/autoform_cli/provenance.py new file mode 100644 index 00000000..2ee5dc73 --- /dev/null +++ b/autoform_cli/provenance.py @@ -0,0 +1,1101 @@ +"""Resolve and verify immutable provenance for the running Autoform plugin. + +The source and revision emitted here are persisted in generated workflows. A +candidate is therefore returned only when its remote commit is obtainable and +the installed runtime and plugin surface match that commit. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import py_compile +import re +import selectors +import stat +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +try: # pragma: no cover - exercised only on Python 3.10 + import tomllib +except ModuleNotFoundError: # pragma: no cover + import tomli as tomllib + + +INSTALL_RECORD = ".codex-marketplace-install.json" +MAX_INSTALL_RECORD_BYTES = 64 * 1024 + +_MAX_GIT_TEXT_BYTES = 16 * 1024 +_MAX_GIT_LIST_BYTES = 8 * 1024 * 1024 +_MAX_MANIFEST_ENTRIES = 20_000 +_MAX_SHIPPED_FILE_BYTES = 16 * 1024 * 1024 +_MAX_SHIPPED_TOTAL_BYTES = 64 * 1024 * 1024 +_MAX_PATH_DEPTH = 64 + +_FULL_SHA = re.compile(r"[0-9a-f]{40}") +_SOURCE_HOST = re.compile( + r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*" + r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?" +) +_SOURCE_PATH_PART = re.compile(r"[A-Za-z0-9._~-]+") +_GITHUB_SCP_SOURCE = re.compile( + r"git@github\.com:(?P[A-Za-z0-9._~-]+(?:/[A-Za-z0-9._~-]+)+)" +) +_BYTECODE_NAME = re.compile( + r"(?P.+?)\.(?P[A-Za-z0-9_-]+)" + r"(?:\.opt-(?P[A-Za-z0-9]+))?\.pyc" +) + +# These paths are consumed by a plugin host or by the packaged Python runtime. +# Tests, repository policy, and CI files are development inputs, not installed +# executable state. Package roots declared by pyproject.toml are added below. +_SHIPPED_ROOTS = frozenset( + { + ".claude-plugin", + ".codex-plugin", + ".muse-plugin", + "assets", + "skills", + } +) +_OPTIONAL_SHIPPED_ROOTS = frozenset({"agents", "commands", "hooks"}) +_SHIPPED_FILES = frozenset({".mcp.json", "pyproject.toml", "uv.lock"}) + +# These are host- or tool-owned state rather than source. The exact list is +# deliberately local; arbitrary gitignored paths are not automatically trusted. +_DERIVED_ROOTS = frozenset( + { + ".claude", + ".git", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".venv", + "dist", + "node_modules", + } +) +_DERIVED_DIRECTORY_NAMES = frozenset({".lake", "__pycache__", "site", "site-src"}) +_DERIVED_FILE_NAMES = frozenset({".DS_Store", ".zuliprc"}) +_IMPORTABLE_SUFFIXES = frozenset({".py", ".pyc", ".pyo", ".pth", ".so", ".pyd", ".dylib"}) +_PLUGIN_ROOT = Path(os.path.abspath(Path(__file__).parent.parent)) + + +class ProvenanceError(ValueError): + """The running plugin could not be tied to one verified remote commit.""" + + code = "project-provenance-unavailable" + + def __init__(self, message: str) -> None: + self.message = message + super().__init__(message) + + +@dataclass(frozen=True, slots=True) +class PluginProvenance: + """A credential-free Git source and the exact verified commit it serves.""" + + source: str + revision: str + + def as_dict(self) -> dict[str, object]: + return {"ok": True, "revision": self.revision, "source": self.source} + + +@dataclass(frozen=True, slots=True) +class _Candidate: + source: str + revision: str + + +@dataclass(frozen=True, slots=True) +class _TreeObject: + mode: int + kind: str + object_id: str + + +@dataclass(frozen=True, slots=True) +class _ManifestEntry: + mode: int + content: bytes + + +@dataclass(frozen=True, slots=True) +class _SourceLayout: + files: dict[str, _ManifestEntry] + all_files: frozenset[str] + roots: tuple[str, ...] + package_roots: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _ActualEntry: + mode: int + content: bytes + size: int + mtime_ns: int + + +@dataclass(frozen=True, slots=True) +class _CachedBytecode: + parent: str + name: str + content: bytes + + +class _GitFailure(RuntimeError): + pass + + +class _InvalidJson(ValueError): + pass + + +def plugin_root() -> Path: + """Return the source root that contains the running ``autoform_cli``.""" + + return _PLUGIN_ROOT + + +def normalize_git_source( + source: str, + *, + allow_github_scp: bool = False, + add_git_suffix: bool = False, +) -> str | None: + """Return a canonical credential-free HTTPS Git URL, or ``None``.""" + + if not isinstance(source, str) or not source or source != source.strip(): + return None + if any(character.isspace() or ord(character) < 32 or ord(character) == 127 for character in source): + return None + if allow_github_scp: + scp = _GITHUB_SCP_SOURCE.fullmatch(source) + if scp is not None: + source = f"https://github.com/{scp.group('path')}" + try: + parsed = urlsplit(source) + port = parsed.port + except ValueError: + return None + hostname = parsed.hostname + if ( + parsed.scheme.lower() != "https" + or not hostname + or parsed.username is not None + or parsed.password is not None + or port is not None + or parsed.query + or parsed.fragment + or parsed.netloc.lower() != hostname.lower() + or _SOURCE_HOST.fullmatch(hostname) is None + ): + return None + parts = parsed.path.split("/") + if ( + len(parts) < 2 + or parts[0] + or any(part in {"", ".", ".."} for part in parts[1:]) + or any(_SOURCE_PATH_PART.fullmatch(part) is None for part in parts[1:]) + ): + return None + if not parts[-1].endswith(".git"): + if not add_git_suffix: + return None + parts[-1] += ".git" + if parts[-1] == ".git": + return None + return urlunsplit(("https", hostname.lower(), "/".join(parts), "", "")) + + +def _git_environment() -> dict[str, str]: + """Build an environment that cannot redirect Git outside owned scratch.""" + + environment = { + key: value + for key, value in os.environ.items() + if not key.upper().startswith("GIT_") + } + environment.update( + { + "GCM_INTERACTIVE": "never", + "GIT_ASKPASS": os.devnull, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0", + "LC_ALL": "C", + } + ) + return environment + + +def _stop_process(process: subprocess.Popen[bytes]) -> None: + if process.poll() is None: + process.kill() + try: + process.wait(timeout=5) + except (OSError, subprocess.SubprocessError): + pass + + +def _run_git( + arguments: list[str], + *, + cwd: Path, + timeout: int = 15, + max_stdout_bytes: int = _MAX_GIT_TEXT_BYTES, +) -> bytes: + """Run Git with bounded output and no inherited Git control variables.""" + + process: subprocess.Popen[bytes] | None = None + try: + process = subprocess.Popen( + [ + "git", + "-c", + "credential.helper=", + "-c", + f"core.hooksPath={os.devnull}", + "-c", + "protocol.allow=never", + "-c", + "protocol.https.allow=always", + *arguments, + ], + cwd=cwd, + env=_git_environment(), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + assert process.stdout is not None + deadline = time.monotonic() + timeout + output = bytearray() + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _GitFailure + if not selector.select(remaining): + raise _GitFailure + chunk = os.read( + process.stdout.fileno(), + min(64 * 1024, max_stdout_bytes + 1 - len(output)), + ) + if not chunk: + break + output.extend(chunk) + if len(output) > max_stdout_bytes: + raise _GitFailure + remaining = max(0.1, deadline - time.monotonic()) + if process.wait(timeout=remaining) != 0: + raise _GitFailure + return bytes(output) + except _GitFailure: + if process is not None: + _stop_process(process) + raise + except (OSError, subprocess.SubprocessError) as error: + if process is not None: + _stop_process(process) + raise _GitFailure from error + finally: + if process is not None and process.stdout is not None: + process.stdout.close() + + +def _git_text(arguments: list[str], *, cwd: Path) -> str: + try: + value = _run_git(arguments, cwd=cwd).decode("utf-8", errors="strict").strip() + except (UnicodeDecodeError, _GitFailure) as error: + raise _GitFailure from error + if not value or "\n" in value or "\r" in value: + raise _GitFailure + return value + + +def _directory_flags() -> int: + no_follow = getattr(os, "O_NOFOLLOW", None) + directory = getattr(os, "O_DIRECTORY", None) + if ( + no_follow is None + or directory is None + or os.open not in os.supports_dir_fd + or os.stat not in os.supports_dir_fd + or os.stat not in os.supports_follow_symlinks + or os.listdir not in os.supports_fd + ): + raise ProvenanceError("This platform cannot inspect Autoform provenance safely.") + return os.O_RDONLY | no_follow | directory | getattr(os, "O_CLOEXEC", 0) + + +def _stat_signature(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _open_root(root: Path) -> tuple[Path, int]: + selected = Path(os.path.abspath(root.expanduser())) + try: + before = selected.lstat() + if not stat.S_ISDIR(before.st_mode) or stat.S_ISLNK(before.st_mode): + raise ProvenanceError("The Autoform plugin root is invalid.") + descriptor = os.open(selected, _directory_flags()) + opened = os.fstat(descriptor) + if _stat_signature(opened) != _stat_signature(before): + os.close(descriptor) + raise ProvenanceError("The Autoform plugin root changed during inspection.") + except ProvenanceError: + raise + except OSError as error: + raise ProvenanceError("The Autoform plugin root is unavailable.") from error + return selected, descriptor + + +def _require_root_identity(root: Path, descriptor: int) -> None: + try: + path_status = root.lstat() + opened = os.fstat(descriptor) + except OSError as error: + raise ProvenanceError("The Autoform plugin root changed during inspection.") from error + if _stat_signature(path_status) != _stat_signature(opened): + raise ProvenanceError("The Autoform plugin root changed during inspection.") + + +def _checkout_candidate(root: Path, root_descriptor: int) -> _Candidate | None: + try: + marker = os.stat(".git", dir_fd=root_descriptor, follow_symlinks=False) + except FileNotFoundError: + return None + except OSError as error: + raise ProvenanceError("The Autoform checkout metadata is invalid.") from error + if not (stat.S_ISDIR(marker.st_mode) or stat.S_ISREG(marker.st_mode)): + raise ProvenanceError("The Autoform checkout metadata is invalid.") + try: + top = Path(_git_text(["rev-parse", "--show-toplevel"], cwd=root)) + top_status = top.stat() + if (top_status.st_dev, top_status.st_ino) != ( + os.fstat(root_descriptor).st_dev, + os.fstat(root_descriptor).st_ino, + ): + raise ProvenanceError("The Autoform checkout is not rooted at the plugin root.") + raw_source = _git_text(["remote", "get-url", "origin"], cwd=root) + revision = _git_text(["rev-parse", "--verify", "HEAD^{commit}"], cwd=root).lower() + except _GitFailure as error: + raise ProvenanceError("The Autoform checkout metadata is invalid.") from error + except OSError as error: + raise ProvenanceError("The Autoform checkout metadata is invalid.") from error + source = normalize_git_source(raw_source, allow_github_scp=True, add_git_suffix=True) + if source is None or _FULL_SHA.fullmatch(revision) is None: + raise ProvenanceError("The Autoform checkout metadata is invalid.") + return _Candidate(source=source, revision=revision) + + +def _read_bounded_regular( + parent_descriptor: int, + name: str, + *, + limit: int, + message: str, +) -> tuple[bytes, os.stat_result] | None: + try: + before = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + return None + except OSError as error: + raise ProvenanceError(message) from error + if not stat.S_ISREG(before.st_mode) or before.st_size > limit: + raise ProvenanceError(message) + flags = os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NONBLOCK", 0) + try: + descriptor = os.open(name, flags, dir_fd=parent_descriptor) + try: + opened = os.fstat(descriptor) + if _stat_signature(opened) != _stat_signature(before): + raise ProvenanceError(message) + chunks: list[bytes] = [] + remaining = limit + 1 + while remaining: + chunk = os.read(descriptor, min(remaining, 64 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + content = b"".join(chunks) + after = os.fstat(descriptor) + final = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + len(content) > limit + or len(content) != opened.st_size + or _stat_signature(after) != _stat_signature(opened) + or _stat_signature(final) != _stat_signature(opened) + ): + raise ProvenanceError(message) + return content, opened + finally: + os.close(descriptor) + except ProvenanceError: + raise + except OSError as error: + raise ProvenanceError(message) from error + + +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise _InvalidJson + result[key] = value + return result + + +def _read_install_record(root_descriptor: int) -> _Candidate | None: + read = _read_bounded_regular( + root_descriptor, + INSTALL_RECORD, + limit=MAX_INSTALL_RECORD_BYTES, + message="The Autoform installer record is invalid.", + ) + if read is None: + return None + encoded, _ = read + try: + payload = json.loads( + encoded.decode("utf-8", errors="strict"), + object_pairs_hook=_unique_object, + ) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError, _InvalidJson) as error: + raise ProvenanceError("The Autoform installer record is invalid.") from error + if not isinstance(payload, dict): + raise ProvenanceError("The Autoform installer record is invalid.") + source_type = payload.get("source_type") + raw_source = payload.get("source") + raw_revision = payload.get("revision") + ref_name = payload.get("ref_name") + sparse_paths = payload.get("sparse_paths") + if ( + type(source_type) is not str + or source_type != "git" + or type(raw_source) is not str + or type(raw_revision) is not str + or type(ref_name) is not str + or type(sparse_paths) is not list + or any(type(path) is not str for path in sparse_paths) + ): + raise ProvenanceError("The Autoform installer record is invalid.") + source = normalize_git_source(raw_source, allow_github_scp=True, add_git_suffix=True) + revision = raw_revision.lower() + if source is None or _FULL_SHA.fullmatch(revision) is None: + raise ProvenanceError("The Autoform installer record is invalid.") + normalized_ref = ref_name.lower() + if _FULL_SHA.fullmatch(normalized_ref) is not None and normalized_ref != revision: + raise ProvenanceError("The Autoform installer record conflicts with its revision.") + return _Candidate(source=source, revision=revision) + + +def _valid_relative_path(encoded: bytes) -> str: + relative = os.fsdecode(encoded) + path = PurePosixPath(relative) + if ( + not relative + or relative.startswith("/") + or "\\" in relative + or path.is_absolute() + or len(path.parts) > _MAX_PATH_DEPTH + or path.as_posix() != relative + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise _GitFailure + return relative + + +def _read_git_blob(repository: Path, entry: _TreeObject) -> bytes: + try: + size = int(_git_text(["cat-file", "-s", entry.object_id], cwd=repository)) + except (ValueError, _GitFailure) as error: + raise _GitFailure from error + if size < 0 or size > _MAX_SHIPPED_FILE_BYTES: + raise _GitFailure + content = _run_git( + ["cat-file", "blob", entry.object_id], + cwd=repository, + max_stdout_bytes=size, + ) + if len(content) != size: + raise _GitFailure + return content + + +def _package_roots(pyproject: bytes) -> tuple[str, ...]: + try: + project = tomllib.loads(pyproject.decode("utf-8", errors="strict")) + name = project["project"]["name"] + entry_point = project["project"]["scripts"]["autoform"] + packages = project["tool"]["hatch"]["build"]["targets"]["wheel"]["packages"] + except (KeyError, TypeError, UnicodeDecodeError, ValueError) as error: + raise _GitFailure from error + if name != "autoform" or entry_point != "autoform_cli.__main__:main": + raise _GitFailure + if type(packages) is not list or any(type(path) is not str for path in packages): + raise _GitFailure + roots: list[str] = [] + for raw in packages: + path = PurePosixPath(raw) + if ( + not raw + or raw.startswith("/") + or "\\" in raw + or len(path.parts) != 1 + or path.as_posix() != raw + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise _GitFailure + roots.append(raw) + if "autoform_cli" not in roots or len(set(roots)) != len(roots): + raise _GitFailure + return tuple(sorted(roots)) + + +def _under_root(relative: str, root: str) -> bool: + return relative == root or relative.startswith(f"{root}/") + + +def _fetch_source_layout(source: str, revision: str, scratch: Path) -> _SourceLayout: + repository = scratch / "repository.git" + _run_git(["init", "--bare", "--template=", str(repository)], cwd=scratch) + _run_git( + ["fetch", "--no-tags", "--no-recurse-submodules", "--depth=1", source, revision], + cwd=repository, + timeout=60, + max_stdout_bytes=1024 * 1024, + ) + resolved = _git_text(["rev-parse", "--verify", "FETCH_HEAD^{commit}"], cwd=repository).lower() + if resolved != revision: + raise _GitFailure + listing = _run_git( + ["ls-tree", "-rz", "--full-tree", resolved], + cwd=repository, + max_stdout_bytes=_MAX_GIT_LIST_BYTES, + ) + objects: dict[str, _TreeObject] = {} + for raw_entry in listing.split(b"\0"): + if not raw_entry: + continue + if len(objects) >= _MAX_MANIFEST_ENTRIES: + raise _GitFailure + try: + raw_header, raw_path = raw_entry.split(b"\t", 1) + raw_mode, raw_kind, raw_object = raw_header.split(b" ", 2) + mode = int(raw_mode, 8) + kind = raw_kind.decode("ascii") + object_id = raw_object.decode("ascii") + except (UnicodeDecodeError, ValueError) as error: + raise _GitFailure from error + relative = _valid_relative_path(raw_path) + if relative in objects: + raise _GitFailure + objects[relative] = _TreeObject(mode=mode, kind=kind, object_id=object_id) + + pyproject_object = objects.get("pyproject.toml") + if pyproject_object is None or pyproject_object.kind != "blob" or pyproject_object.mode != 0o100644: + raise _GitFailure + pyproject = _read_git_blob(repository, pyproject_object) + package_roots = _package_roots(pyproject) + optional_roots = { + root + for root in _OPTIONAL_SHIPPED_ROOTS + if any(_under_root(path, root) for path in objects) + } + roots = tuple(sorted(set((*_SHIPPED_ROOTS, *optional_roots, *package_roots)))) + for root in roots: + if not any(_under_root(path, root) for path in objects): + raise _GitFailure + if not _SHIPPED_FILES.issubset(objects): + raise _GitFailure + + manifest: dict[str, _ManifestEntry] = {} + total = 0 + for relative, tree_object in sorted(objects.items()): + in_boundary = relative in _SHIPPED_FILES or any( + _under_root(relative, root) for root in roots + ) + if not in_boundary or PurePosixPath(relative).name == ".DS_Store": + continue + path = PurePosixPath(relative) + if "__pycache__" in path.parts or path.suffix in {".pyc", ".pyo"}: + raise _GitFailure + if tree_object.kind != "blob" or tree_object.mode not in {0o100644, 0o100755}: + raise _GitFailure + content = pyproject if relative == "pyproject.toml" else _read_git_blob(repository, tree_object) + total += len(content) + if total > _MAX_SHIPPED_TOTAL_BYTES: + raise _GitFailure + manifest[relative] = _ManifestEntry(mode=tree_object.mode, content=content) + return _SourceLayout( + files=manifest, + all_files=frozenset(objects), + roots=roots, + package_roots=package_roots, + ) + + +def _safe_names(directory_descriptor: int, counter: list[int]) -> list[str]: + try: + names = os.listdir(directory_descriptor) + except OSError as error: + raise ProvenanceError("The installed Autoform files could not be inspected safely.") from error + counter[0] += len(names) + if counter[0] > _MAX_MANIFEST_ENTRIES: + raise ProvenanceError("The installed Autoform tree is too large to verify safely.") + if any( + not isinstance(name, str) + or not name + or name in {".", ".."} + or "/" in name + or "\\" in name + for name in names + ): + raise ProvenanceError("The installed Autoform tree contains an invalid path.") + return sorted(names) + + +def _open_child_directory(parent_descriptor: int, name: str, before: os.stat_result) -> int: + try: + child = os.open(name, _directory_flags(), dir_fd=parent_descriptor) + opened = os.fstat(child) + except OSError as error: + raise ProvenanceError("The installed Autoform files could not be inspected safely.") from error + if _stat_signature(opened) != _stat_signature(before): + os.close(child) + raise ProvenanceError("The installed Autoform tree changed during inspection.") + return child + + +def _require_child_identity(parent_descriptor: int, name: str, descriptor: int) -> None: + try: + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + opened = os.fstat(descriptor) + except OSError as error: + raise ProvenanceError("The installed Autoform tree changed during inspection.") from error + if _stat_signature(current) != _stat_signature(opened): + raise ProvenanceError("The installed Autoform tree changed during inspection.") + + +def _read_actual_file( + parent_descriptor: int, + name: str, + budget: list[int], +) -> _ActualEntry: + read = _read_bounded_regular( + parent_descriptor, + name, + limit=_MAX_SHIPPED_FILE_BYTES, + message="The installed Autoform files do not match the recorded commit.", + ) + if read is None: + raise ProvenanceError("The installed Autoform files do not match the recorded commit.") + content, metadata = read + budget[0] += len(content) + if budget[0] > _MAX_SHIPPED_TOTAL_BYTES: + raise ProvenanceError("The installed Autoform tree is too large to verify safely.") + mode = 0o100755 if metadata.st_mode & 0o111 else 0o100644 + return _ActualEntry( + mode=mode, + content=content, + size=metadata.st_size, + mtime_ns=metadata.st_mtime_ns, + ) + + +def _scan_pycache( + parent_descriptor: int, + name: str, + before: os.stat_result, + *, + source_parent: str, + counter: list[int], + budget: list[int], + bytecode: list[_CachedBytecode], +) -> None: + descriptor = _open_child_directory(parent_descriptor, name, before) + try: + for child_name in _safe_names(descriptor, counter): + try: + metadata = os.stat(child_name, dir_fd=descriptor, follow_symlinks=False) + except OSError as error: + raise ProvenanceError("The installed bytecode cache is invalid.") from error + if not stat.S_ISREG(metadata.st_mode) or not child_name.endswith(".pyc"): + raise ProvenanceError("The installed bytecode cache is invalid.") + entry = _read_actual_file(descriptor, child_name, budget) + if entry.mode != 0o100644: + raise ProvenanceError("The installed bytecode cache is invalid.") + bytecode.append(_CachedBytecode(source_parent, child_name, entry.content)) + _require_child_identity(parent_descriptor, name, descriptor) + finally: + os.close(descriptor) + + +def _scan_boundary_directory( + descriptor: int, + prefix: str, + *, + files: dict[str, _ActualEntry], + directories: set[str], + bytecode: list[_CachedBytecode], + counter: list[int], + budget: list[int], + depth: int, +) -> None: + if depth > _MAX_PATH_DEPTH: + raise ProvenanceError("The installed Autoform tree is too deep to verify safely.") + for name in _safe_names(descriptor, counter): + relative = f"{prefix}/{name}" if prefix else name + try: + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + except OSError as error: + raise ProvenanceError("The installed Autoform files could not be inspected safely.") from error + if name == ".DS_Store" and stat.S_ISREG(metadata.st_mode): + continue + if stat.S_ISDIR(metadata.st_mode): + if name == "__pycache__": + _scan_pycache( + descriptor, + name, + metadata, + source_parent=prefix, + counter=counter, + budget=budget, + bytecode=bytecode, + ) + continue + directories.add(relative) + child = _open_child_directory(descriptor, name, metadata) + try: + _scan_boundary_directory( + child, + relative, + files=files, + directories=directories, + bytecode=bytecode, + counter=counter, + budget=budget, + depth=depth + 1, + ) + _require_child_identity(descriptor, name, child) + finally: + os.close(child) + continue + if not stat.S_ISREG(metadata.st_mode): + raise ProvenanceError("The installed Autoform tree contains a link or special file.") + files[relative] = _read_actual_file(descriptor, name, budget) + + +def _open_boundary_root( + root_descriptor: int, + relative: str, + directories: set[str], +) -> tuple[int, list[tuple[int, str, int]]]: + descriptor = os.dup(root_descriptor) + opened: list[tuple[int, str, int]] = [] + prefix: list[str] = [] + try: + for name in PurePosixPath(relative).parts: + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode): + raise ProvenanceError("The installed Autoform files do not match the recorded commit.") + child = _open_child_directory(descriptor, name, metadata) + opened.append((descriptor, name, child)) + prefix.append(name) + directories.add("/".join(prefix)) + descriptor = child + return descriptor, opened + except (OSError, ProvenanceError): + for parent, _, child in reversed(opened): + os.close(child) + os.close(parent) + if not opened: + os.close(descriptor) + raise ProvenanceError("The installed Autoform files do not match the recorded commit.") from None + + +def _close_boundary_root(opened: list[tuple[int, str, int]]) -> None: + for parent, name, child in reversed(opened): + try: + _require_child_identity(parent, name, child) + finally: + os.close(child) + os.close(parent) + + +def _expected_directories(files: dict[str, _ManifestEntry]) -> set[str]: + directories: set[str] = set() + for relative in files: + parent = PurePosixPath(relative).parent + while parent != PurePosixPath("."): + directories.add(parent.as_posix()) + parent = parent.parent + return directories + + +def _validate_current_bytecode( + root: Path, + cached: _CachedBytecode, + source_relative: str, + source: _ActualEntry, + expected_source: bytes, + optimization: int, +) -> None: + content = cached.content + if len(content) < 16 or content[:4] != importlib.util.MAGIC_NUMBER: + raise ProvenanceError("The installed bytecode cache does not match its source.") + flags = int.from_bytes(content[4:8], "little") + if flags == 0: + timestamp = int.from_bytes(content[8:12], "little") + source_size = int.from_bytes(content[12:16], "little") + if timestamp != (int(source.mtime_ns // 1_000_000_000) & 0xFFFFFFFF): + raise ProvenanceError("The installed bytecode cache does not match its source.") + if source_size != (source.size & 0xFFFFFFFF): + raise ProvenanceError("The installed bytecode cache does not match its source.") + elif flags == 3: + if content[8:16] != importlib.util.source_hash(expected_source): + raise ProvenanceError("The installed bytecode cache does not match its source.") + else: + # Unchecked-hash bytecode can supersede the verified source by design. + raise ProvenanceError("The installed bytecode cache does not match its source.") + source_path = root.joinpath(*PurePosixPath(source_relative).parts) + try: + with tempfile.TemporaryDirectory(prefix="autoform-bytecode-") as directory: + temporary_source = Path(directory, "source.py") + temporary_cache = Path(directory, "source.pyc") + temporary_source.write_bytes(expected_source) + py_compile.compile( + os.fspath(temporary_source), + cfile=os.fspath(temporary_cache), + dfile=os.fspath(source_path), + doraise=True, + optimize=optimization, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + expected_payload = temporary_cache.read_bytes()[16:] + except (MemoryError, OSError, OverflowError, py_compile.PyCompileError) as error: + raise ProvenanceError("The installed bytecode cache could not be verified.") from error + if content[16:] != expected_payload: + raise ProvenanceError("The installed bytecode cache does not match its source.") + + +def _validate_bytecode( + root: Path, + bytecode: list[_CachedBytecode], + actual: dict[str, _ActualEntry], + expected: dict[str, _ManifestEntry], +) -> None: + current_tag = sys.implementation.cache_tag + if not current_tag: + raise ProvenanceError("The installed bytecode cache cannot be verified.") + for cached in bytecode: + match = _BYTECODE_NAME.fullmatch(cached.name) + if match is None: + raise ProvenanceError("The installed bytecode cache is invalid.") + source_relative = PurePosixPath(cached.parent, f"{match.group('stem')}.py").as_posix() + expected_entry = expected.get(source_relative) + actual_entry = actual.get(source_relative) + if expected_entry is None or actual_entry is None: + raise ProvenanceError("The installed bytecode cache has no verified source.") + if match.group("tag") != current_tag: + continue + raw_optimization = match.group("optimization") + if raw_optimization is None: + optimization = 0 + elif raw_optimization in {"1", "2"}: + optimization = int(raw_optimization) + else: + raise ProvenanceError("The installed bytecode cache is invalid.") + _validate_current_bytecode( + root, + cached, + source_relative, + actual_entry, + expected_entry.content, + optimization, + ) + + +def _is_derived_path(relative: str) -> bool: + path = PurePosixPath(relative) + return ( + path.parts[0] in _DERIVED_ROOTS + or any(part in _DERIVED_DIRECTORY_NAMES for part in path.parts) + or path.name in _DERIVED_FILE_NAMES + or relative == INSTALL_RECORD + ) + + +def _looks_importable(relative: str) -> bool: + name = PurePosixPath(relative).name + return any(name.endswith(suffix) for suffix in _IMPORTABLE_SUFFIXES) + + +def _scan_for_extra_importable( + descriptor: int, + prefix: str, + *, + layout: _SourceLayout, + counter: list[int], + depth: int, +) -> None: + if depth > _MAX_PATH_DEPTH: + raise ProvenanceError("The installed Autoform tree is too deep to verify safely.") + for name in _safe_names(descriptor, counter): + relative = f"{prefix}/{name}" if prefix else name + if any(_under_root(relative, root) for root in layout.roots): + continue + if _is_derived_path(relative): + continue + try: + metadata = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + except OSError as error: + raise ProvenanceError("The installed Autoform files could not be inspected safely.") from error + if stat.S_ISDIR(metadata.st_mode): + child = _open_child_directory(descriptor, name, metadata) + try: + _scan_for_extra_importable( + child, + relative, + layout=layout, + counter=counter, + depth=depth + 1, + ) + _require_child_identity(descriptor, name, child) + finally: + os.close(child) + elif relative not in layout.all_files and ( + not stat.S_ISREG(metadata.st_mode) or _looks_importable(relative) + ): + raise ProvenanceError("The installed Autoform tree contains extra importable code.") + + +def _compare_installed_tree( + root: Path, + root_descriptor: int, + layout: _SourceLayout, +) -> None: + actual_files: dict[str, _ActualEntry] = {} + actual_directories: set[str] = set() + bytecode: list[_CachedBytecode] = [] + counter = [0] + budget = [0] + + roots: list[str] = [] + for candidate in sorted(layout.roots, key=lambda value: (len(PurePosixPath(value).parts), value)): + if not any(_under_root(candidate, selected) for selected in roots): + roots.append(candidate) + for relative in roots: + descriptor, opened = _open_boundary_root(root_descriptor, relative, actual_directories) + try: + _scan_boundary_directory( + descriptor, + relative, + files=actual_files, + directories=actual_directories, + bytecode=bytecode, + counter=counter, + budget=budget, + depth=len(PurePosixPath(relative).parts), + ) + finally: + _close_boundary_root(opened) + + for relative in _SHIPPED_FILES: + if len(PurePosixPath(relative).parts) != 1: + raise ProvenanceError("The installed Autoform boundary is invalid.") + actual_files[relative] = _read_actual_file(root_descriptor, relative, budget) + + expected_directories = _expected_directories(layout.files) + if set(actual_files) != set(layout.files) or actual_directories != expected_directories: + raise ProvenanceError("The installed Autoform tree does not match the recorded commit.") + for relative, expected in layout.files.items(): + found = actual_files[relative] + if found.mode != expected.mode or found.content != expected.content: + raise ProvenanceError("The installed Autoform files do not match the recorded commit.") + _validate_bytecode(root, bytecode, actual_files, layout.files) + _scan_for_extra_importable( + root_descriptor, + "", + layout=layout, + counter=[0], + depth=0, + ) + + +def verify_plugin_provenance(root: Path | None = None) -> PluginProvenance: + """Verify the source, remote commit, and installed plugin before returning.""" + + selected_root, root_descriptor = _open_root(root or plugin_root()) + try: + checkout = _checkout_candidate(selected_root, root_descriptor) + record = _read_install_record(root_descriptor) + if checkout is not None and record is not None and checkout != record: + raise ProvenanceError("The Autoform checkout and installer record conflict.") + candidate = checkout or record + if candidate is None: + raise ProvenanceError("No trustworthy Autoform source and commit are available.") + _require_root_identity(selected_root, root_descriptor) + try: + with tempfile.TemporaryDirectory(prefix="autoform-provenance-") as temporary: + layout = _fetch_source_layout( + candidate.source, + candidate.revision, + Path(temporary), + ) + _compare_installed_tree(selected_root, root_descriptor, layout) + # Re-read the complete boundary before committing the result. + # A mutation after an earlier root was scanned must not be + # hidden by that root's unchanged parent-directory identity. + _compare_installed_tree(selected_root, root_descriptor, layout) + except ProvenanceError: + raise + except (_GitFailure, OSError) as error: + raise ProvenanceError("The recorded Autoform commit could not be verified.") from error + _require_root_identity(selected_root, root_descriptor) + return PluginProvenance(source=candidate.source, revision=candidate.revision) + finally: + os.close(root_descriptor) + + +def plugin_pin() -> tuple[str, str]: + """Compatibility tuple for callers that need all-or-nothing provenance.""" + + try: + provenance = verify_plugin_provenance() + except ProvenanceError: + return "", "" + return provenance.source, provenance.revision + + +__all__ = [ + "INSTALL_RECORD", + "MAX_INSTALL_RECORD_BYTES", + "PluginProvenance", + "ProvenanceError", + "normalize_git_source", + "plugin_pin", + "plugin_root", + "verify_plugin_provenance", +] diff --git a/autoform_cli/ready.py b/autoform_cli/ready.py new file mode 100644 index 00000000..3030145c --- /dev/null +++ b/autoform_cli/ready.py @@ -0,0 +1,252 @@ +"""List work that is ready for a host agent to formalize.""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path + +from .execution_input import ( + ExecutionInput, + ExecutionInputError, + ExecutionInputIssue, + load_execution_input, +) +from .runtime import RuntimeNode, RuntimeStatus + + +READY_SCHEMA = "autoform-ready/v1" + + +@dataclass(frozen=True, order=True, slots=True) +class ReadyItem: + """One formalization phase whose authored prerequisites are satisfied.""" + + node_id: str + article_id: str + article_path: str + title: str + phase: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True, order=True, slots=True) +class ReadyBlock: + """One dispatchable phase that cannot start from the current graph state.""" + + node_id: str + article_id: str + article_path: str + title: str + phase: str + blocked_by: tuple[str, ...] + reasons: tuple[str, ...] + + def as_dict(self) -> dict[str, object]: + return { + "article_id": self.article_id, + "article_path": self.article_path, + "blocked_by": list(self.blocked_by), + "node_id": self.node_id, + "phase": self.phase, + "reasons": list(self.reasons), + "title": self.title, + } + + +@dataclass(frozen=True, slots=True) +class ReadyResult: + """A deterministic work projection over one immutable execution input.""" + + source_revision: str + source_contract_sha256: str + items: tuple[ReadyItem, ...] + blocked_items: tuple[ReadyBlock, ...] + blocked: int + complete: int + workspace_project_id: str | None + workspace_project_binding_sha256: str | None + + def as_dict(self) -> dict[str, object]: + return { + "blocked": self.blocked, + "blocked_items": [item.as_dict() for item in self.blocked_items], + "complete": self.complete, + "items": [item.as_dict() for item in self.items], + "schema": READY_SCHEMA, + "source_contract_sha256": self.source_contract_sha256, + "source_revision": self.source_revision, + "workspace": { + "project_binding_sha256": self.workspace_project_binding_sha256, + "project_id": self.workspace_project_id, + }, + } + + def to_json(self) -> str: + return json.dumps(self.as_dict(), sort_keys=True, separators=(",", ":")) + + +def list_ready_work( + project_or_blueprint: str | Path, + *, + lean_root: str | Path | None = None, + project_id: str | None = None, +) -> ReadyResult: + """Load the exhaustive source contract and list its ready leaf phases.""" + + execution = load_execution_input( + project_or_blueprint, + lean_root=lean_root, + project_id=project_id, + ) + completion_issues = _completion_issues(execution) + if completion_issues: + raise ExecutionInputError(completion_issues) + + items: list[ReadyItem] = [] + blocked_items: list[ReadyBlock] = [] + complete = 0 + statuses = {node.id: node.status for node in execution.runtime.nodes} + for node in execution.runtime.nodes: + if not node.dispatchable: + continue + phase = _ready_phase(node) + if phase is not None: + assert node.article_id is not None + items.append( + ReadyItem( + node_id=node.id, + article_id=node.article_id, + article_path=node.article_path, + title=node.title, + phase=phase, + ) + ) + elif node.mathlib or node.status.proved: + complete += 1 + else: + blocked_items.append(_blocked_item(node, statuses)) + return ReadyResult( + source_revision=execution.runtime.source_revision, + source_contract_sha256=execution.source_contract_sha256, + items=tuple(sorted(items)), + blocked_items=tuple(sorted(blocked_items)), + blocked=len(blocked_items), + complete=complete, + workspace_project_id=execution.workspace_project_id, + workspace_project_binding_sha256=execution.workspace_project_binding_sha256, + ) + + +def _ready_phase(node: RuntimeNode) -> str | None: + if not node.dispatchable or node.assertions.not_ready or node.mathlib: + return None + if not node.status.stated: + return "statement" if node.status.can_state else None + if not node.status.proved: + return "proof" if node.status.can_prove else None + return None + + +def _completion_issues(execution: ExecutionInput) -> list[ExecutionInputIssue]: + issues: list[ExecutionInputIssue] = [] + for node in execution.runtime.nodes: + if not node.dispatchable: + continue + if node.mathlib: + if not node.mathlib_declarations or not node.mathlib_file: + issues.append( + ExecutionInputIssue( + "mathlib-evidence-missing", + f"{node.article_path}: mathlib completion lacks declaration or file evidence", + ) + ) + continue + if not ( + node.assertions.statement_formalized + or node.assertions.proof_formalized + ): + continue + if execution.lean_source_revision is None: + issues.append( + ExecutionInputIssue( + "lean-root-required", + f"{node.article_path}: formalized local work requires --lean-root", + ) + ) + continue + if not node.lean_targets: + issues.append( + ExecutionInputIssue( + "missing-lean-target", + f"{node.article_path}: formalized local work has no Lean declaration target", + ) + ) + continue + missing = tuple( + target.declaration + for target in node.lean_targets + if target.source_file is None + ) + if missing: + issues.append( + ExecutionInputIssue( + "lean-target-not-found", + f"{node.article_path}: Lean declaration target was not found: {', '.join(missing)}", + ) + ) + return issues + + +def _blocked_item( + node: RuntimeNode, + statuses: dict[str, RuntimeStatus], +) -> ReadyBlock: + phase = "statement" if not node.status.stated else "proof" + reasons: list[str] = [] + blocked_by: set[str] = set() + if node.assertions.not_ready: + reasons.append("authored-not-ready") + + missing_statements = tuple( + dependency + for dependency in node.statement_dependencies + if not statuses[dependency].stated + ) + if missing_statements: + reasons.append("statement-dependency-not-stated") + blocked_by.update(missing_statements) + + if phase == "proof": + missing_proofs = tuple( + dependency + for dependency in node.proof_dependencies + if not statuses[dependency].proved + ) + if missing_proofs: + reasons.append("proof-dependency-not-proved") + blocked_by.update(missing_proofs) + + if not reasons: + reasons.append("prerequisites-not-satisfied") + assert node.article_id is not None + return ReadyBlock( + node_id=node.id, + article_id=node.article_id, + article_path=node.article_path, + title=node.title, + phase=phase, + blocked_by=tuple(sorted(blocked_by)), + reasons=tuple(reasons), + ) + + +__all__ = [ + "READY_SCHEMA", + "ReadyBlock", + "ReadyItem", + "ReadyResult", + "list_ready_work", +] diff --git a/autoform_cli/render.py b/autoform_cli/render.py index 7bc5894c..af50f522 100644 --- a/autoform_cli/render.py +++ b/autoform_cli/render.py @@ -9,25 +9,68 @@ from __future__ import annotations +import ctypes +import errno import hashlib import html import json +import os import re +import secrets import shutil +import stat +import tempfile +import unicodedata from collections.abc import Iterable from dataclasses import dataclass, field +from html.parser import HTMLParser from pathlib import Path +from pathlib import PurePosixPath +from typing import Callable from urllib.parse import quote, unquote, urlsplit from . import graph_pages, graph_views, mermaid, status -from .coverage import CoverageSummary, load_coverage +from . import workspace as workspace_module +from ._tree_snapshot import ( + BoundDirectoryTree, + TreeSelection, + TreeSnapshot, + TreeSnapshotError, + bind_directory_tree, +) +from .coverage import COVERAGE_V2_SCHEMA, CoverageSummary, load_coverage from .graph import Graph, Node, load_graph -from .lean import SourceLinker, build_linker, declaration_names +from .lean import ( + BoundProjectSources, + IndexedSourceSnapshot, + SourceLinker, + build_linker, + declaration_names, + detect_ref, + detect_repository_url, + open_project_sources, + project_source_revision, + snapshot_project_sources, +) +from .markdown import INLINE_CODE, content, render_html from .status import is_definition +try: + import fcntl +except ImportError: # pragma: no cover - Windows import compatibility + fcntl = None # type: ignore[assignment] + _HEADING = re.compile(r"^ {0,3}(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$") _FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") -_MARKDOWN_LINK = re.compile(r"(?[^\]]*)\]\(\s*(?P[^)\s]+)(?:\s+[^)]*)?\)") +_MARKDOWN_LINK = re.compile( + r"(?[^\]]*)\]\(\s*" + r"(?P<[^>\r\n]+>|[^)\s]+)(?P\s+[^)]*)?\)" +) +_ANY_INLINE_LINK = re.compile( + r"(?P!?)\[(?P