From edcdd1b8b100fe586bdd260fb568a7239ed7a886 Mon Sep 17 00:00:00 2001 From: mulatta <67085791+mulatta@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:11:13 +0900 Subject: [PATCH 1/5] flake: support portable container tooling --- devshell.nix | 16 ++++++++++++ flake.nix | 72 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 26 deletions(-) create mode 100644 devshell.nix diff --git a/devshell.nix b/devshell.nix new file mode 100644 index 0000000..f4ad314 --- /dev/null +++ b/devshell.nix @@ -0,0 +1,16 @@ +{ pkgs, formatter }: +pkgs.mkShellNoCC { + packages = with pkgs; [ + cosign + curl + docker-buildx + docker-client + gh + git + jq + python312 + skopeo + syft + formatter + ]; +} diff --git a/flake.nix b/flake.nix index 530d7c0..9998eaf 100644 --- a/flake.nix +++ b/flake.nix @@ -8,40 +8,60 @@ }; outputs = - { + inputs@{ self, nixpkgs, treefmt-nix, }: let - system = "x86_64-linux"; - pkgs = nixpkgs.legacyPackages.${system}; - treefmtEval = treefmt-nix.lib.evalModule pkgs { - projectRootFile = "flake.nix"; - programs = { - deadnix.enable = true; - keep-sorted.enable = true; - nixfmt.enable = true; - ruff-format.enable = true; - statix.enable = true; - }; + inherit (nixpkgs) lib; + + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ]; + eachSystem = lib.genAttrs systems; + + flake = self // { + inherit inputs; }; + + pkgsFor = eachSystem (system: import nixpkgs { inherit system; }); + + scopes = eachSystem ( + system: + let + pkgs = pkgsFor.${system}; + treefmtEval = treefmt-nix.lib.evalModule pkgs { + projectRootFile = "flake.nix"; + programs = { + deadnix.enable = true; + keep-sorted.enable = true; + nixfmt.enable = true; + ruff-format.enable = true; + statix.enable = true; + }; + }; + in + lib.makeScope pkgs.newScope (self: { + inherit flake inputs system; + formatter = treefmtEval.config.build.wrapper; + formatting = treefmtEval.config.build.check flake; + devshell = self.callPackage ./devshell.nix { }; + }) + ); in { - checks.${system}.formatting = treefmtEval.config.build.check self; - - devShells.${system}.default = pkgs.mkShell { - packages = with pkgs; [ - cosign - docker-client - gh - git - jq - python312 - syft - ]; - }; + checks = eachSystem (system: { + formatting = scopes.${system}.formatting; + devshell-default = scopes.${system}.devshell; + }); + + devShells = eachSystem (system: { + default = scopes.${system}.devshell; + }); - formatter.${system} = treefmtEval.config.build.wrapper; + formatter = eachSystem (system: scopes.${system}.formatter); }; } From a5a60f173e75d03cf9f40cfc3e2801d94f910bd6 Mon Sep 17 00:00:00 2001 From: mulatta <67085791+mulatta@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:25:26 +0900 Subject: [PATCH 2/5] tooling: make image releases reproducible --- .gitignore | 3 + flake.nix | 1 + scripts/__init__.py | 0 scripts/_git_tags.py | 35 +++++++++ scripts/builder/__init__.py | 5 ++ scripts/builder/_metadata.py | 18 +++++ scripts/builder/build.py | 127 +++++++++++++++++++++++++++++++ scripts/builder/changed.py | 77 +++++++++++++++++++ scripts/builder/metadata_test.py | 26 +++++++ scripts/builder/publish_test.py | 43 +++++++++++ scripts/updater/__init__.py | 5 ++ scripts/updater/_git.py | 42 ++++++++++ scripts/updater/git_test.py | 78 +++++++++++++++++++ scripts/updater/run.py | 39 ++++++++++ scripts/updater/run_test.py | 23 ++++++ scripts/updater/semver.py | 48 ++++++++++++ scripts/updater/semver_test.py | 30 ++++++++ scripts/updater/update.py | 66 ++++++++++++++++ scripts/updater/update_test.py | 22 ++++++ 19 files changed, 688 insertions(+) create mode 100644 scripts/__init__.py create mode 100644 scripts/_git_tags.py create mode 100644 scripts/builder/__init__.py create mode 100644 scripts/builder/_metadata.py create mode 100644 scripts/builder/build.py create mode 100644 scripts/builder/changed.py create mode 100644 scripts/builder/metadata_test.py create mode 100644 scripts/builder/publish_test.py create mode 100644 scripts/updater/__init__.py create mode 100644 scripts/updater/_git.py create mode 100644 scripts/updater/git_test.py create mode 100644 scripts/updater/run.py create mode 100644 scripts/updater/run_test.py create mode 100644 scripts/updater/semver.py create mode 100644 scripts/updater/semver_test.py create mode 100644 scripts/updater/update.py create mode 100644 scripts/updater/update_test.py diff --git a/.gitignore b/.gitignore index e83dfce..80accf8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .direnv/ result result-* + +__pycache__/ +*.py[cod] diff --git a/flake.nix b/flake.nix index 9998eaf..5d3ab0b 100644 --- a/flake.nix +++ b/flake.nix @@ -35,6 +35,7 @@ pkgs = pkgsFor.${system}; treefmtEval = treefmt-nix.lib.evalModule pkgs { projectRootFile = "flake.nix"; + settings.global.excludes = [ "images/*/src/**" ]; programs = { deadnix.enable = true; keep-sorted.enable = true; diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/_git_tags.py b/scripts/_git_tags.py new file mode 100644 index 0000000..42df59b --- /dev/null +++ b/scripts/_git_tags.py @@ -0,0 +1,35 @@ +"""Resolve lightweight and annotated remote Git tags to commits.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +def parse(lines: list[str]) -> dict[str, str]: + """Map tags to commits, preferring peeled annotated-tag revisions.""" + direct: dict[str, str] = {} + peeled: dict[str, str] = {} + prefix = "refs/tags/" + for line in lines: + revision, ref = line.split("\t", 1) + if not ref.startswith(prefix): + continue + tag = ref[len(prefix) :] + if tag.endswith("^{}"): + peeled[tag[:-3]] = revision + else: + direct[tag] = revision + return direct | peeled + + +def remote(remote: str, *, cwd: Path | None = None) -> dict[str, str]: + """List remote tags without changing local refs or working trees.""" + lines = subprocess.run( + ["git", "ls-remote", "--tags", remote], + cwd=cwd, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.splitlines() + return parse(lines) diff --git a/scripts/builder/__init__.py b/scripts/builder/__init__.py new file mode 100644 index 0000000..18c6f45 --- /dev/null +++ b/scripts/builder/__init__.py @@ -0,0 +1,5 @@ +"""Shared image builder API.""" + +from .build import BuildResult, build + +__all__ = ["BuildResult", "build"] diff --git a/scripts/builder/_metadata.py b/scripts/builder/_metadata.py new file mode 100644 index 0000000..6793d71 --- /dev/null +++ b/scripts/builder/_metadata.py @@ -0,0 +1,18 @@ +"""Read structured metadata emitted by Docker Buildx.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") + + +def read_image_digest(path: Path) -> str: + """Return validated image digest from a Buildx metadata file.""" + metadata = json.loads(path.read_text()) + digest = metadata.get("containerimage.digest") + if not isinstance(digest, str) or _DIGEST.fullmatch(digest) is None: + raise ValueError(f"missing or invalid image digest in {path}") + return digest diff --git a/scripts/builder/build.py b/scripts/builder/build.py new file mode 100644 index 0000000..4d2bcee --- /dev/null +++ b/scripts/builder/build.py @@ -0,0 +1,127 @@ +"""Build one immutable submodule snapshot with Docker Buildx.""" + +from __future__ import annotations + +import subprocess +import tarfile +import tempfile +from dataclasses import dataclass +from pathlib import Path + +from scripts import _git_tags +from scripts.updater.semver import SemVer, latest_matching_tag + +from ._metadata import read_image_digest + + +@dataclass(frozen=True) +class BuildResult: + """Identity of a completed image build.""" + + digest: str + revision: str + version: str + + +def _output(arguments: list[str], *, cwd: Path) -> str: + return subprocess.run( + ["git", *arguments], + cwd=cwd, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + +def _snapshot_revision(image_dir: Path) -> tuple[Path, Path, str]: + image_dir = image_dir.resolve() + root = Path(_output(["rev-parse", "--show-toplevel"], cwd=image_dir)).resolve() + source = image_dir / "src" + relative = source.relative_to(root) + fields = _output(["ls-tree", "HEAD", "--", str(relative)], cwd=root).split() + if len(fields) < 3 or fields[0] != "160000" or fields[1] != "commit": + raise ValueError(f"no committed source submodule at {relative}") + return root, source, fields[2] + + +def _version_for_revision(source: Path, revision: str) -> str: + tags = _git_tags.remote("origin", cwd=source) + matching = [tag for tag, commit in tags.items() if commit == revision] + tag = latest_matching_tag(matching, r"^v?[0-9]+\.[0-9]+\.[0-9]+$") + return str(SemVer.parse(tag)) + + +def _export_snapshot(source: Path, revision: str, destination: Path) -> None: + archive = destination.parent / "source.tar" + with archive.open("wb") as output: + subprocess.run( + ["git", "archive", "--format=tar", revision], + cwd=source, + check=True, + stdout=output, + ) + destination.mkdir() + with tarfile.open(archive) as source_archive: + source_archive.extractall(destination, filter="data") + + +def _require_unpublished(image: str) -> None: + result = subprocess.run( + ["skopeo", "inspect", f"docker://{image}"], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode == 0: + raise ValueError(f"refusing to overwrite published image: {image}") + error = result.stderr.lower() + if "manifest unknown" not in error and "manifest_unknown" not in error: + raise subprocess.CalledProcessError( + result.returncode, + result.args, + stderr=result.stderr, + ) + + +def build( + *, + image_dir: Path, + image: str, + dockerfile: str = "Dockerfile", + push: bool = False, +) -> BuildResult: + """Build source recorded by committed submodule gitlink.""" + image_dir = image_dir.resolve() + _root, source, revision = _snapshot_revision(image_dir) + version = _version_for_revision(source, revision) + tag = f"{image}:{version}" + if push: + _require_unpublished(tag) + + with tempfile.TemporaryDirectory(prefix="container-build-") as directory: + temporary = Path(directory) + context = temporary / "src" + metadata = temporary / "metadata.json" + _export_snapshot(source, revision, context) + subprocess.run( + [ + "docker", + "buildx", + "build", + "--file", + str(context / dockerfile), + "--platform", + "linux/amd64", + "--tag", + tag, + "--provenance=mode=max", + "--sbom=true", + "--metadata-file", + str(metadata), + "--push" if push else "--load", + str(context), + ], + check=True, + ) + return BuildResult(read_image_digest(metadata), revision, version) diff --git a/scripts/builder/changed.py b/scripts/builder/changed.py new file mode 100644 index 0000000..33a740c --- /dev/null +++ b/scripts/builder/changed.py @@ -0,0 +1,77 @@ +"""Select images whose committed source gitlinks changed.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +from pathlib import Path + +_SOURCE_PATH = re.compile(r"^images/([^/]+)/src$") + + +def _output(root: Path, arguments: list[str]) -> str: + return subprocess.run( + ["git", *arguments], + cwd=root, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + +def _image_at_revision(root: Path, revision: str, path: str) -> str | None: + match = _SOURCE_PATH.fullmatch(path) + if match is None: + return None + entry = _output(root, ["ls-tree", revision, "--", path]).split() + if len(entry) < 3 or entry[0] != "160000" or entry[1] != "commit": + return None + return match[1] + + +def all_images(root: Path, revision: str) -> list[str]: + """List images backed by source submodules at a revision.""" + lines = _output(root, ["ls-tree", "-r", "--full-tree", revision, "--", "images"]) + paths = [line.split("\t", 1)[1] for line in lines.splitlines() if "\t" in line] + return sorted( + image + for path in paths + if (image := _image_at_revision(root, revision, path)) is not None + ) + + +def changed_images(root: Path, before: str, after: str) -> list[str]: + """List images whose source gitlink changed between two revisions.""" + if set(before) == {"0"}: + return all_images(root, after) + paths = _output( + root, + ["diff", "--name-only", "--diff-filter=AMRT", before, after, "--", "images"], + ).splitlines() + return sorted( + image + for path in paths + if (image := _image_at_revision(root, after, path)) is not None + ) + + +def main() -> None: + """Print one selected image name per line.""" + parser = argparse.ArgumentParser() + parser.add_argument("before", nargs="?") + parser.add_argument("after", nargs="?", default="HEAD") + parser.add_argument("--all", action="store_true") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + if args.all: + images = all_images(root, args.after) + elif args.before is None: + parser.error("before revision is required unless --all is used") + else: + images = changed_images(root, args.before, args.after) + print(*images, sep="\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/builder/metadata_test.py b/scripts/builder/metadata_test.py new file mode 100644 index 0000000..d28db0a --- /dev/null +++ b/scripts/builder/metadata_test.py @@ -0,0 +1,26 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.builder._metadata import read_image_digest + + +class MetadataTest(unittest.TestCase): + def test_reads_buildx_image_digest(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "metadata.json" + path.write_text(json.dumps({"containerimage.digest": "sha256:" + "a" * 64})) + self.assertEqual(read_image_digest(path), "sha256:" + "a" * 64) + + def test_rejects_missing_or_invalid_digest(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "metadata.json" + for metadata in ({}, {"containerimage.digest": "latest"}): + path.write_text(json.dumps(metadata)) + with self.subTest(metadata=metadata), self.assertRaises(ValueError): + read_image_digest(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/builder/publish_test.py b/scripts/builder/publish_test.py new file mode 100644 index 0000000..34c6d4b --- /dev/null +++ b/scripts/builder/publish_test.py @@ -0,0 +1,43 @@ +import os +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +from scripts.builder.build import _require_unpublished + + +class PublishPreflightTest(unittest.TestCase): + def run_skopeo(self, exit_code: int, error: str) -> Exception | None: + with tempfile.TemporaryDirectory() as directory: + executable = Path(directory) / "skopeo" + executable.write_text( + f"#!/bin/sh\nprintf '%s' '{error}' >&2\nexit {exit_code}\n" + ) + executable.chmod(executable.stat().st_mode | stat.S_IXUSR) + previous = os.environ["PATH"] + os.environ["PATH"] = f"{directory}:{previous}" + try: + try: + _require_unpublished("registry.example/image:1.0.0") + except (ValueError, subprocess.CalledProcessError) as error_result: + return error_result + return None + finally: + os.environ["PATH"] = previous + + def test_rejects_existing_tag(self) -> None: + self.assertIsInstance(self.run_skopeo(0, ""), ValueError) + + def test_allows_only_missing_manifest(self) -> None: + self.assertIsNone(self.run_skopeo(1, "manifest unknown")) + + def test_propagates_registry_failure(self) -> None: + error = self.run_skopeo(1, "unauthorized") + self.assertIsNotNone(error) + self.assertNotIsInstance(error, ValueError) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/updater/__init__.py b/scripts/updater/__init__.py new file mode 100644 index 0000000..96d6ef1 --- /dev/null +++ b/scripts/updater/__init__.py @@ -0,0 +1,5 @@ +"""Shared image source updater API.""" + +from .update import UpdateResult, update + +__all__ = ["UpdateResult", "update"] diff --git a/scripts/updater/_git.py b/scripts/updater/_git.py new file mode 100644 index 0000000..c1437a3 --- /dev/null +++ b/scripts/updater/_git.py @@ -0,0 +1,42 @@ +"""Git operations used by image source updates.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +def output(arguments: list[str], *, cwd: Path) -> str: + """Run Git and return stripped stdout.""" + return subprocess.run( + ["git", *arguments], + cwd=cwd, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + +def repository_root(path: Path) -> Path: + """Find superproject root containing an image directory.""" + return Path(output(["rev-parse", "--show-toplevel"], cwd=path)).resolve() + + +def gitlink_revision(root: Path, source: Path) -> str: + """Read exact commit recorded for a submodule path.""" + relative = source.relative_to(root) + fields = output(["ls-files", "--stage", "--", str(relative)], cwd=root).split() + if len(fields) < 2 or fields[0] != "160000": + raise ValueError(f"not a registered submodule: {relative}") + return fields[1] + + +def revision(repository: Path) -> str: + """Read commit currently checked out in a repository.""" + return output(["rev-parse", "HEAD^{commit}"], cwd=repository) + + +def require_clean(source: Path) -> None: + """Refuse to overwrite local source changes.""" + if output(["status", "--porcelain"], cwd=source): + raise ValueError(f"submodule has local changes: {source}") diff --git a/scripts/updater/git_test.py b/scripts/updater/git_test.py new file mode 100644 index 0000000..5e4dbf8 --- /dev/null +++ b/scripts/updater/git_test.py @@ -0,0 +1,78 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + +from scripts.builder.build import _snapshot_revision +from scripts.builder.changed import changed_images +from scripts.updater import _git + + +def git(repository: Path, *arguments: str) -> str: + return subprocess.run( + ["git", *arguments], + cwd=repository, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + +class SubmoduleRevisionTest(unittest.TestCase): + def test_reads_checked_out_revision_before_gitlink_is_staged(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + upstream = root / "upstream" + superproject = root / "superproject" + upstream.mkdir() + superproject.mkdir() + + git(upstream, "init") + git(upstream, "config", "user.name", "Test") + git(upstream, "config", "user.email", "test@example.invalid") + (upstream / "source").write_text("first") + git(upstream, "add", "source") + git(upstream, "commit", "-m", "first") + first = git(upstream, "rev-parse", "HEAD") + (upstream / "source").write_text("second") + git(upstream, "commit", "-am", "second") + second = git(upstream, "rev-parse", "HEAD") + + git(superproject, "init") + git(superproject, "config", "user.name", "Test") + git(superproject, "config", "user.email", "test@example.invalid") + source = superproject / "images" / "example" / "src" + git( + superproject, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + str(upstream), + str(source.relative_to(superproject)), + ) + git(source, "checkout", "--detach", first) + git(superproject, "add", ".") + git(superproject, "commit", "-m", "pin first") + first_superproject = git(superproject, "rev-parse", "HEAD") + + git(source, "checkout", "--detach", second) + + self.assertEqual(_git.gitlink_revision(superproject, source), first) + self.assertEqual(_git.revision(source), second) + _root, _source, build_revision = _snapshot_revision( + superproject / "images" / "example" + ) + self.assertEqual(build_revision, first) + + git(superproject, "add", str(source.relative_to(superproject))) + git(superproject, "commit", "-m", "pin second") + second_superproject = git(superproject, "rev-parse", "HEAD") + self.assertEqual( + changed_images(superproject, first_superproject, second_superproject), + ["example"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/updater/run.py b/scripts/updater/run.py new file mode 100644 index 0000000..b3e0cd9 --- /dev/null +++ b/scripts/updater/run.py @@ -0,0 +1,39 @@ +"""Run project-specific image updaters.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + + +def discover(root: Path) -> list[Path]: + """Find image directories with an updater and source submodule.""" + images = root / "images" + if not images.is_dir(): + return [] + return sorted( + path.parent + for path in images.glob("*/update.py") + if (path.parent / "src").is_dir() + ) + + +def main() -> None: + """Run all updaters, or selected image updaters.""" + parser = argparse.ArgumentParser() + parser.add_argument("images", nargs="*") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + available = {path.name: path for path in discover(root)} + selected = args.images or sorted(available) + unknown = sorted(set(selected) - available.keys()) + if unknown: + parser.error(f"unknown images: {', '.join(unknown)}") + for name in selected: + subprocess.run([sys.executable, str(available[name] / "update.py")], check=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/updater/run_test.py b/scripts/updater/run_test.py new file mode 100644 index 0000000..ed35f89 --- /dev/null +++ b/scripts/updater/run_test.py @@ -0,0 +1,23 @@ +import tempfile +import unittest +from pathlib import Path + +from scripts.updater.run import discover + + +class DiscoverTest(unittest.TestCase): + def test_discovers_only_complete_image_updaters(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + complete = root / "images" / "complete" + incomplete = root / "images" / "incomplete" + complete.mkdir(parents=True) + incomplete.mkdir(parents=True) + (complete / "src").mkdir() + (complete / "update.py").touch() + (incomplete / "update.py").touch() + self.assertEqual(discover(root), [complete]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/updater/semver.py b/scripts/updater/semver.py new file mode 100644 index 0000000..ae7da49 --- /dev/null +++ b/scripts/updater/semver.py @@ -0,0 +1,48 @@ +"""Strict stable SemVer tag handling.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Iterable + +_STABLE_TAG = re.compile( + r"^v?(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)$" +) + + +@dataclass(frozen=True, order=True) +class SemVer: + """Stable SemVer core version.""" + + major: int + minor: int + patch: int + + def __str__(self) -> str: + """Render normalized version without a tag prefix.""" + return f"{self.major}.{self.minor}.{self.patch}" + + @classmethod + def parse(cls, tag: str) -> SemVer: + """Parse a stable SemVer tag with an optional ``v`` prefix.""" + match = _STABLE_TAG.fullmatch(tag) + if match is None: + raise ValueError(f"not a stable SemVer tag: {tag!r}") + return cls(*(int(match[name]) for name in ("major", "minor", "patch"))) + + +def latest_matching_tag(tags: Iterable[str], tag_pattern: str) -> str: + """Return highest stable SemVer tag accepted by project policy.""" + policy = re.compile(tag_pattern) + candidates: list[tuple[SemVer, str]] = [] + for tag in tags: + if policy.fullmatch(tag) is None: + continue + try: + candidates.append((SemVer.parse(tag), tag)) + except ValueError: + continue + if not candidates: + raise ValueError("no stable SemVer tag matches project policy") + return max(candidates)[1] diff --git a/scripts/updater/semver_test.py b/scripts/updater/semver_test.py new file mode 100644 index 0000000..52ace23 --- /dev/null +++ b/scripts/updater/semver_test.py @@ -0,0 +1,30 @@ +import unittest + +from scripts.updater.semver import SemVer, latest_matching_tag + + +class SemVerTest(unittest.TestCase): + def test_orders_numeric_components(self) -> None: + self.assertGreater(SemVer.parse("v1.10.0"), SemVer.parse("v1.9.9")) + + def test_rejects_non_stable_versions(self) -> None: + for tag in ("latest", "v1.2", "v1.2.3-rc.1", "v01.2.3"): + with self.subTest(tag=tag), self.assertRaises(ValueError): + SemVer.parse(tag) + + def test_selects_latest_tag_allowed_by_project_policy(self) -> None: + self.assertEqual( + latest_matching_tag( + ["nightly", "v2.0.0-rc.1", "v1.9.0", "v2.0.0"], + r"^v[0-9]+\.[0-9]+\.[0-9]+$", + ), + "v2.0.0", + ) + + def test_rejects_pattern_that_admits_no_stable_tag(self) -> None: + with self.assertRaisesRegex(ValueError, "no stable SemVer tag"): + latest_matching_tag(["nightly", "v1.0.0-rc.1"], r"^.*$") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/updater/update.py b/scripts/updater/update.py new file mode 100644 index 0000000..ed595b4 --- /dev/null +++ b/scripts/updater/update.py @@ -0,0 +1,66 @@ +"""Update an image source submodule to latest stable upstream tag.""" + +from __future__ import annotations + +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from scripts import _git_tags + +from . import _git +from .semver import SemVer, latest_matching_tag + +_UPSTREAM = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") + + +@dataclass(frozen=True) +class UpdateResult: + """Source snapshot selected by an update.""" + + old_revision: str + new_revision: str + version: str + + +def _remote_tags(repository: str) -> dict[str, str]: + if _UPSTREAM.fullmatch(repository) is None: + raise ValueError(f"invalid GitHub repository: {repository!r}") + return _git_tags.remote(f"https://github.com/{repository}.git") + + +def update( + *, + image_dir: Path, + upstream: str, + tag_pattern: str = r"^v?[0-9]+\.[0-9]+\.[0-9]+$", +) -> UpdateResult: + """Update ``image_dir/src`` to highest permitted stable SemVer tag.""" + image_dir = image_dir.resolve() + source = image_dir / "src" + root = _git.repository_root(image_dir) + if source.parent != image_dir or not source.is_dir(): + raise ValueError(f"missing source submodule: {source}") + + _git.gitlink_revision(root, source) + old_revision = _git.revision(source) + _git.require_clean(source) + tags = _remote_tags(upstream) + tag = latest_matching_tag(tags, tag_pattern) + new_revision = tags[tag] + version = str(SemVer.parse(tag)) + + if old_revision != new_revision: + url = f"https://github.com/{upstream}.git" + subprocess.run( + ["git", "fetch", "--depth=1", url, f"refs/tags/{tag}"], + cwd=source, + check=True, + ) + fetched = _git.output(["rev-parse", "FETCH_HEAD^{commit}"], cwd=source) + if fetched != new_revision: + raise ValueError(f"upstream tag changed while updating: {tag}") + subprocess.run(["git", "checkout", "--detach", fetched], cwd=source, check=True) + + return UpdateResult(old_revision, new_revision, version) diff --git a/scripts/updater/update_test.py b/scripts/updater/update_test.py new file mode 100644 index 0000000..a9af414 --- /dev/null +++ b/scripts/updater/update_test.py @@ -0,0 +1,22 @@ +import unittest + +from scripts._git_tags import parse + + +class RemoteTagsTest(unittest.TestCase): + def test_prefers_peeled_commit_for_annotated_tag(self) -> None: + tag_object = "a" * 40 + commit = "b" * 40 + self.assertEqual( + parse( + [ + f"{tag_object}\trefs/tags/v1.2.3", + f"{commit}\trefs/tags/v1.2.3^{{}}", + ] + ), + {"v1.2.3": commit}, + ) + + +if __name__ == "__main__": + unittest.main() From 0793b33678f1c13771a1631c9f8e18cee1af6d8a Mon Sep 17 00:00:00 2001 From: mulatta <67085791+mulatta@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:26:55 +0900 Subject: [PATCH 3/5] alphafold3: pin stable upstream source --- .gitmodules | 3 +++ images/alphafold3/build.py | 28 ++++++++++++++++++++++++++++ images/alphafold3/src | 1 + images/alphafold3/update.py | 17 +++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 .gitmodules create mode 100644 images/alphafold3/build.py create mode 160000 images/alphafold3/src create mode 100644 images/alphafold3/update.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..56d725c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "images/alphafold3/src"] + path = images/alphafold3/src + url = https://github.com/google-deepmind/alphafold3.git diff --git a/images/alphafold3/build.py b/images/alphafold3/build.py new file mode 100644 index 0000000..aaa6382 --- /dev/null +++ b/images/alphafold3/build.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Build AlphaFold 3 image from pinned upstream source.""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[2])) + +from scripts.builder import build + + +def main() -> None: + """Build AlphaFold 3 and optionally push its immutable version tag.""" + parser = argparse.ArgumentParser() + parser.add_argument("--push", action="store_true") + args = parser.parse_args() + result = build( + image_dir=Path(__file__).parent, + image="registry.sjanglab.org/sjanglab/alphafold3", + dockerfile="docker/Dockerfile", + push=args.push, + ) + print(result.digest) + + +if __name__ == "__main__": + main() diff --git a/images/alphafold3/src b/images/alphafold3/src new file mode 160000 index 0000000..85c4d20 --- /dev/null +++ b/images/alphafold3/src @@ -0,0 +1 @@ +Subproject commit 85c4d20505fd5cef05eac22b534d4e793971ae69 diff --git a/images/alphafold3/update.py b/images/alphafold3/update.py new file mode 100644 index 0000000..70248ae --- /dev/null +++ b/images/alphafold3/update.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Update AlphaFold 3 source to latest stable upstream release.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[2])) + +from scripts.updater import update + + +if __name__ == "__main__": + result = update( + image_dir=Path(__file__).parent, + upstream="google-deepmind/alphafold3", + ) + print(f"{result.old_revision} -> {result.new_revision} ({result.version})") From c3636b07e4f511f526a2b37fa88fb9221e65bea0 Mon Sep 17 00:00:00 2001 From: mulatta <67085791+mulatta@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:27:06 +0900 Subject: [PATCH 4/5] ci: isolate trusted image publication --- .github/actionlint.yaml | 5 +++ .github/workflows/check.yaml | 28 ++++++++++++++++ .github/workflows/release.yaml | 60 ++++++++++++++++++++++++++++++++++ .github/workflows/update.yaml | 56 +++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/check.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/update.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..8ee75d5 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - psi + - trusted-release + - container-release diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml new file mode 100644 index 0000000..f8a023f --- /dev/null +++ b/.github/workflows/check.yaml @@ -0,0 +1,28 @@ +name: Check + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Install Nix + uses: cachix/install-nix-action@v31 + - name: Check formatting + run: nix fmt -- --ci + - name: Run unit tests + run: nix develop -c python -m unittest discover -s scripts -p '*_test.py' + - name: Check image entrypoints + run: | + for script in images/*/build.py; do + nix develop -c python "$script" --help + done diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..68aef6b --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,60 @@ +name: Release Images + +on: + push: + branches: [main] + paths: + - "images/*/src" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-images + cancel-in-progress: false + +jobs: + release: + runs-on: [self-hosted, psi, trusted-release, container-release] + environment: container-release + steps: + - name: Checkout trusted revision + uses: actions/checkout@v7 + with: + fetch-depth: 0 + submodules: true + - name: Select images + id: images + env: + BEFORE: ${{ github.event.before }} + run: | + if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then + images=$(nix develop -c python scripts/builder/changed.py --all) + else + images=$(nix develop -c python scripts/builder/changed.py "$BEFORE" "$GITHUB_SHA") + fi + { + echo "names<> "$GITHUB_OUTPUT" + - name: Log in to registry + if: steps.images.outputs.names != '' + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + printf '%s' "$REGISTRY_PASSWORD" | docker login registry.sjanglab.org --username "$REGISTRY_USERNAME" --password-stdin + - name: Build and push images + if: steps.images.outputs.names != '' + env: + IMAGES: ${{ steps.images.outputs.names }} + run: | + while IFS= read -r image; do + [[ -n "$image" ]] || continue + nix develop -c python "images/$image/build.py" --push + done <<< "$IMAGES" + - name: Log out of registry + if: always() && steps.images.outputs.names != '' + run: docker logout registry.sjanglab.org diff --git a/.github/workflows/update.yaml b/.github/workflows/update.yaml new file mode 100644 index 0000000..1e76f7a --- /dev/null +++ b/.github/workflows/update.yaml @@ -0,0 +1,56 @@ +name: Update Images + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + actions: write + contents: write + pull-requests: write + +concurrency: + group: update-images + cancel-in-progress: false + +jobs: + update: + runs-on: ubuntu-latest + steps: + - name: Skip while update PR is open + id: existing + env: + GH_TOKEN: ${{ github.token }} + run: | + count=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --search 'in:title "images: update upstream releases"' --json number --jq length) + echo "skip=$([[ $count -gt 0 ]] && echo true || echo false)" >> "$GITHUB_OUTPUT" + - name: Checkout + if: steps.existing.outputs.skip != 'true' + uses: actions/checkout@v7 + with: + fetch-depth: 0 + submodules: true + - name: Install Nix + if: steps.existing.outputs.skip != 'true' + uses: cachix/install-nix-action@v31 + - name: Update images + if: steps.existing.outputs.skip != 'true' + run: nix develop -c python scripts/updater/run.py + - name: Create pull request + if: steps.existing.outputs.skip != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + if git diff --quiet -- images; then + echo "No image updates found" + exit 0 + fi + branch="automation/update-images-${GITHUB_RUN_ID}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$branch" + git add images/*/src + git commit -m "images: update upstream releases" + git push origin "$branch" + gh pr create --base main --head "$branch" --title "images: update upstream releases" --body "Update image source submodules to latest stable SemVer releases." From dab64cf1a6e54535a36cfcddd6de9aa949eb4c09 Mon Sep 17 00:00:00 2001 From: mulatta <67085791+mulatta@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:11:58 +0900 Subject: [PATCH 5/5] ci: let nixbot own repository checks Expose formatting, Python validation, tests, and workflow linting as flake checks so branch rules depend on one trusted evaluator instead of duplicate GitHub jobs. --- .github/workflows/check.yaml | 28 ------------------ .github/workflows/release.yaml | 1 - .github/workflows/update.yaml | 3 +- checks.nix | 52 ++++++++++++++++++++++++++++++++++ flake.nix | 14 ++++++--- images/alphafold3/build.py | 0 images/alphafold3/update.py | 1 - scripts/updater/semver.py | 2 +- 8 files changed, 64 insertions(+), 37 deletions(-) delete mode 100644 .github/workflows/check.yaml create mode 100644 checks.nix mode change 100644 => 100755 images/alphafold3/build.py mode change 100644 => 100755 images/alphafold3/update.py diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml deleted file mode 100644 index f8a023f..0000000 --- a/.github/workflows/check.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: Check - -on: - pull_request: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v7 - - name: Install Nix - uses: cachix/install-nix-action@v31 - - name: Check formatting - run: nix fmt -- --ci - - name: Run unit tests - run: nix develop -c python -m unittest discover -s scripts -p '*_test.py' - - name: Check image entrypoints - run: | - for script in images/*/build.py; do - nix develop -c python "$script" --help - done diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 68aef6b..d0b5988 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,7 +17,6 @@ concurrency: jobs: release: runs-on: [self-hosted, psi, trusted-release, container-release] - environment: container-release steps: - name: Checkout trusted revision uses: actions/checkout@v7 diff --git a/.github/workflows/update.yaml b/.github/workflows/update.yaml index 1e76f7a..84ed01d 100644 --- a/.github/workflows/update.yaml +++ b/.github/workflows/update.yaml @@ -6,7 +6,6 @@ on: workflow_dispatch: permissions: - actions: write contents: write pull-requests: write @@ -33,7 +32,7 @@ jobs: submodules: true - name: Install Nix if: steps.existing.outputs.skip != 'true' - uses: cachix/install-nix-action@v31 + uses: NixOS/nix-installer-action@main - name: Update images if: steps.existing.outputs.skip != 'true' run: nix develop -c python scripts/updater/run.py diff --git a/checks.nix b/checks.nix new file mode 100644 index 0000000..e50ec47 --- /dev/null +++ b/checks.nix @@ -0,0 +1,52 @@ +{ pkgs }: +let + source = pkgs.lib.cleanSource ./.; +in +{ + python-lint = + pkgs.runCommand "containers-python-lint" + { + nativeBuildInputs = with pkgs; [ + mypy + ruff + ]; + } + '' + cp -r ${source} source + chmod -R u+w source + cd source + ruff check scripts images/*/*.py + mypy scripts images/*/*.py + touch $out + ''; + + python-tests = + pkgs.runCommand "containers-python-tests" + { + nativeBuildInputs = with pkgs; [ + git + python312 + ]; + } + '' + cp -r ${source} source + chmod -R u+w source + cd source + export HOME=$TMPDIR + python -m unittest discover -s scripts -p '*_test.py' + touch $out + ''; + + workflows-lint = + pkgs.runCommand "containers-workflows-lint" + { + nativeBuildInputs = [ pkgs.actionlint ]; + } + '' + cp -r ${source} source + chmod -R u+w source + cd source + actionlint -config-file .github/actionlint.yaml .github/workflows/*.yaml + touch $out + ''; +} diff --git a/flake.nix b/flake.nix index 5d3ab0b..13c8e17 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,7 @@ deadnix.enable = true; keep-sorted.enable = true; nixfmt.enable = true; + ruff-check.enable = true; ruff-format.enable = true; statix.enable = true; }; @@ -50,14 +51,19 @@ formatter = treefmtEval.config.build.wrapper; formatting = treefmtEval.config.build.check flake; devshell = self.callPackage ./devshell.nix { }; + checks = self.callPackage ./checks.nix { }; }) ); in { - checks = eachSystem (system: { - formatting = scopes.${system}.formatting; - devshell-default = scopes.${system}.devshell; - }); + checks = eachSystem ( + system: + scopes.${system}.checks + // { + formatting = scopes.${system}.formatting; + devshell-default = scopes.${system}.devshell; + } + ); devShells = eachSystem (system: { default = scopes.${system}.devshell; diff --git a/images/alphafold3/build.py b/images/alphafold3/build.py old mode 100644 new mode 100755 diff --git a/images/alphafold3/update.py b/images/alphafold3/update.py old mode 100644 new mode 100755 index 70248ae..62940d7 --- a/images/alphafold3/update.py +++ b/images/alphafold3/update.py @@ -8,7 +8,6 @@ from scripts.updater import update - if __name__ == "__main__": result = update( image_dir=Path(__file__).parent, diff --git a/scripts/updater/semver.py b/scripts/updater/semver.py index ae7da49..5e2a290 100644 --- a/scripts/updater/semver.py +++ b/scripts/updater/semver.py @@ -3,8 +3,8 @@ from __future__ import annotations import re +from collections.abc import Iterable from dataclasses import dataclass -from typing import Iterable _STABLE_TAG = re.compile( r"^v?(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)$"