ci: automate trusted container releases

.github/actionlint.yaml

@@ -0,0 +1,5 @@
+self-hosted-runner:
+  labels:
+    - psi
+    - trusted-release
+    - container-release

.github/workflows/release.yaml

@@ -0,0 +1,59 @@
+name: Release Images
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "images/*/src"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-images
+  cancel-in-progress: false
+
+jobs:
+  release:
+    runs-on: [self-hosted, psi, trusted-release, container-release]
+    steps:
+      - name: Checkout trusted revision
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          submodules: true
+      - name: Select images
+        id: images
+        env:
+          BEFORE: ${{ github.event.before }}
+        run: |
+          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
+            images=$(nix develop -c python scripts/builder/changed.py --all)
+          else
+            images=$(nix develop -c python scripts/builder/changed.py "$BEFORE" "$GITHUB_SHA")
+          fi
+          {
+            echo "names<<EOF"
+            echo "$images"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+      - name: Log in to registry
+        if: steps.images.outputs.names != ''
+        env:
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+        run: |
+          printf '%s' "$REGISTRY_PASSWORD" | docker login registry.sjanglab.org             --username "$REGISTRY_USERNAME" --password-stdin
+      - name: Build and push images
+        if: steps.images.outputs.names != ''
+        env:
+          IMAGES: ${{ steps.images.outputs.names }}
+        run: |
+          while IFS= read -r image; do
+            [[ -n "$image" ]] || continue
+            nix develop -c python "images/$image/build.py" --push
+          done <<< "$IMAGES"
+      - name: Log out of registry
+        if: always() && steps.images.outputs.names != ''
+        run: docker logout registry.sjanglab.org

.github/workflows/update.yaml

@@ -0,0 +1,55 @@
+name: Update Images
+
+on:
+  schedule:
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-images
+  cancel-in-progress: false
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skip while update PR is open
+        id: existing
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          count=$(gh pr list --repo "$GITHUB_REPOSITORY" --state open             --search 'in:title "images: update upstream releases"'             --json number --jq length)
+          echo "skip=$([[ $count -gt 0 ]] && echo true || echo false)" >> "$GITHUB_OUTPUT"
+      - name: Checkout
+        if: steps.existing.outputs.skip != 'true'
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          submodules: true
+      - name: Install Nix
+        if: steps.existing.outputs.skip != 'true'
+        uses: NixOS/nix-installer-action@main
+      - name: Update images
+        if: steps.existing.outputs.skip != 'true'
+        run: nix develop -c python scripts/updater/run.py
+      - name: Create pull request
+        if: steps.existing.outputs.skip != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if git diff --quiet -- images; then
+            echo "No image updates found"
+            exit 0
+          fi
+          branch="automation/update-images-${GITHUB_RUN_ID}"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git switch -c "$branch"
+          git add images/*/src
+          git commit -m "images: update upstream releases"
+          git push origin "$branch"
+          gh pr create             --base main             --head "$branch"             --title "images: update upstream releases"             --body "Update image source submodules to latest stable SemVer releases."

.gitignore

@@ -1,3 +1,6 @@
 .direnv/
 result
 result-*
+
+__pycache__/
+*.py[cod]

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "images/alphafold3/src"]
+	path = images/alphafold3/src
+	url = https://github.com/google-deepmind/alphafold3.git

checks.nix

@@ -0,0 +1,52 @@
+{ pkgs }:
+let
+  source = pkgs.lib.cleanSource ./.;
+in
+{
+  python-lint =
+    pkgs.runCommand "containers-python-lint"
+      {
+        nativeBuildInputs = with pkgs; [
+          mypy
+          ruff
+        ];
+      }
+      ''
+        cp -r ${source} source
+        chmod -R u+w source
+        cd source
+        ruff check scripts images/*/*.py
+        mypy scripts images/*/*.py
+        touch $out
+      '';
+
+  python-tests =
+    pkgs.runCommand "containers-python-tests"
+      {
+        nativeBuildInputs = with pkgs; [
+          git
+          python312
+        ];
+      }
+      ''
+        cp -r ${source} source
+        chmod -R u+w source
+        cd source
+        export HOME=$TMPDIR
+        python -m unittest discover -s scripts -p '*_test.py'
+        touch $out
+      '';
+
+  workflows-lint =
+    pkgs.runCommand "containers-workflows-lint"
+      {
+        nativeBuildInputs = [ pkgs.actionlint ];
+      }
+      ''
+        cp -r ${source} source
+        chmod -R u+w source
+        cd source
+        actionlint -config-file .github/actionlint.yaml .github/workflows/*.yaml
+        touch $out
+      '';
+}

devshell.nix

@@ -0,0 +1,16 @@
+{ pkgs, formatter }:
+pkgs.mkShellNoCC {
+  packages = with pkgs; [
+    cosign
+    curl
+    docker-buildx
+    docker-client
+    gh
+    git
+    jq
+    python312
+    skopeo
+    syft
+    formatter
+  ];
+}

flake.nix

@@ -8,40 +8,67 @@
   };
 
   outputs =
-    {
+    inputs@{
       self,
       nixpkgs,
       treefmt-nix,
     }:
     let
-      system = "x86_64-linux";
-      pkgs = nixpkgs.legacyPackages.${system};
-      treefmtEval = treefmt-nix.lib.evalModule pkgs {
-        projectRootFile = "flake.nix";
-        programs = {
-          deadnix.enable = true;
-          keep-sorted.enable = true;
-          nixfmt.enable = true;
-          ruff-format.enable = true;
-          statix.enable = true;
-        };
+      inherit (nixpkgs) lib;
+
+      systems = [
+        "x86_64-linux"
+        "aarch64-linux"
+        "aarch64-darwin"
+      ];
+      eachSystem = lib.genAttrs systems;
+
+      flake = self // {
+        inherit inputs;
       };
+
+      pkgsFor = eachSystem (system: import nixpkgs { inherit system; });
+
+      scopes = eachSystem (
+        system:
+        let
+          pkgs = pkgsFor.${system};
+          treefmtEval = treefmt-nix.lib.evalModule pkgs {
+            projectRootFile = "flake.nix";
+            settings.global.excludes = [ "images/*/src/**" ];
+            programs = {
+              deadnix.enable = true;
+              keep-sorted.enable = true;
+              nixfmt.enable = true;
+              ruff-check.enable = true;
+              ruff-format.enable = true;
+              statix.enable = true;
+            };
+          };
+        in
+        lib.makeScope pkgs.newScope (self: {
+          inherit flake inputs system;
+          formatter = treefmtEval.config.build.wrapper;
+          formatting = treefmtEval.config.build.check flake;
+          devshell = self.callPackage ./devshell.nix { };
+          checks = self.callPackage ./checks.nix { };
+        })
+      );
     in
     {
-      checks.${system}.formatting = treefmtEval.config.build.check self;
-
-      devShells.${system}.default = pkgs.mkShell {
-        packages = with pkgs; [
-          cosign
-          docker-client
-          gh
-          git
-          jq
-          python312
-          syft
-        ];
-      };
+      checks = eachSystem (
+        system:
+        scopes.${system}.checks
+        // {
+          formatting = scopes.${system}.formatting;
+          devshell-default = scopes.${system}.devshell;
+        }
+      );
+
+      devShells = eachSystem (system: {
+        default = scopes.${system}.devshell;
+      });
 
-      formatter.${system} = treefmtEval.config.build.wrapper;
+      formatter = eachSystem (system: scopes.${system}.formatter);
     };
 }

images/alphafold3/build.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+"""Build AlphaFold 3 image from pinned upstream source."""
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parents[2]))
+
+from scripts.builder import build
+
+
+def main() -> None:
+    """Build AlphaFold 3 and optionally push its immutable version tag."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--push", action="store_true")
+    args = parser.parse_args()
+    result = build(
+        image_dir=Path(__file__).parent,
+        image="registry.sjanglab.org/sjanglab/alphafold3",
+        dockerfile="docker/Dockerfile",
+        push=args.push,
+    )
+    print(result.digest)
+
+
+if __name__ == "__main__":
+    main()

images/alphafold3/update.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+"""Update AlphaFold 3 source to latest stable upstream release."""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parents[2]))
+
+from scripts.updater import update
+
+if __name__ == "__main__":
+    result = update(
+        image_dir=Path(__file__).parent,
+        upstream="google-deepmind/alphafold3",
+    )
+    print(f"{result.old_revision} -> {result.new_revision} ({result.version})")

scripts/_git_tags.py

@@ -0,0 +1,35 @@
+"""Resolve lightweight and annotated remote Git tags to commits."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+def parse(lines: list[str]) -> dict[str, str]:
+    """Map tags to commits, preferring peeled annotated-tag revisions."""
+    direct: dict[str, str] = {}
+    peeled: dict[str, str] = {}
+    prefix = "refs/tags/"
+    for line in lines:
+        revision, ref = line.split("\t", 1)
+        if not ref.startswith(prefix):
+            continue
+        tag = ref[len(prefix) :]
+        if tag.endswith("^{}"):
+            peeled[tag[:-3]] = revision
+        else:
+            direct[tag] = revision
+    return direct | peeled
+
+
+def remote(remote: str, *, cwd: Path | None = None) -> dict[str, str]:
+    """List remote tags without changing local refs or working trees."""
+    lines = subprocess.run(
+        ["git", "ls-remote", "--tags", remote],
+        cwd=cwd,
+        check=True,
+        stdout=subprocess.PIPE,
+        text=True,
+    ).stdout.splitlines()
+    return parse(lines)

scripts/builder/__init__.py

@@ -0,0 +1,5 @@
+"""Shared image builder API."""
+
+from .build import BuildResult, build
+
+__all__ = ["BuildResult", "build"]