Publish signed image provenance manifests

.github/workflows/build-rapids-image.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2025-2026, NVIDIA CORPORATION.
+# Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 name: build-rapids
 
 on:
@@ -25,13 +25,20 @@ on:
       RAPIDS_VER:
         required: true
         type: string
+      RAPIDS_NOTEBOOKS_REF:
+        required: true
+        type: string
       BASE_TAG:
         required: true
         type: string
       NOTEBOOKS_TAG:
         required: true
         type: string
 
+permissions:
+  contents: read
+  id-token: write
+
 env:
   # prevent buildx from creating arch-specific manifest lists...
   # we want single images, that are later referenced together in a multiarch manifest
@@ -75,6 +82,18 @@ jobs:
         with:
           username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
           password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Install Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+        with:
+          go-version: '1.25.x'
+          cache: false
+      - name: Install ORAS
+        run: |
+          GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0
+          echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}"
+          "${RUNNER_TEMP}/oras-bin/oras" version
+      - name: Set up Cosign
+        uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3
       - name: Set up Docker Context for Buildx
         id: buildx-context
         run: |
@@ -99,8 +118,10 @@ jobs:
           LINUX_DISTRO_VER: ${{ inputs.LINUX_DISTRO_VER }}
           LINUX_VER: ${{ inputs.LINUX_VER }}
           PYTHON_VER: ${{ inputs.PYTHON_VER }}
+          RAPIDS_NOTEBOOKS_REF: ${{ inputs.RAPIDS_NOTEBOOKS_REF }}
           RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
       - name: Build base image
+        id: build-base
         uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
         with:
           context: context
@@ -113,7 +134,26 @@ jobs:
           tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
           # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types
           outputs: type=registry,oci-mediatypes=true
+      - name: Export base image package provenance
+        env:
+          DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
+          PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }}
+          PROVENANCE_TARGET: provenance-base
+        run: ci/export-image-provenance.sh
+      - name: Publish base image provenance
+        env:
+          CUDA_VER: ${{ inputs.CUDA_VER }}
+          DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
+          IMAGE_DIGEST: ${{ steps.build-base.outputs.digest }}
+          IMAGE_KIND: base
+          IMAGE_PLATFORM: linux/${{ matrix.ARCH }}
+          IMAGE_REFERENCE: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
+          PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }}
+          PYTHON_VER: ${{ inputs.PYTHON_VER }}
+          RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
+        run: ci/publish-image-provenance.sh
       - name: Build notebooks image
+        id: build-notebooks
         uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
         with:
           context: context
@@ -126,3 +166,21 @@ jobs:
           tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
           # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types
           outputs: type=registry,oci-mediatypes=true
+      - name: Export notebooks image package provenance
+        env:
+          DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
+          PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }}
+          PROVENANCE_TARGET: provenance-notebooks
+        run: ci/export-image-provenance.sh
+      - name: Publish notebooks image provenance
+        env:
+          CUDA_VER: ${{ inputs.CUDA_VER }}
+          DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
+          IMAGE_DIGEST: ${{ steps.build-notebooks.outputs.digest }}
+          IMAGE_KIND: notebooks
+          IMAGE_PLATFORM: linux/${{ matrix.ARCH }}
+          IMAGE_REFERENCE: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
+          PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }}
+          PYTHON_VER: ${{ inputs.PYTHON_VER }}
+          RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
+        run: ci/publish-image-provenance.sh

.github/workflows/build-test-publish-images.yml

@@ -55,6 +55,10 @@ jobs:
       - name: Run hadolint
         run: |
           ci/lint-dockerfiles.sh
+      - name: Test image provenance manifest generator
+        run: |
+          python3 -m pip install --disable-pip-version-check -r requirements-test.txt
+          python3 -m pytest -q tests
   compute-matrix:
     runs-on: ubuntu-latest
     outputs:
@@ -68,6 +72,7 @@ jobs:
       NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }}
       CUVS_BENCH_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_TAG_PREFIX }}
       CUVS_BENCH_CPU_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_CPU_TAG_PREFIX }}
+      RAPIDS_NOTEBOOKS_REF: ${{ steps.compute-rapids-ver.outputs.RAPIDS_NOTEBOOKS_REF }}
       RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }}
       ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }}
     steps:
@@ -78,7 +83,15 @@ jobs:
           persist-credentials: false
       - name: Compute matrix
         id: compute-matrix
+        env:
+          BUILD_TYPE: ${{ inputs.build_type }}
         run: |
+          # PR builds temporarily use published 26.08 packages. Their
+          # aarch64 RAPIDS metapackage requires Python >=3.12, but still
+          # supports both CUDA 12 and CUDA 13 package lines.
+          if [[ $BUILD_TYPE == "pull-request" ]]; then
+            export MATRIX_PYTHON_VERSIONS='["3.12", "3.13", "3.14"]'
+          fi
           MATRIX=$(ci/compute-matrix.sh)
           echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT}
       - name: Compute tag prefix
@@ -119,18 +132,7 @@ jobs:
           echo "CUVS_BENCH_CPU_IMAGE_REPO=${cuvs_bench_cpu_repo}" | tee -a ${GITHUB_OUTPUT}
       - name: Compute RAPIDS_VER
         id: compute-rapids-ver
-        run: |
-          GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)"
-          GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v'
-          ALPHA_TAG=""
-          if [[ $GIT_DESCRIBE_TAG =~ [a-z] ]]; then
-            echo "Most recent tag is an alpha tag"
-            ALPHA_TAG="a"
-          fi
-          RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM
-
-          echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT}
-          echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT}
+        run: ci/compute-rapids-version.sh
       - name: Compute test matrix
         id: compute-test-matrix
         env:
@@ -159,6 +161,7 @@ jobs:
       LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }}
       LINUX_VER: ${{ matrix.LINUX_VER }}
       PYTHON_VER: ${{ matrix.PYTHON_VER }}
+      RAPIDS_NOTEBOOKS_REF: ${{ needs.compute-matrix.outputs.RAPIDS_NOTEBOOKS_REF }}
       RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
       BASE_TAG:
         "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
@@ -224,6 +227,18 @@ jobs:
         with:
           username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
           password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Install Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+        with:
+          go-version: '1.25.x'
+          cache: false
+      - name: Install ORAS
+        run: |
+          GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0
+          echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}"
+          "${RUNNER_TEMP}/oras-bin/oras" version
+      - name: Set up Cosign
+        uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3
       - name: Create multiarch manifest
         shell: bash
         env:
@@ -239,6 +254,38 @@ jobs:
           GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
           ARCHES: ${{ toJSON(matrix.ARCHES) }}
         run: ci/create-rapids-multiarch-manifest.sh
+      - name: Publish multiarch image provenance indexes
+        shell: bash
+        env:
+          BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}
+          BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
+          RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+          ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }}
+          CUDA_TAG: ${{ matrix.CUDA_TAG }}
+          PYTHON_VER: ${{ matrix.PYTHON_VER }}
+          NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}
+          NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
+          ARCHES: ${{ toJSON(matrix.ARCHES) }}
+        run: |
+          base_reference="rapidsai/${BASE_IMAGE_REPO}:${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
+          notebooks_reference="rapidsai/${NOTEBOOKS_IMAGE_REPO}:${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
+          for image_kind in base notebooks; do
+            if [[ "$image_kind" == base ]]; then
+              image_reference="$base_reference"
+            else
+              image_reference="$notebooks_reference"
+            fi
+            platform_references="$({
+              while IFS= read -r arch; do
+                printf 'linux/%s=%s-%s\n' "$arch" "$image_reference" "$arch"
+              done < <(jq -r '.[]' <<< "$ARCHES")
+            })"
+            IMAGE_KIND="$image_kind" \
+              IMAGE_REFERENCE="$image_reference" \
+              PLATFORM_REFERENCES="$platform_references" \
+              CUDA_VER="$CUDA_TAG" \
+              ci/publish-image-provenance-index.sh
+          done
   build-cuvs-multiarch-manifest:
     needs: [build-cuvs, compute-matrix]
     strategy:

.gitignore

@@ -1,4 +1,6 @@
 .vscode
+__pycache__/
+*.py[cod]
 
 *.a
 *.bz2

CONTRIBUTING.md

@@ -63,3 +63,24 @@ See https://github.com/rapidsai/workflows/blob/main/.github/workflows/cleanup_st
 
 If you come back to a pull request here after more than a few days and find that jobs are failing with errors
 that suggest that some necessary images don't exist, re-run all of CI on that pull request to produce new images.
+
+## Preparing a release branch
+
+Run the version update with the release context when preparing `release/YY.MM`:
+
+```sh
+bash ci/release/update-version.sh YY.MM.00 --run-context=release
+```
+
+This updates the Dockerfile's `RAPIDS_NOTEBOOKS_REF` default to `release/YY.MM`.
+Branch CI also derives the notebook ref directly from `GITHUB_REF_NAME`, so alpha-tagged
+builds on `release/YY.MM` clone notebook inputs from the matching `cudf`, `cuml`, and
+`cugraph` release branches instead of `main`. Those release branches must exist before
+the first Docker build runs; no separate notebook-ref edit is otherwise required during
+the branch cut.
+
+Pull requests targeting `main` use the current version derived from the repository tag
+and clone notebook inputs from `main`. Pull requests targeting `release/YY.MM` use that
+target branch for both the package version and notebook inputs. This keeps package and
+source lines matched without hardcoded version pins or manual updates when the next
+release branch is created.

Dockerfile

@@ -14,7 +14,7 @@ FROM python:${PYTHON_VER} AS dependencies
 ARG CPU_ARCH=notset
 ARG CUDA_VER=notset
 ARG PYTHON_VER=notset
-ARG RAPIDS_BRANCH="main"
+ARG RAPIDS_NOTEBOOKS_REF="main"
 ARG RAPIDS_VER=26.10
 ARG YQ_VER=notset
 
@@ -258,6 +258,7 @@ LABEL com.nvidia.workbench.os="linux"
 LABEL com.nvidia.workbench.package-manager-environment.target="/opt/conda"
 LABEL com.nvidia.workbench.package-manager-environment.type="conda"
 LABEL com.nvidia.workbench.package-manager.apt.binary="/usr/bin/apt"
+
 LABEL com.nvidia.workbench.package-manager.apt.installed-packages=""
 LABEL com.nvidia.workbench.package-manager.conda3.binary="/opt/conda/bin/conda"
 LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cucim xgboost jupyterlab"
@@ -268,3 +269,25 @@ LABEL com.nvidia.workbench.schema-version="v2"
 LABEL com.nvidia.workbench.user.gid="1000"
 LABEL com.nvidia.workbench.user.uid="1001"
 LABEL com.nvidia.workbench.user.username="rapids"
+
+# Minimal BuildKit export targets used to publish image provenance without
+# exporting or pulling the image filesystem. Each contains only conda package
+# metadata copied from the corresponding final image target.
+FROM base AS provenance-base-inventory
+COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory
+RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json
+
+FROM scratch AS provenance-base
+COPY --from=base /opt/conda/conda-meta /conda-meta
+COPY --from=provenance-base-inventory /tmp/pip-packages.json /pip-packages.json
+
+FROM notebooks AS provenance-notebooks-inventory
+COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory
+RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json
+
+FROM scratch AS provenance-notebooks
+COPY --from=notebooks /opt/conda/conda-meta /conda-meta
+COPY --from=provenance-notebooks-inventory /tmp/pip-packages.json /pip-packages.json
+
+# Keep the default Docker build target runnable for local users.
+FROM notebooks

README.md

@@ -39,6 +39,29 @@ The following environment variables can be passed to the `docker run` commands f
 - `PIP_TIMEOUT` - how long (in seconds) the `pip` install should wait before exiting
 - `UNQUOTE` - Whether the command line args to `docker run` should be [executed with or without being quoted](./context/entrypoint.sh). Default to false and it is unlikely that you need to change this.
 
+## Image Provenance Manifests
+
+Published `base` and `notebooks` images have an OCI referrer containing a
+RAPIDS image provenance manifest. It is attached to the immutable image digest,
+not to a mutable tag, and can be retrieved from the registry without downloading
+image layers. The platform-specific manifest records the source commit and
+workflow run plus the exact conda package name, version, build, channel, and
+source URL taken from `conda-meta`. The multiarch image-index manifest links the
+corresponding platform manifests.
+
+Conda package records intentionally leave `purls` empty until an upstream pURL
+mapping has been verified. Conda is a distribution format, not an upstream pURL
+type, so consumers must not derive `pkg:conda/...` identifiers from these
+records. The manifest uses the OCI artifact type
+`application/vnd.rapids.image.provenance.v1+json` for platform images and
+`application/vnd.rapids.image.provenance.index.v1+json` for multiarch indexes.
+
+The build publishes these records with [ORAS](https://oras.land/), using an OCI
+referrer to the image digest, and keylessly signs the attached artifact with
+GitHub Actions OIDC. Scanner and triage tooling can discover the referrer,
+retrieve its JSON, and compare the scanner-reported component version with the
+build facts without pulling the image.
+
 ## Bind Mounts
 
 Mounting files/folders to the locations specified below provide additional functionality for the images.

ci/common.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2023-2025, NVIDIA CORPORATION.
+# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 set -eEuo pipefail
 
@@ -17,14 +17,43 @@ export HUB_TOKEN
 check_tag_exists() {
     local repo="$1"
     local tag="$2"
-    local exists
-    exists=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: JWT $HUB_TOKEN" \
-        "https://hub.docker.com/v2/repositories/${org}/${repo}/tags/${tag}/")
-
-    if [ "$exists" -ne 200 ]; then
-        echo "Error: Required image tag ${repo}:${tag} does not exist. This implies that the image was not built successfully in the build job."
-        exit 1
-    fi
+    local attempts=6
+    local attempt=1
+    local delay=5
+    local http_code
+
+    while ((attempt <= attempts)); do
+        http_code=$(curl -sS -o /dev/null -w "%{http_code}" -H "Authorization: JWT $HUB_TOKEN" \
+            "https://hub.docker.com/v2/repositories/${org}/${repo}/tags/${tag}/") || http_code="000"
+
+        if [[ $http_code == "200" ]]; then
+            return 0
+        fi
+
+        if ((attempt == attempts)); then
+            break
+        fi
+
+        case "$http_code" in
+            000 | 404 | 429 | 5??)
+                echo "Required image tag ${repo}:${tag} is not visible yet (HTTP ${http_code}); retrying in ${delay}s (${attempt}/${attempts})."
+                sleep "$delay"
+                delay=$((delay * 2))
+                if ((delay > 60)); then
+                    delay=60
+                fi
+                ;;
+            *)
+                echo "Error: Failed to check required image tag ${repo}:${tag} (HTTP ${http_code})."
+                return 1
+                ;;
+        esac
+
+        attempt=$((attempt + 1))
+    done
+
+    echo "Error: Required image tag ${repo}:${tag} was not visible after ${attempts} attempts (last HTTP ${http_code}). The image build may have failed, or Docker Hub may not have propagated the tag yet."
+    return 1
 }
 
 export org="rapidsai"