From 0fec87b76b07162b944158c4822a6b64ace1f4f1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 3 Aug 2026 09:59:21 -0500 Subject: [PATCH] Publish signed image provenance manifests --- .github/workflows/build-rapids-image.yml | 60 +++- .../workflows/build-test-publish-images.yml | 71 ++++- .gitignore | 2 + CONTRIBUTING.md | 21 ++ Dockerfile | 25 +- README.md | 23 ++ ci/common.sh | 47 +++- ci/compute-build-args.sh | 3 +- ci/compute-matrix.sh | 14 +- ci/compute-rapids-version.sh | 35 +++ ci/export-image-provenance.sh | 27 ++ ci/image_provenance_manifest.py | 264 ++++++++++++++++++ ci/publish-image-provenance-index.sh | 67 +++++ ci/publish-image-provenance.sh | 63 +++++ ci/release/update-version.sh | 12 +- context/.dockerignore | 1 + context/notebooks.sh | 6 +- context/scripts/export-pip-package-inventory | 42 +++ matrix-test.yaml | 4 +- requirements-test.txt | 1 + tests/test_common.py | 90 ++++++ tests/test_image_provenance_manifest.py | 174 ++++++++++++ tests/test_workflow_configuration.py | 119 ++++++++ 23 files changed, 1134 insertions(+), 37 deletions(-) create mode 100755 ci/compute-rapids-version.sh create mode 100755 ci/export-image-provenance.sh create mode 100755 ci/image_provenance_manifest.py create mode 100755 ci/publish-image-provenance-index.sh create mode 100755 ci/publish-image-provenance.sh create mode 100644 context/scripts/export-pip-package-inventory create mode 100644 requirements-test.txt create mode 100644 tests/test_common.py create mode 100644 tests/test_image_provenance_manifest.py create mode 100644 tests/test_workflow_configuration.py diff --git a/.github/workflows/build-rapids-image.yml b/.github/workflows/build-rapids-image.yml index 02947e37..af03aad9 100644 --- a/.github/workflows/build-rapids-image.yml +++ b/.github/workflows/build-rapids-image.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2025-2026, NVIDIA CORPORATION. +# Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. name: build-rapids on: @@ -25,6 +25,9 @@ on: RAPIDS_VER: required: true type: string + RAPIDS_NOTEBOOKS_REF: + required: true + type: string BASE_TAG: required: true type: string @@ -32,6 +35,10 @@ on: required: true type: string +permissions: + contents: read + id-token: write + env: # prevent buildx from creating arch-specific manifest lists... # we want single images, that are later referenced together in a multiarch manifest @@ -75,6 +82,18 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: '1.25.x' + cache: false + - name: Install ORAS + run: | + GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 + echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" + "${RUNNER_TEMP}/oras-bin/oras" version + - name: Set up Cosign + uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Set up Docker Context for Buildx id: buildx-context run: | @@ -99,8 +118,10 @@ jobs: LINUX_DISTRO_VER: ${{ inputs.LINUX_DISTRO_VER }} LINUX_VER: ${{ inputs.LINUX_VER }} PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_NOTEBOOKS_REF: ${{ inputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - name: Build base image + id: build-base uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -113,7 +134,26 @@ jobs: tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true + - name: Export base image package provenance + env: + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} + PROVENANCE_TARGET: provenance-base + run: ci/export-image-provenance.sh + - name: Publish base image provenance + env: + CUDA_VER: ${{ inputs.CUDA_VER }} + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + IMAGE_DIGEST: ${{ steps.build-base.outputs.digest }} + IMAGE_KIND: base + IMAGE_PLATFORM: linux/${{ matrix.ARCH }} + IMAGE_REFERENCE: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} + run: ci/publish-image-provenance.sh - name: Build notebooks image + id: build-notebooks uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -126,3 +166,21 @@ jobs: tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true + - name: Export notebooks image package provenance + env: + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} + PROVENANCE_TARGET: provenance-notebooks + run: ci/export-image-provenance.sh + - name: Publish notebooks image provenance + env: + CUDA_VER: ${{ inputs.CUDA_VER }} + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + IMAGE_DIGEST: ${{ steps.build-notebooks.outputs.digest }} + IMAGE_KIND: notebooks + IMAGE_PLATFORM: linux/${{ matrix.ARCH }} + IMAGE_REFERENCE: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} + run: ci/publish-image-provenance.sh diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml index 1d3bf902..23ca3801 100644 --- a/.github/workflows/build-test-publish-images.yml +++ b/.github/workflows/build-test-publish-images.yml @@ -55,6 +55,10 @@ jobs: - name: Run hadolint run: | ci/lint-dockerfiles.sh + - name: Test image provenance manifest generator + run: | + python3 -m pip install --disable-pip-version-check -r requirements-test.txt + python3 -m pytest -q tests compute-matrix: runs-on: ubuntu-latest outputs: @@ -68,6 +72,7 @@ jobs: NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }} CUVS_BENCH_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_TAG_PREFIX }} CUVS_BENCH_CPU_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_CPU_TAG_PREFIX }} + RAPIDS_NOTEBOOKS_REF: ${{ steps.compute-rapids-ver.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }} ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }} steps: @@ -78,7 +83,15 @@ jobs: persist-credentials: false - name: Compute matrix id: compute-matrix + env: + BUILD_TYPE: ${{ inputs.build_type }} run: | + # PR builds temporarily use published 26.08 packages. Their + # aarch64 RAPIDS metapackage requires Python >=3.12, but still + # supports both CUDA 12 and CUDA 13 package lines. + if [[ $BUILD_TYPE == "pull-request" ]]; then + export MATRIX_PYTHON_VERSIONS='["3.12", "3.13", "3.14"]' + fi MATRIX=$(ci/compute-matrix.sh) echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT} - name: Compute tag prefix @@ -119,18 +132,7 @@ jobs: echo "CUVS_BENCH_CPU_IMAGE_REPO=${cuvs_bench_cpu_repo}" | tee -a ${GITHUB_OUTPUT} - name: Compute RAPIDS_VER id: compute-rapids-ver - run: | - GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)" - GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v' - ALPHA_TAG="" - if [[ $GIT_DESCRIBE_TAG =~ [a-z] ]]; then - echo "Most recent tag is an alpha tag" - ALPHA_TAG="a" - fi - RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM - - echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT} - echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT} + run: ci/compute-rapids-version.sh - name: Compute test matrix id: compute-test-matrix env: @@ -159,6 +161,7 @@ jobs: LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }} LINUX_VER: ${{ matrix.LINUX_VER }} PYTHON_VER: ${{ matrix.PYTHON_VER }} + RAPIDS_NOTEBOOKS_REF: ${{ needs.compute-matrix.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} BASE_TAG: "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\ @@ -224,6 +227,18 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: '1.25.x' + cache: false + - name: Install ORAS + run: | + GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 + echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" + "${RUNNER_TEMP}/oras-bin/oras" version + - name: Set up Cosign + uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Create multiarch manifest shell: bash env: @@ -239,6 +254,38 @@ jobs: GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} ARCHES: ${{ toJSON(matrix.ARCHES) }} run: ci/create-rapids-multiarch-manifest.sh + - name: Publish multiarch image provenance indexes + shell: bash + env: + BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }} + BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }} + RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} + ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }} + CUDA_TAG: ${{ matrix.CUDA_TAG }} + PYTHON_VER: ${{ matrix.PYTHON_VER }} + NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }} + NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }} + ARCHES: ${{ toJSON(matrix.ARCHES) }} + run: | + base_reference="rapidsai/${BASE_IMAGE_REPO}:${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" + notebooks_reference="rapidsai/${NOTEBOOKS_IMAGE_REPO}:${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" + for image_kind in base notebooks; do + if [[ "$image_kind" == base ]]; then + image_reference="$base_reference" + else + image_reference="$notebooks_reference" + fi + platform_references="$({ + while IFS= read -r arch; do + printf 'linux/%s=%s-%s\n' "$arch" "$image_reference" "$arch" + done < <(jq -r '.[]' <<< "$ARCHES") + })" + IMAGE_KIND="$image_kind" \ + IMAGE_REFERENCE="$image_reference" \ + PLATFORM_REFERENCES="$platform_references" \ + CUDA_VER="$CUDA_TAG" \ + ci/publish-image-provenance-index.sh + done build-cuvs-multiarch-manifest: needs: [build-cuvs, compute-matrix] strategy: diff --git a/.gitignore b/.gitignore index f99d452a..2d5c5c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .vscode +__pycache__/ +*.py[cod] *.a *.bz2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9aae81a3..97f3d4a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,3 +63,24 @@ See https://github.com/rapidsai/workflows/blob/main/.github/workflows/cleanup_st If you come back to a pull request here after more than a few days and find that jobs are failing with errors that suggest that some necessary images don't exist, re-run all of CI on that pull request to produce new images. + +## Preparing a release branch + +Run the version update with the release context when preparing `release/YY.MM`: + +```sh +bash ci/release/update-version.sh YY.MM.00 --run-context=release +``` + +This updates the Dockerfile's `RAPIDS_NOTEBOOKS_REF` default to `release/YY.MM`. +Branch CI also derives the notebook ref directly from `GITHUB_REF_NAME`, so alpha-tagged +builds on `release/YY.MM` clone notebook inputs from the matching `cudf`, `cuml`, and +`cugraph` release branches instead of `main`. Those release branches must exist before +the first Docker build runs; no separate notebook-ref edit is otherwise required during +the branch cut. + +Pull requests targeting `main` use the current version derived from the repository tag +and clone notebook inputs from `main`. Pull requests targeting `release/YY.MM` use that +target branch for both the package version and notebook inputs. This keeps package and +source lines matched without hardcoded version pins or manual updates when the next +release branch is created. diff --git a/Dockerfile b/Dockerfile index 756b38ad..5a7338d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ FROM python:${PYTHON_VER} AS dependencies ARG CPU_ARCH=notset ARG CUDA_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_BRANCH="main" +ARG RAPIDS_NOTEBOOKS_REF="main" ARG RAPIDS_VER=26.10 ARG YQ_VER=notset @@ -258,6 +258,7 @@ LABEL com.nvidia.workbench.os="linux" LABEL com.nvidia.workbench.package-manager-environment.target="/opt/conda" LABEL com.nvidia.workbench.package-manager-environment.type="conda" LABEL com.nvidia.workbench.package-manager.apt.binary="/usr/bin/apt" + LABEL com.nvidia.workbench.package-manager.apt.installed-packages="" LABEL com.nvidia.workbench.package-manager.conda3.binary="/opt/conda/bin/conda" LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cucim xgboost jupyterlab" @@ -268,3 +269,25 @@ LABEL com.nvidia.workbench.schema-version="v2" LABEL com.nvidia.workbench.user.gid="1000" LABEL com.nvidia.workbench.user.uid="1001" LABEL com.nvidia.workbench.user.username="rapids" + +# Minimal BuildKit export targets used to publish image provenance without +# exporting or pulling the image filesystem. Each contains only conda package +# metadata copied from the corresponding final image target. +FROM base AS provenance-base-inventory +COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory +RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json + +FROM scratch AS provenance-base +COPY --from=base /opt/conda/conda-meta /conda-meta +COPY --from=provenance-base-inventory /tmp/pip-packages.json /pip-packages.json + +FROM notebooks AS provenance-notebooks-inventory +COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory +RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json + +FROM scratch AS provenance-notebooks +COPY --from=notebooks /opt/conda/conda-meta /conda-meta +COPY --from=provenance-notebooks-inventory /tmp/pip-packages.json /pip-packages.json + +# Keep the default Docker build target runnable for local users. +FROM notebooks diff --git a/README.md b/README.md index d074ef1e..1f99d314 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,29 @@ The following environment variables can be passed to the `docker run` commands f - `PIP_TIMEOUT` - how long (in seconds) the `pip` install should wait before exiting - `UNQUOTE` - Whether the command line args to `docker run` should be [executed with or without being quoted](./context/entrypoint.sh). Default to false and it is unlikely that you need to change this. +## Image Provenance Manifests + +Published `base` and `notebooks` images have an OCI referrer containing a +RAPIDS image provenance manifest. It is attached to the immutable image digest, +not to a mutable tag, and can be retrieved from the registry without downloading +image layers. The platform-specific manifest records the source commit and +workflow run plus the exact conda package name, version, build, channel, and +source URL taken from `conda-meta`. The multiarch image-index manifest links the +corresponding platform manifests. + +Conda package records intentionally leave `purls` empty until an upstream pURL +mapping has been verified. Conda is a distribution format, not an upstream pURL +type, so consumers must not derive `pkg:conda/...` identifiers from these +records. The manifest uses the OCI artifact type +`application/vnd.rapids.image.provenance.v1+json` for platform images and +`application/vnd.rapids.image.provenance.index.v1+json` for multiarch indexes. + +The build publishes these records with [ORAS](https://oras.land/), using an OCI +referrer to the image digest, and keylessly signs the attached artifact with +GitHub Actions OIDC. Scanner and triage tooling can discover the referrer, +retrieve its JSON, and compare the scanner-reported component version with the +build facts without pulling the image. + ## Bind Mounts Mounting files/folders to the locations specified below provide additional functionality for the images. diff --git a/ci/common.sh b/ci/common.sh index d630c7b1..7ac27010 100644 --- a/ci/common.sh +++ b/ci/common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. set -eEuo pipefail @@ -17,14 +17,43 @@ export HUB_TOKEN check_tag_exists() { local repo="$1" local tag="$2" - local exists - exists=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: JWT $HUB_TOKEN" \ - "https://hub.docker.com/v2/repositories/${org}/${repo}/tags/${tag}/") - - if [ "$exists" -ne 200 ]; then - echo "Error: Required image tag ${repo}:${tag} does not exist. This implies that the image was not built successfully in the build job." - exit 1 - fi + local attempts=6 + local attempt=1 + local delay=5 + local http_code + + while ((attempt <= attempts)); do + http_code=$(curl -sS -o /dev/null -w "%{http_code}" -H "Authorization: JWT $HUB_TOKEN" \ + "https://hub.docker.com/v2/repositories/${org}/${repo}/tags/${tag}/") || http_code="000" + + if [[ $http_code == "200" ]]; then + return 0 + fi + + if ((attempt == attempts)); then + break + fi + + case "$http_code" in + 000 | 404 | 429 | 5??) + echo "Required image tag ${repo}:${tag} is not visible yet (HTTP ${http_code}); retrying in ${delay}s (${attempt}/${attempts})." + sleep "$delay" + delay=$((delay * 2)) + if ((delay > 60)); then + delay=60 + fi + ;; + *) + echo "Error: Failed to check required image tag ${repo}:${tag} (HTTP ${http_code})." + return 1 + ;; + esac + + attempt=$((attempt + 1)) + done + + echo "Error: Required image tag ${repo}:${tag} was not visible after ${attempts} attempts (last HTTP ${http_code}). The image build may have failed, or Docker Hub may not have propagated the tag yet." + return 1 } export org="rapidsai" diff --git a/ci/compute-build-args.sh b/ci/compute-build-args.sh index 38ffb60c..fd52fb55 100755 --- a/ci/compute-build-args.sh +++ b/ci/compute-build-args.sh @@ -12,6 +12,7 @@ LINUX_DISTRO: ${LINUX_DISTRO:-notset} LINUX_DISTRO_VER: ${LINUX_DISTRO_VER:-notset} LINUX_VER: ${LINUX_VER} PYTHON_VER: ${PYTHON_VER} +RAPIDS_NOTEBOOKS_REF: ${RAPIDS_NOTEBOOKS_REF:-main} RAPIDS_VER: ${RAPIDS_VER} " export ARGS @@ -30,7 +31,7 @@ if [ -n "${GITHUB_ACTIONS:-}" ]; then # cat < "${GITHUB_OUTPUT:-/dev/stdout}" DOCKER_BUILD_ARGS< str: + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +def parse_build_args(values: list[str]) -> dict[str, str]: + """Convert repeated ``NAME=VALUE`` CLI options to a stable mapping.""" + build_args: dict[str, str] = {} + for value in values: + name, separator, argument = value.partition("=") + if not separator or not name: + raise ValueError(f"build argument must use NAME=VALUE syntax: {value!r}") + build_args[name] = argument + return dict(sorted(build_args.items())) + + +def conda_packages(metadata_dir: Path | None) -> list[dict[str, Any]]: + """Read exact installed-package facts from conda-meta records.""" + if metadata_dir is None: + return [] + packages: list[dict[str, Any]] = [] + for path in sorted(metadata_dir.glob("*.json")): + try: + record = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid conda metadata JSON: {path}") from exc + if not isinstance(record, dict): + raise ValueError(f"conda metadata must be an object: {path}") + name = str(record.get("name") or "").strip() + version = str(record.get("version") or "").strip() + build = str(record.get("build") or "").strip() + if not name or not version: + raise ValueError(f"conda metadata lacks name or version: {path}") + packages.append( + { + "name": name, + "version": version, + "build": build, + "build_number": record.get("build_number"), + "channel": str(record.get("channel") or record.get("url") or ""), + "url": str(record.get("url") or ""), + # Conda is a distribution format, not an upstream pURL type. + # A pURL is populated only once a vetted package mapping exists. + "purls": [], + "purl_source": "unmapped", + } + ) + return sorted( + packages, + key=lambda package: ( + package["name"], + package["version"], + package["build"], + package["channel"], + ), + ) + + +def installed_packages( + conda_package_records: list[dict[str, Any]], + pip_packages_path: Path | None, +) -> list[dict[str, Any]]: + """Combine package-manager inventories without inventing upstream identities.""" + packages = [ + { + **package, + "ecosystem": "conda", + "installer": "conda", + "evidence": "conda-meta", + } + for package in conda_package_records + ] + if pip_packages_path is not None: + try: + pip_records = json.loads(pip_packages_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid pip package inventory: {pip_packages_path}") from exc + if not isinstance(pip_records, list): + raise ValueError("pip package inventory must be a JSON list") + for record in pip_records: + if not isinstance(record, dict): + raise ValueError("pip package inventory entries must be JSON objects") + name = str(record.get("name") or "").strip() + version = str(record.get("version") or "").strip() + purls = record.get("purls") + if not name or not version or not isinstance(purls, list): + raise ValueError("pip package inventory entries require name, version, and purls") + packages.append( + { + "name": name, + "version": version, + "purls": [str(purl) for purl in purls if str(purl).startswith("pkg:")], + "ecosystem": "pypi", + "installer": str(record.get("installer") or "pip"), + "evidence": str(record.get("evidence") or "python-distribution-metadata"), + } + ) + return sorted( + packages, + key=lambda package: ( + str(package["ecosystem"]), + str(package["name"]), + str(package["version"]), + ), + ) + + +def parse_platform_manifest(value: str) -> dict[str, str]: + """Parse ``os/architecture|reference|digest`` into a manifest entry.""" + platform, separator, remainder = value.partition("|") + reference, separator2, digest = remainder.partition("|") + _os_name, separator3, _architecture = platform.partition("/") + if not separator or not separator2 or not separator3: + raise ValueError( + "platform manifests must use os/architecture|reference|sha256:digest" + ) + if not digest.startswith("sha256:"): + raise ValueError(f"platform manifest digest must be sha256: {digest!r}") + return { + "platform": platform, + "reference": reference, + "digest": digest, + } + + +def build_manifest( + context: ManifestContext, + *, + metadata_dir: Path | None = None, + pip_packages_path: Path | None = None, + platform_manifests: list[str] | None = None, +) -> dict[str, Any]: + """Construct the schema payload for a platform image or multiarch index.""" + if not context.image_digest.startswith("sha256:"): + raise ValueError(f"image digest must be sha256: {context.image_digest!r}") + packages = conda_packages(metadata_dir) + package_payload = json.dumps(packages, sort_keys=True, separators=(",", ":")).encode() + all_packages = installed_packages(packages, pip_packages_path) + all_package_payload = json.dumps( + all_packages, sort_keys=True, separators=(",", ":") + ).encode() + subject: dict[str, Any] = { + "reference": context.image_reference, + "digest": context.image_digest, + } + if context.platform != "multiarch": + os_name, separator, architecture = context.platform.partition("/") + if not separator: + raise ValueError("platform must be os/architecture or multiarch") + subject["platform"] = {"os": os_name, "architecture": architecture} + manifest: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "generated_at": datetime.now(timezone.utc).isoformat(), + "subject": subject, + "image": { + "kind": context.image_kind, + "rapids_version": context.rapids_version, + "cuda_version": context.cuda_version, + "python_version": context.python_version, + }, + "build": { + "source_repository": context.source_repository, + "source_commit": context.source_commit, + "workflow_ref": context.workflow_ref, + "workflow_run_url": context.workflow_run_url, + "build_args": parse_build_args(context.build_args), + }, + "conda_packages": packages, + "conda_packages_sha256": _sha256(package_payload), + "installed_packages": all_packages, + "installed_packages_sha256": _sha256(all_package_payload), + "platform_manifests": sorted( + (parse_platform_manifest(value) for value in platform_manifests or []), + key=lambda item: item["platform"], + ), + } + return manifest + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--image-reference", required=True) + parser.add_argument("--image-digest", required=True) + parser.add_argument("--image-kind", required=True) + parser.add_argument("--rapids-version", required=True) + parser.add_argument("--cuda-version", required=True) + parser.add_argument("--python-version", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--source-repository", required=True) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--workflow-ref", required=True) + parser.add_argument("--workflow-run-url", required=True) + parser.add_argument("--conda-meta-dir", type=Path) + parser.add_argument("--pip-packages", type=Path) + parser.add_argument("--build-arg", action="append", default=[]) + parser.add_argument("--platform-manifest", action="append", default=[]) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + manifest = build_manifest( + ManifestContext( + image_reference=args.image_reference, + image_digest=args.image_digest, + image_kind=args.image_kind, + rapids_version=args.rapids_version, + cuda_version=args.cuda_version, + python_version=args.python_version, + platform=args.platform, + source_repository=args.source_repository, + source_commit=args.source_commit, + workflow_ref=args.workflow_ref, + workflow_run_url=args.workflow_run_url, + build_args=args.build_arg, + ), + metadata_dir=args.conda_meta_dir, + pip_packages_path=args.pip_packages, + platform_manifests=args.platform_manifest, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/ci/publish-image-provenance-index.sh b/ci/publish-image-provenance-index.sh new file mode 100755 index 00000000..64032802 --- /dev/null +++ b/ci/publish-image-provenance-index.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eEuo pipefail + +: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the multiarch image tag}" +: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" +: "${RAPIDS_VER:?Set RAPIDS_VER}" +: "${CUDA_VER:?Set CUDA_VER}" +: "${PYTHON_VER:?Set PYTHON_VER}" +: "${PLATFORM_REFERENCES:?Set PLATFORM_REFERENCES as platform=tag lines}" + +normalize_registry_reference() { + local reference="$1" + local first_component="${reference%%/*}" + if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then + reference="docker.io/${reference}" + fi + printf '%s\n' "$reference" +} + +output_dir="${RUNNER_TEMP:-/tmp}/image-provenance-index-${IMAGE_KIND}" +manifest_path="$output_dir/image-provenance-index.json" +workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" +registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" +mkdir -p "$output_dir" + +image_digest="$(oras manifest fetch --descriptor "$registry_reference" | jq -r '.digest')" +platform_args=() +while IFS='=' read -r platform reference; do + [[ -z "$platform" ]] && continue + platform_reference="$(normalize_registry_reference "$reference")" + digest="$(oras manifest fetch --descriptor "$platform_reference" | jq -r '.digest')" + platform_args+=(--platform-manifest "${platform}|${platform_reference}|${digest}") +done <<<"$PLATFORM_REFERENCES" + +python3 ci/image_provenance_manifest.py \ + --output "$manifest_path" \ + --image-reference "$IMAGE_REFERENCE" \ + --image-digest "$image_digest" \ + --image-kind "$IMAGE_KIND" \ + --rapids-version "$RAPIDS_VER" \ + --cuda-version "$CUDA_VER" \ + --python-version "$PYTHON_VER" \ + --platform multiarch \ + --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --source-commit "$GITHUB_SHA" \ + --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ + --workflow-run-url "$workflow_run_url" \ + --build-arg "RAPIDS_VER=$RAPIDS_VER" \ + --build-arg "CUDA_VER=$CUDA_VER" \ + --build-arg "PYTHON_VER=$PYTHON_VER" \ + "${platform_args[@]}" + +attached_manifest="$output_dir/attached-image-provenance-index.json" +( + cd "$output_dir" + oras attach \ + --artifact-type application/vnd.rapids.image.provenance.index.v1+json \ + --export-manifest "$attached_manifest" \ + "${registry_reference%@*}@${image_digest}" \ + "image-provenance-index.json:application/vnd.rapids.image.provenance.index.v1+json" +) + +artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" +cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/publish-image-provenance.sh b/ci/publish-image-provenance.sh new file mode 100755 index 00000000..685d8017 --- /dev/null +++ b/ci/publish-image-provenance.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eEuo pipefail + +: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the pushed image tag}" +: "${IMAGE_DIGEST:?Set IMAGE_DIGEST from docker/build-push-action output}" +: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" +: "${IMAGE_PLATFORM:?Set IMAGE_PLATFORM to linux/architecture}" +: "${PROVENANCE_OUTPUT_DIR:?Set PROVENANCE_OUTPUT_DIR from export-image-provenance.sh}" +: "${RAPIDS_VER:?Set RAPIDS_VER}" +: "${CUDA_VER:?Set CUDA_VER}" +: "${PYTHON_VER:?Set PYTHON_VER}" + +normalize_registry_reference() { + local reference="$1" + local first_component="${reference%%/*}" + if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then + reference="docker.io/${reference}" + fi + printf '%s\n' "$reference" +} + +manifest_path="$PROVENANCE_OUTPUT_DIR/image-provenance.json" +workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" +registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" + +generator_args=() +while IFS= read -r argument; do + [[ -z "$argument" ]] && continue + generator_args+=(--build-arg "$argument") +done <<<"${DOCKER_BUILD_ARGS:-}" + +python3 ci/image_provenance_manifest.py \ + --output "$manifest_path" \ + --image-reference "$IMAGE_REFERENCE" \ + --image-digest "$IMAGE_DIGEST" \ + --image-kind "$IMAGE_KIND" \ + --rapids-version "$RAPIDS_VER" \ + --cuda-version "$CUDA_VER" \ + --python-version "$PYTHON_VER" \ + --platform "$IMAGE_PLATFORM" \ + --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --source-commit "$GITHUB_SHA" \ + --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ + --workflow-run-url "$workflow_run_url" \ + --conda-meta-dir "$PROVENANCE_OUTPUT_DIR/conda-meta" \ + --pip-packages "$PROVENANCE_OUTPUT_DIR/pip-packages.json" \ + "${generator_args[@]}" + +attached_manifest="$PROVENANCE_OUTPUT_DIR/attached-image-provenance.json" +( + cd "$PROVENANCE_OUTPUT_DIR" + oras attach \ + --artifact-type application/vnd.rapids.image.provenance.v1+json \ + --export-manifest "$attached_manifest" \ + "${registry_reference%@*}@${IMAGE_DIGEST}" \ + "image-provenance.json:application/vnd.rapids.image.provenance.v1+json" +) + +artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" +cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d480f942..c5392dc2 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ## Usage # Primary interface: bash update-version.sh [--run-context=main|release] @@ -76,11 +76,11 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Set branch references based on RUN_CONTEXT if [[ "${RUN_CONTEXT}" == "main" ]]; then - RAPIDS_BRANCH_NAME="main" + RAPIDS_NOTEBOOKS_REF="main" WORKFLOW_BRANCH_REF="main" echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" elif [[ "${RUN_CONTEXT}" == "release" ]]; then - RAPIDS_BRANCH_NAME="release/${NEXT_SHORT_TAG}" + RAPIDS_NOTEBOOKS_REF="release/${NEXT_SHORT_TAG}" WORKFLOW_BRANCH_REF="release/${NEXT_SHORT_TAG}" echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" fi @@ -96,9 +96,9 @@ done sed_runner "s/com\.nvidia\.workbench\.image-version=.*/com.nvidia.workbench.image-version=\"${NEXT_FULL_TAG}\"/g" Dockerfile -# Dockerfile RAPIDS_BRANCH -sed_runner "s|ARG RAPIDS_BRANCH=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile -sed_runner "s|ARG RAPIDS_BRANCH=\"main\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile +# Dockerfile RAPIDS_NOTEBOOKS_REF +sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile +sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"main\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile # docs sed_runner "s|RAPIDS_VER=[[:digit:]]\+\.[[:digit:]]|RAPIDS_VER=${NEXT_SHORT_TAG}|g" CONTRIBUTING.md diff --git a/context/.dockerignore b/context/.dockerignore index 90ffe897..1acef0b8 100644 --- a/context/.dockerignore +++ b/context/.dockerignore @@ -7,6 +7,7 @@ !pinned !scripts/configure-apt !scripts/configure-conda-base-environment +!scripts/export-pip-package-inventory !scripts/install-gha-tools !scripts/install-tzdata-packages !scripts/update-base-conda-environment diff --git a/context/notebooks.sh b/context/notebooks.sh index 5677cc29..f442a331 100755 --- a/context/notebooks.sh +++ b/context/notebooks.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Clones repos with notebooks & compiles notebook test dependencies # Requires environment variables: -# RAPIDS_BRANCH +# RAPIDS_NOTEBOOKS_REF # CUDA_VER # PYTHON_VER @@ -14,7 +14,7 @@ NOTEBOOK_REPOS=(cudf cuml cugraph) mkdir -p /notebooks /dependencies for REPO in "${NOTEBOOK_REPOS[@]}"; do echo "Cloning $REPO..." - git clone -b "${RAPIDS_BRANCH}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" + git clone -b "${RAPIDS_NOTEBOOKS_REF}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" SOURCE="$REPO/notebooks" DESTINATION="/notebooks/$REPO" diff --git a/context/scripts/export-pip-package-inventory b/context/scripts/export-pip-package-inventory new file mode 100644 index 00000000..3202835e --- /dev/null +++ b/context/scripts/export-pip-package-inventory @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Export pip-installed distributions with their canonical PyPI pURLs.""" + +from __future__ import annotations + +import json +import re +import sys +from importlib import metadata +from pathlib import Path + + +def main() -> None: + output = Path(sys.argv[1]) + packages = [] + for distribution in metadata.distributions(): + name = str(distribution.metadata.get("Name") or "").strip() + version = str(distribution.version or "").strip() + installer = (distribution.read_text("INSTALLER") or "").strip().lower() + if not name or not version or installer != "pip": + continue + normalized_name = re.sub(r"[-_.]+", "-", name).lower() + packages.append( + { + "name": name, + "version": version, + "purls": [f"pkg:pypi/{normalized_name}"], + "installer": "pip", + "evidence": "python-distribution-metadata; INSTALLER=pip", + } + ) + output.write_text( + json.dumps(sorted(packages, key=lambda item: item["name"].lower())), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/matrix-test.yaml b/matrix-test.yaml index 77320c31..36cbd07c 100644 --- a/matrix-test.yaml +++ b/matrix-test.yaml @@ -1,8 +1,8 @@ -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # CUDA_VER is `` (e.g. `13`) pull-request: - - { CUDA_VER: '12', ARCH: 'arm64', PYTHON_VER: '3.11', GPU: 'a100', DRIVER: 'earliest' } + - { CUDA_VER: '12', ARCH: 'arm64', PYTHON_VER: '3.12', GPU: 'a100', DRIVER: 'earliest' } - { CUDA_VER: '12', ARCH: 'amd64', PYTHON_VER: '3.12', GPU: 'l4', DRIVER: 'latest' } - { CUDA_VER: '13', ARCH: 'amd64', PYTHON_VER: '3.13', GPU: 'h100', DRIVER: 'latest' } - { CUDA_VER: '13', ARCH: 'arm64', PYTHON_VER: '3.14', GPU: 'l4', DRIVER: 'latest' } diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..2c78728c --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +pytest==8.3.5 diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 00000000..4d555c9d --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,90 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: S101 + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).parents[1] + + +def run_tag_check( + tmp_path: Path, http_codes: list[str] +) -> tuple[subprocess.CompletedProcess[str], str]: + """Run check_tag_exists with deterministic curl and sleep replacements.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + codes_file = tmp_path / "http-codes" + codes_file.write_text("\n".join(http_codes) + "\n", encoding="utf-8") + sleep_log = tmp_path / "sleep-log" + + curl = bin_dir / "curl" + curl.write_text( + """#!/bin/bash +set -euo pipefail +if [[ $* == *'/users/login/'* ]]; then + printf '{"token":"test-token"}' + exit 0 +fi +http_code="$(head -n 1 "$HTTP_CODES_FILE")" +tail -n +2 "$HTTP_CODES_FILE" > "${HTTP_CODES_FILE}.tmp" +mv "${HTTP_CODES_FILE}.tmp" "$HTTP_CODES_FILE" +printf '%s' "$http_code" +[[ $http_code != 000 ]] +""", + encoding="utf-8", + ) + curl.chmod(0o700) + + sleep = bin_dir / "sleep" + sleep.write_text( + """#!/bin/bash +set -euo pipefail +printf '%s\n' "$1" >> "$SLEEP_LOG" +""", + encoding="utf-8", + ) + sleep.chmod(0o700) + + environment = os.environ.copy() + environment.update( + { + "GPUCIBOT_DOCKERHUB_TOKEN": "test-password", + "GPUCIBOT_DOCKERHUB_USER": "test-user", + "HTTP_CODES_FILE": str(codes_file), + "PATH": f"{bin_dir}:{environment['PATH']}", + "SLEEP_LOG": str(sleep_log), + } + ) + result = subprocess.run( + ["/bin/bash", "-c", "source ci/common.sh; check_tag_exists staging test-tag"], + cwd=REPOSITORY_ROOT, + env=environment, + check=False, + capture_output=True, + text=True, + ) + sleeps = sleep_log.read_text(encoding="utf-8") if sleep_log.exists() else "" + return result, sleeps + + +def test_tag_check_retries_transient_responses(tmp_path: Path) -> None: + result, sleeps = run_tag_check(tmp_path, ["404", "429", "500", "200"]) + + assert result.returncode == 0 + assert sleeps == "5\n10\n20\n" + assert "HTTP 404" in result.stdout + assert "HTTP 429" in result.stdout + assert "HTTP 500" in result.stdout + + +def test_tag_check_fails_immediately_for_authentication_errors(tmp_path: Path) -> None: + result, sleeps = run_tag_check(tmp_path, ["401"]) + + assert result.returncode == 1 + assert sleeps == "" + assert "HTTP 401" in result.stdout diff --git a/tests/test_image_provenance_manifest.py b/tests/test_image_provenance_manifest.py new file mode 100644 index 00000000..948471b9 --- /dev/null +++ b/tests/test_image_provenance_manifest.py @@ -0,0 +1,174 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: S101 + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +MODULE_PATH = Path(__file__).parents[1] / "ci" / "image_provenance_manifest.py" +REPOSITORY_ROOT = MODULE_PATH.parents[1] +SPEC = importlib.util.spec_from_file_location("image_provenance_manifest", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def manifest_context(**overrides: object) -> object: + """Create valid provenance inputs, with targeted test overrides.""" + defaults = { + "image_reference": "rapidsai/base:26.08-cuda12-py312-amd64", + "image_digest": "sha256:image", + "image_kind": "base", + "rapids_version": "26.08", + "cuda_version": "12", + "python_version": "3.12", + "platform": "linux/amd64", + "source_repository": "https://github.com/rapidsai/docker", + "source_commit": "abc123", + "workflow_ref": ".github/workflows/build-rapids-image.yml@abc123", + "workflow_run_url": "https://github.com/rapidsai/docker/actions/runs/1", + "build_args": ["PYTHON_VER=3.12", "CUDA_VER=12"], + } + defaults.update(overrides) + return MODULE.ManifestContext(**defaults) + + +def test_build_manifest_records_exact_conda_package_facts(tmp_path: Path) -> None: + metadata_dir = tmp_path / "conda-meta" + metadata_dir.mkdir() + (metadata_dir / "c-ares-1.34.8-h1.json").write_text( + json.dumps( + { + "name": "c-ares", + "version": "1.34.8", + "build": "h1", + "build_number": 2, + "channel": "https://conda.anaconda.org/conda-forge", + "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", + } + ) + ) + + manifest = MODULE.build_manifest( + manifest_context(), + metadata_dir=metadata_dir, + ) + + assert manifest["subject"]["platform"] == { + "os": "linux", + "architecture": "amd64", + } + assert manifest["conda_packages"] == [ + { + "build": "h1", + "build_number": 2, + "channel": "https://conda.anaconda.org/conda-forge", + "name": "c-ares", + "purl_source": "unmapped", + "purls": [], + "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", + "version": "1.34.8", + } + ] + assert manifest["build"]["build_args"] == {"CUDA_VER": "12", "PYTHON_VER": "3.12"} + assert manifest["conda_packages_sha256"].startswith("sha256:") + + +def test_build_manifest_records_direct_pip_purl_evidence(tmp_path: Path) -> None: + pip_packages = tmp_path / "pip-packages.json" + pip_packages.write_text( + json.dumps( + [ + { + "name": "torch", + "version": "2.8.0", + "purls": ["pkg:pypi/torch"], + "installer": "pip", + "evidence": "python-distribution-metadata; INSTALLER=pip", + } + ] + ) + ) + + manifest = MODULE.build_manifest( + manifest_context(), + pip_packages_path=pip_packages, + ) + + assert manifest["installed_packages"] == [ + { + "ecosystem": "pypi", + "evidence": "python-distribution-metadata; INSTALLER=pip", + "installer": "pip", + "name": "torch", + "purls": ["pkg:pypi/torch"], + "version": "2.8.0", + } + ] + assert manifest["installed_packages_sha256"].startswith("sha256:") + + +def test_build_manifest_links_platform_manifests_for_multiarch_images() -> None: + manifest = MODULE.build_manifest( + manifest_context( + image_reference="rapidsai/base:26.08-cuda12-py312", + image_digest="sha256:index", + platform="multiarch", + workflow_ref="workflow", + workflow_run_url="run", + build_args=[], + ), + platform_manifests=[ + "linux/arm64|rapidsai/base:tag-arm64|sha256:arm", + "linux/amd64|rapidsai/base:tag-amd64|sha256:amd", + ], + ) + + assert "platform" not in manifest["subject"] + assert [item["platform"] for item in manifest["platform_manifests"]] == [ + "linux/amd64", + "linux/arm64", + ] + + +def test_invalid_build_argument_is_rejected() -> None: + with pytest.raises(ValueError, match="NAME=VALUE"): + MODULE.parse_build_args(["CUDA_VER"]) + + +def test_pip_inventory_collector_is_in_docker_build_context() -> None: + dockerignore = (REPOSITORY_ROOT / "context" / ".dockerignore").read_text() + + assert "!scripts/export-pip-package-inventory" in dockerignore + + +def test_provenance_collector_writes_outside_the_nonroot_image_root() -> None: + dockerfile = (REPOSITORY_ROOT / "Dockerfile").read_text() + + assert "export-pip-package-inventory /tmp/pip-packages.json" in dockerfile + assert "COPY --from=provenance-base-inventory /tmp/pip-packages.json" in dockerfile + + +@pytest.mark.parametrize( + ("script_name", "safe_title"), + [ + ("publish-image-provenance.sh", "image-provenance.json"), + ("publish-image-provenance-index.sh", "image-provenance-index.json"), + ], +) +def test_published_provenance_uses_safe_oci_layer_titles( + script_name: str, + safe_title: str, +) -> None: + script = (REPOSITORY_ROOT / "ci" / script_name).read_text() + + assert "--disable-path-validation" not in script + assert f'"{safe_title}:application/vnd.rapids.image.provenance' in script diff --git a/tests/test_workflow_configuration.py b/tests/test_workflow_configuration.py new file mode 100644 index 00000000..478e1f7f --- /dev/null +++ b/tests/test_workflow_configuration.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: S101 + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).parents[1] +VERSION_SCRIPT = REPOSITORY_ROOT / "ci/compute-rapids-version.sh" + + +def git(repository: Path, *arguments: str) -> None: + subprocess.run( # noqa: S603 + ["/usr/bin/git", *arguments], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + + +def tagged_repository(tmp_path: Path) -> Path: + repository = tmp_path / "repository" + repository.mkdir() + git(repository, "init") + git(repository, "config", "user.email", "test@example.com") + git(repository, "config", "user.name", "Test User") + git(repository, "config", "commit.gpgsign", "false") + + tracked_file = repository / "tracked" + tracked_file.write_text("26.08\n", encoding="utf-8") + git(repository, "add", "tracked") + git(repository, "commit", "-m", "26.08 alpha") + git(repository, "tag", "v26.08.00a") + + tracked_file.write_text("26.10\n", encoding="utf-8") + git(repository, "add", "tracked") + git(repository, "commit", "-m", "26.10 alpha") + git(repository, "tag", "v26.10.00a") + return repository + + +def compute_version( + repository: Path, + tmp_path: Path, + *, + base_ref: str = "", + ref_name: str, +) -> dict[str, str]: + output = tmp_path / "output" + environment = os.environ.copy() + environment.update( + { + "GITHUB_BASE_REF": base_ref, + "GITHUB_OUTPUT": str(output), + "GITHUB_REF_NAME": ref_name, + } + ) + subprocess.run( # noqa: S603 + [str(VERSION_SCRIPT)], + cwd=repository, + env=environment, + check=True, + capture_output=True, + text=True, + ) + return dict(line.split("=", 1) for line in output.read_text().splitlines()) + + +def test_alpha_pull_request_to_main_uses_current_main_line(tmp_path: Path) -> None: + repository = tagged_repository(tmp_path) + + version = compute_version( + repository, + tmp_path, + base_ref="main", + ref_name="904/merge", + ) + + assert version == { + "ALPHA_TAG": "a", + "RAPIDS_NOTEBOOKS_REF": "main", + "RAPIDS_VER": "26.10", + } + + +def test_alpha_release_branch_uses_its_matching_notebook_ref(tmp_path: Path) -> None: + repository = tagged_repository(tmp_path) + + version = compute_version( + repository, tmp_path, ref_name="release/26.10" + ) + + assert version == { + "ALPHA_TAG": "a", + "RAPIDS_NOTEBOOKS_REF": "release/26.10", + "RAPIDS_VER": "26.10", + } + + +def test_alpha_pull_request_uses_its_release_target(tmp_path: Path) -> None: + repository = tagged_repository(tmp_path) + + version = compute_version( + repository, + tmp_path, + base_ref="release/26.10", + ref_name="904/merge", + ) + + assert version == { + "ALPHA_TAG": "a", + "RAPIDS_NOTEBOOKS_REF": "release/26.10", + "RAPIDS_VER": "26.10", + }