diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 000000000..b4b5e8598 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,101 @@ +# Manually-triggered build+push of the endpoints client image to GHCR. +# +# Run it from the Actions tab ("Run workflow"): pick the branch/tag in the UI (that +# selection is github.sha), or type an explicit SHA/branch/tag in the `ref` input. +# The image is pushed to ghcr.io/mlcommons/endpoints tagged with the short commit +# SHA and the ref name, using the automatic GITHUB_TOKEN (no stored secret needed). +name: Publish client image + +on: + workflow_dispatch: + inputs: + ref: + description: "Git ref to build (branch/tag/SHA). Blank = the ref selected above." + required: false + type: string + platforms: + description: "Target platform(s)" + required: true + default: linux/amd64 + type: choice + options: + - linux/amd64 + - linux/amd64,linux/arm64 + provision_dsr1: + description: "Bake in the DeepSeek-R1 accuracy evaluator (heavier; needs build-time network)" + required: true + default: "1" + type: choice + options: + - "1" + - "0" + provision_vbench: + description: "Bake in the WAN 2.2 VBench accuracy scorer (heavier; downloads AMT/RAFT/RAM weights at build time)" + required: true + default: "1" + type: choice + options: + - "1" + - "0" + no_cache: + description: "Build with --no-cache (clean/reproducible, slower)" + required: true + default: "true" + type: choice + options: + - "true" + - "false" + force: + description: "Overwrite an existing : tag. platform/DSR1/cache are NOT part of the tag, so the first successful publish of a SHA wins; use force to re-publish a SHA (e.g. a different arch, or a failed run)." + required: true + default: "false" + type: choice + options: + - "false" + - "true" + +# GITHUB_TOKEN needs packages:write to push to the org's GHCR package. +permissions: + contents: read + packages: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout selected ref + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Check out the ref *name* (not github.sha) so a local branch head exists + # and the script can resolve/tag it; blank input falls back to the + # dispatched branch/tag. Full history for `git rev-parse --short`. + ref: ${{ inputs.ref || github.ref_name }} + fetch-depth: 0 + + # Registers QEMU binfmt handlers so the docker-container builder can build + # the non-native arch when platforms includes linux/arm64. + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Logs in as the user who triggered the run; the password is this run's + # automatic GITHUB_TOKEN (scoped by the permissions block above). + - name: Log in to GitHub Container Registry + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + env: + ENDPOINTS_REF: ${{ inputs.ref || github.ref_name }} + PLATFORM: ${{ inputs.platforms }} + PROVISION_DSR1: ${{ inputs.provision_dsr1 }} + PROVISION_VBENCH: ${{ inputs.provision_vbench }} + NO_CACHE: ${{ inputs.no_cache == 'true' && '1' || '0' }} + # Fixed flag or empty — the input value itself is never interpolated into the command. + FORCE_FLAG: ${{ inputs.force == 'true' && '--force' || '' }} + run: ./scripts/push_docker_image.sh $FORCE_FLAG diff --git a/scripts/Dockerfile.dev b/scripts/Dockerfile.dev index 2751dc830..f008a4b29 100644 --- a/scripts/Dockerfile.dev +++ b/scripts/Dockerfile.dev @@ -102,3 +102,14 @@ RUN if [ "${PROVISION_VBENCH}" = "1" ]; then \ else \ echo "PROVISION_VBENCH=${PROVISION_VBENCH}: skipping VBench provisioning" ; \ fi + +# OCI image metadata so `docker inspect` self-identifies this image (the endpoints benchmark +# client, distinct from the lcb-service evaluator image). Declared last so a wording change +# only rebuilds this tiny config layer, not the apt/uv/DSR1/VBench stages above. +# push_docker_image.sh additionally stamps dynamic source/revision/version/description +# annotations on the pushed manifest. The description is deliberately capability-neutral: +# PROVISION_DSR1 / PROVISION_VBENCH can be 0, so it names the evaluators without asserting +# they are present in every build. +LABEL org.opencontainers.image.title="inference-endpoint" \ + org.opencontainers.image.description="MLPerf inference endpoint benchmarking client (the inference-endpoint CLI); DeepSeek-R1 and VBench accuracy evaluators are provisioned by default, toggled by PROVISION_DSR1 / PROVISION_VBENCH." \ + org.opencontainers.image.source="https://github.com/mlcommons/endpoints" diff --git a/scripts/lib_registry.sh b/scripts/lib_registry.sh new file mode 100644 index 000000000..a3f2b1965 --- /dev/null +++ b/scripts/lib_registry.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# lib_registry.sh — shared registry helpers for the image push scripts. +# +# Not meant to be executed directly. `source` it from a push script. + +# assert_gzip_layers REF — fail unless every layer of REF (walking a manifest list) +# is gzip-compressed. enroot/pyxis routes a layer to its decompressor by media type +# and does not recognise the docker-namespaced zstd type +# (application/vnd.docker.image.rootfs.diff.tar.zstd) that buildx emits for zstd base +# layers under oci-mediatypes=false; it hands the raw blob to tar, which dies with +# "tar: This does not look like a tar archive" (mlcommons/endpoints#467). Reads registry +# metadata only (no blob pull), via `docker buildx imagetools inspect`. Fails CLOSED: +# any inspect error blocks the publish rather than silently passing. +assert_gzip_layers() { + local ref="$1" + docker buildx imagetools inspect "$ref" --raw >/dev/null 2>&1 \ + || { echo "error: cannot inspect ${ref} to verify layer compression." >&2; return 2; } + # --raw yields either an image index (has .manifests) or a single manifest (has + # .layers). Parse with python for robustness across both shapes and media-type + # namespaces (docker + oci); descend one level for a manifest list. + python3 - "$ref" <<'PY' +import sys, json, subprocess +ref = sys.argv[1] +base = ref.split("@", 1)[0] +def raw(r): + return json.loads(subprocess.check_output( + ["docker", "buildx", "imagetools", "inspect", r, "--raw"])) +def layer_types(man): + return [layer["mediaType"] for layer in man.get("layers", [])] +top = raw(ref) +types = [] +if top.get("manifests"): # image index / manifest list + for child in top["manifests"]: + plat = child.get("platform", {}) + if plat.get("os") == "unknown" or plat.get("architecture") == "unknown": + continue # skip attestation manifests + types += layer_types(raw(f"{base}@{child['digest']}")) +else: # single-arch manifest + types = layer_types(top) +bad = sorted({mt for mt in types if not mt.endswith("gzip")}) +if bad: + sys.stderr.write( + f"error: {ref} has non-gzip layer(s): {', '.join(bad)}\n" + " enroot/pyxis cannot extract these (mlcommons/endpoints#467).\n" + " Rebuild via the buildx --platform path, which forces gzip.\n") + sys.exit(1) +print(f">> Verified: all {len(types)} layers of {ref} are gzip (enroot-safe).") +PY +} + +# ref_exists_in_registry REF — probe whether REF is already published, reading registry +# metadata only (no blob pull) via `docker buildx imagetools inspect`. Return codes: +# 0 present +# 1 definitely absent (registry reported not-found) +# 2 indeterminate (auth / network / tooling error) — output echoed to stderr +# Callers enforcing an immutable tag MUST treat 2 as "block" (fail CLOSED): never +# overwrite when existence can't be verified, or the guard silently no-ops on exactly +# the hosts/creds where it can't check. +ref_exists_in_registry() { + local ref="$1" out + if out="$(docker buildx imagetools inspect "$ref" 2>&1)"; then + return 0 + fi + # Two error classes contain not-found-ish phrasing but are NOT an absent manifest, and + # classifying either as absent would let a push overwrite an immutable tag: + # - tooling failures: "docker-credential-xxx: executable file not found" + # - authorization hidden as absence: many registries (e.g. Docker Hub) answer an + # unauthorized/private repo with "repository does not exist or may require 'docker + # login': denied", which also matches "does not exist" below. + # Screen both to indeterminate FIRST so the not-found match can't fire on them (fail + # CLOSED). A truly-absent public tag (GHCR ": not found", ECR "name unknown") has + # none of these phrases and still resolves to rc=1. + if grep -qiE 'executable file not found|command not found|no such file or directory|permission denied|credential|denied|unauthorized|forbidden|requires? .*(login|auth)|may require|401|403|authentication' <<<"$out"; then + printf '%s\n' "$out" >&2 + return 2 + fi + # Registry "absent" phrasings: GHCR (`: not found`), OCI/Docker distribution + # (`manifest unknown`, `name unknown`), and ECR (`name unknown … does not exist`). + if grep -qiE 'not found|manifest unknown|manifest_unknown|name[ _]unknown|no such manifest|does not exist' <<<"$out"; then + return 1 + fi + printf '%s\n' "$out" >&2 + return 2 +} diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh new file mode 100755 index 000000000..d02b044e7 --- /dev/null +++ b/scripts/push_docker_image.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# push_docker_image.sh — build and push the endpoints client image to a registry. +# +# Builds scripts/Dockerfile.dev at a given git ref and pushes it to GitHub +# Container Registry (default: ghcr.io/mlcommons/endpoints), tagged with both the +# short commit SHA and the ref name so consumers can pin either. +# +# Prerequisites: +# - docker with buildx (the script provisions a docker-container builder; the +# default 'docker' driver can neither --push nor build multi-platform). +# - Authenticated to the target registry. Either run `docker login ghcr.io` +# yourself, or export GHCR_USER + GHCR_TOKEN (a GitHub PAT / GITHUB_TOKEN with +# write:packages) and this script logs in for you. +# +# Usage: +# ./scripts/push_docker_image.sh # build+push current HEAD (linux/amd64) +# ENDPOINTS_REF=v1.2.0 ./scripts/push_docker_image.sh # build+push a specific tag/branch/sha +# ./scripts/push_docker_image.sh --ref main # same, via flag +# ./scripts/push_docker_image.sh --multi-arch # linux/amd64,linux/arm64 manifest list +# ./scripts/push_docker_image.sh --platform linux/arm64 # single non-native arch +# ./scripts/push_docker_image.sh --cache # allow layer cache (default: --no-cache) +# ./scripts/push_docker_image.sh --multi-arch --no-push # dry build (validate both arches, publish nothing) +# ./scripts/push_docker_image.sh --allow-dirty # build HEAD despite uncommitted tracked changes +# ./scripts/push_docker_image.sh --force # overwrite an existing : tag (default: refuse) +# +# Multi-arch note: --multi-arch (or --platform with a comma) pushes a manifest +# list usable on BOTH x86 and arm hosts; Docker pulls the matching arch. The +# non-native leg needs QEMU registered on the host once: +# docker run --privileged --rm tonistiigi/binfmt --install all +# Dockerfile.dev's DSR1 provisioning runs under emulation and is MUCH slower. +# +# Environment variables (flags take precedence): +# IMAGE full image ref w/o tag (default: ghcr.io/mlcommons/endpoints) +# ENDPOINTS_REF git ref to build (default: current HEAD; checked out if it differs) +# PLATFORM target platform(s) (default: linux/amd64) +# DOCKERFILE Dockerfile path (default: scripts/Dockerfile.dev) +# NO_CACHE 1=--no-cache, 0=cache (default: 1) +# ALLOW_DIRTY 1=build HEAD with uncommitted tracked changes (default: 0) +# PROVISION_DSR1 passthrough build-arg (default: Dockerfile default = 1) +# PROVISION_VBENCH passthrough build-arg (default: Dockerfile default = 1) +# IMAGE_DESCRIPTION GHCR package-page description (default: "endpoints client image (ref , commit )") +# GHCR_USER/GHCR_TOKEN optional registry login (fallback: GITHUB_ACTOR/GITHUB_TOKEN) +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILDX_BUILDER="endpoints-buildx" + +# shellcheck source=lib_registry.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib_registry.sh" + +usage() { + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//; /^set -euo/d' +} + +IMAGE="${IMAGE:-ghcr.io/mlcommons/endpoints}" +ENDPOINTS_REF="${ENDPOINTS_REF:-}" +PLATFORM="${PLATFORM:-linux/amd64}" +DOCKERFILE="${DOCKERFILE:-scripts/Dockerfile.dev}" +NO_CACHE="${NO_CACHE:-1}" +ALLOW_DIRTY="${ALLOW_DIRTY:-0}" +PUSH=1 +FORCE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --ref) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --ref requires a value" >&2; exit 1; } + ENDPOINTS_REF="$2"; shift ;; + --ref=*) ENDPOINTS_REF="${1#*=}" ;; + --image) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --image requires a value" >&2; exit 1; } + IMAGE="$2"; shift ;; + --image=*) IMAGE="${1#*=}" ;; + --platform) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --platform requires a value, e.g. linux/arm64" >&2; exit 1; } + PLATFORM="$2"; shift ;; + --platform=*) PLATFORM="${1#*=}" ;; + --multi-arch) PLATFORM="linux/amd64,linux/arm64" ;; + --no-cache) NO_CACHE=1 ;; + --cache) NO_CACHE=0 ;; + --no-push) PUSH=0 ;; + --allow-dirty) ALLOW_DIRTY=1 ;; + --force) FORCE=1 ;; + --dockerfile) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --dockerfile requires a value" >&2; exit 1; } + DOCKERFILE="$2"; shift ;; + --dockerfile=*) DOCKERFILE="${1#*=}" ;; + -h | --help) usage; exit 0 ;; + *) echo "error: unknown argument '$1'" >&2; usage; exit 1 ;; + esac + shift +done + +cd "$REPO_ROOT" + +if ! docker buildx version >/dev/null 2>&1; then + echo "error: 'docker buildx' is required but not available." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Optional registry login. Skipped silently if no token is provided (assumes the +# caller already ran `docker login`). Registry host is the first path segment of +# IMAGE (e.g. ghcr.io from ghcr.io/mlcommons/endpoints). +# --------------------------------------------------------------------------- +REGISTRY_HOST="${IMAGE%%/*}" +LOGIN_USER="${GHCR_USER:-${GITHUB_ACTOR:-}}" +LOGIN_TOKEN="${GHCR_TOKEN:-${GITHUB_TOKEN:-}}" +if [[ "$PUSH" == "1" && -n "$LOGIN_TOKEN" ]]; then + echo ">> Logging in to ${REGISTRY_HOST} as ${LOGIN_USER:-}" + echo "$LOGIN_TOKEN" | docker login "$REGISTRY_HOST" -u "${LOGIN_USER:-x-access-token}" --password-stdin +fi + +# --------------------------------------------------------------------------- +# Resolve the ref to build. If ENDPOINTS_REF is set and points at a different +# commit than the current HEAD, check it out (requiring a clean tree) and restore +# the original ref on exit so the working copy is left as we found it. +# --------------------------------------------------------------------------- +ORIGINAL_REF="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)" +[[ "$ORIGINAL_REF" == "HEAD" ]] && ORIGINAL_REF="$(git rev-parse HEAD)" + +restore_ref() { + if [[ -n "${DID_CHECKOUT:-}" && -n "$ORIGINAL_REF" ]]; then + echo ">> Restoring original ref ${ORIGINAL_REF}" + git checkout --quiet "$ORIGINAL_REF" + fi +} +trap restore_ref EXIT + +# Uncommitted changes to TRACKED files (staged or unstaged) make the working tree +# differ from HEAD. Untracked files are ignored here: they are not part of any commit +# and .dockerignore governs the build context. +tree_has_tracked_changes() { ! git diff --quiet HEAD 2>/dev/null; } + +if [[ -n "$ENDPOINTS_REF" ]]; then + # Fall back to the remote-tracking ref: after actions/checkout lands a detached + # SHA, a bare branch name like 'main' exists only as origin/main, so the plain + # resolve would fail and abort the CI default-dispatch path. + target_sha="$(git rev-parse --verify "${ENDPOINTS_REF}^{commit}" 2>/dev/null \ + || git rev-parse --verify "origin/${ENDPOINTS_REF}^{commit}" 2>/dev/null || true)" + [[ -n "$target_sha" ]] || { echo "error: '$ENDPOINTS_REF' is not a valid git ref." >&2; exit 1; } + if [[ "$target_sha" != "$(git rev-parse HEAD)" ]]; then + if tree_has_tracked_changes; then + echo "error: working tree has uncommitted changes to tracked files; commit/stash before building a different ref ($ENDPOINTS_REF)." >&2 + exit 1 + fi + echo ">> Checking out ${ENDPOINTS_REF}" + git checkout --quiet "$ENDPOINTS_REF" + DID_CHECKOUT=1 + fi +else + # Tag the built image with the current branch/ref name for readability. + ENDPOINTS_REF="$ORIGINAL_REF" +fi + +SHORT_SHA="$(git rev-parse --short HEAD)" +# Docker tags allow only [A-Za-z0-9_.-], must not start with '.' or '-', and cap at +# 128 chars. Map every other char (e.g. '/', '+', '~', '#') to '-' so a legitimate +# git ref name can't yield an invalid tag that fails `-t` only after the whole build. +REF_TAG="$(printf '%s' "$ENDPOINTS_REF" | tr -c 'A-Za-z0-9_.-' '-')" +[[ "$REF_TAG" == [A-Za-z0-9_]* ]] || REF_TAG="_${REF_TAG}" +REF_TAG="${REF_TAG:0:128}" + +# Building the working tree as-is (no distinct ref checked out): don't publish a +# :$SHORT_SHA tag whose '.' build context doesn't match commit $SHORT_SHA. +if [[ -z "${DID_CHECKOUT:-}" && "$ALLOW_DIRTY" != "1" ]] && tree_has_tracked_changes; then + dirty_msg="working tree has uncommitted changes to tracked files; :${SHORT_SHA} would not match commit ${SHORT_SHA}" + if [[ "$PUSH" == "1" ]]; then + echo "error: ${dirty_msg}." >&2 + echo " Commit/stash the changes, or pass --allow-dirty to publish anyway." >&2 + exit 1 + fi + echo ">> warning: ${dirty_msg} (dry run; nothing pushed)." >&2 +fi + +# --------------------------------------------------------------------------- +# Refuse to overwrite an already-published : tag unless --force. The SHA tag is a +# stable pin (consumers `docker pull …:`), and SHORT_SHA is not a function of the +# build inputs the workflow varies (platform, PROVISION_DSR1, cache), so a second push +# of the same commit would silently replace a published image. Reads registry metadata +# only; fail CLOSED (an indeterminate probe blocks too). This gate covers the whole build: +# if : exists, BOTH it and the moving : tag are refused until --force, so +# refreshing : for an already-published commit needs --force (the CI workflow +# exposes a `force` input for exactly this). +# --------------------------------------------------------------------------- +if [[ "$PUSH" == "1" && "$FORCE" != "1" ]]; then + rc=0; ref_exists_in_registry "${IMAGE}:${SHORT_SHA}" || rc=$? + if [[ "$rc" -eq 0 ]]; then + echo "error: ${IMAGE}:${SHORT_SHA} already exists in the registry." >&2 + echo " The SHA tag is a stable pin; re-run with --force to overwrite it." >&2 + exit 1 + elif [[ "$rc" -ne 1 ]]; then + echo "error: could not verify whether ${IMAGE}:${SHORT_SHA} already exists (see above)." >&2 + echo " Fix registry access/login, or re-run with --force to skip this check." >&2 + exit 1 + fi +fi + +# --------------------------------------------------------------------------- +# Ensure a docker-container builder (supports --push and multi-platform). +# --------------------------------------------------------------------------- +if ! docker buildx inspect "$BUILDX_BUILDER" >/dev/null 2>&1; then + echo ">> Creating buildx builder '${BUILDX_BUILDER}' (docker-container driver)" + docker buildx create --name "$BUILDX_BUILDER" --driver docker-container >/dev/null +fi + +if [[ "$PLATFORM" == *,* ]]; then + echo ">> Multi-arch build for ${PLATFORM} — requires QEMU for non-native archs" + echo " (register once with: docker run --privileged --rm tonistiigi/binfmt --install all)" +fi + +BUILD_ARGS=() +[[ "$NO_CACHE" == "1" ]] && BUILD_ARGS+=(--no-cache) +[[ -n "${PROVISION_DSR1:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_DSR1=${PROVISION_DSR1}") +[[ -n "${PROVISION_VBENCH:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_VBENCH=${PROVISION_VBENCH}") + +# Don't attach SLSA provenance attestations. buildx adds them by default on --push, +# which surfaces a spurious `unknown/unknown` entry on the GHCR package page and +# wraps even a single-arch build in an image index. Opting out yields a clean, +# single-manifest artifact; the source/revision/version annotations below preserve +# the "which commit built this" traceability the attestation would have carried. +BUILD_ARGS+=(--provenance=false) + +# With --no-push we build to cache and discard (validation only). A multi-platform +# result cannot be --load-ed into the local store, so no output flag is added. +# +# On push, force EVERY layer (including inherited base-image layers) to gzip so the +# artifact is extractable by enroot/pyxis on SLURM. Without force-compression, BuildKit +# copies base layers in their original codec (zstd for some bases) and enroot import +# fails with "tar: This does not look like a tar archive" (mlcommons/endpoints#467). +# type=image,push=true is the long form of --push (the two conflict); oci-mediatypes is +# left at its default so the --annotation provenance below still applies. +[[ "$PUSH" == "1" ]] && BUILD_ARGS+=(--output "type=image,push=true,compression=gzip,force-compression=true") + +# OCI annotations for the GHCR package page (description + repo/commit provenance). +# --annotation requires the build to actually produce the component named by the +# level prefix, so gate on --push and pick the level by artifact shape: a multi-arch +# build pushes an image index (GHCR reads the description from the index); a +# single-arch build (provenance off, above) pushes a lone manifest, for which +# `index:` errors with "index annotations not supported for single platform export". +if [[ "$PUSH" == "1" ]]; then + if [[ "$PLATFORM" == *,* ]]; then + ANNOTATION_LEVEL="index" + else + ANNOTATION_LEVEL="manifest" + fi + IMAGE_DESCRIPTION="${IMAGE_DESCRIPTION:-endpoints client image (ref ${ENDPOINTS_REF}, commit ${SHORT_SHA})}" + BUILD_ARGS+=( + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.source=https://github.com/mlcommons/endpoints" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.revision=${SHORT_SHA}" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.version=${ENDPOINTS_REF}" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.description=${IMAGE_DESCRIPTION}" + ) +fi + +BUILD_VERB=$([[ "$PUSH" == "1" ]] && echo "and pushing" || echo "(dry run, --no-push)") +echo ">> Building ${IMAGE}:${SHORT_SHA} (ref: ${ENDPOINTS_REF}) for ${PLATFORM} ${BUILD_VERB}" +docker buildx build \ + --builder "$BUILDX_BUILDER" \ + --platform "$PLATFORM" \ + -f "$DOCKERFILE" \ + -t "${IMAGE}:${SHORT_SHA}" \ + -t "${IMAGE}:${REF_TAG}" \ + ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} \ + . + +# Guard against silently publishing an image enroot/pyxis cannot extract (#467). This runs +# AFTER --push, but the build above forces every layer to gzip (compression=gzip, +# force-compression=true), so the pushed image is enroot-safe by construction — this is a +# post-publish confirmation, not a gate. Known caveat: a transient registry-inspect flake +# here fails the job after the tag is already live; the immutable-tag guard then blocks a +# plain retry, so re-run with --force to republish. (Verify-before-publish would need the +# staging→promote dance the LCB --no-build path uses; not worth it when gzip is forced.) +[[ "$PUSH" == "1" ]] && { assert_gzip_layers "${IMAGE}:${SHORT_SHA}" || exit 1; } + +echo +if [[ "$PUSH" == "1" ]]; then + echo "Done. Pushed:" + echo " ${IMAGE}:${SHORT_SHA}" + echo " ${IMAGE}:${REF_TAG}" + echo "Pull with:" + echo " docker pull ${IMAGE}:${SHORT_SHA}" +else + echo "Done. Dry build succeeded for ${PLATFORM} (nothing pushed)." +fi diff --git a/src/inference_endpoint/evaluation/livecodebench/README.md b/src/inference_endpoint/evaluation/livecodebench/README.md index a5b7fb5dd..3899717ab 100644 --- a/src/inference_endpoint/evaluation/livecodebench/README.md +++ b/src/inference_endpoint/evaluation/livecodebench/README.md @@ -182,56 +182,88 @@ consumer can **pull and run it with no `HF_TOKEN` and no rebuild**. Two scripts in this directory wrap the docker build/tag/push/pull steps. Both resolve the image reference from environment variables (shared via `_image_env.sh`): -| Variable | Required | Default | Meaning | -| -------------------- | -------- | -------------------- | --------------------------------------------------------------------------- | -| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | -| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | -| `LCB_IMAGE_TAG` | no | `release_v6` | tag; defaults to the baked-in dataset version so the tag is self-describing | -| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | +| Variable | Required | Default | Meaning | +| -------------------- | -------------------- | ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | +| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | +| `LCB_IMAGE_TAG` | push: no · pull: yes | `-livecodebench` | image tag; `push_image.sh` defaults the SHA part to the endpoints commit SHA and `_image_env.sh` always appends a `-livecodebench` suffix (one immutable, self-identifying tag per build). Pull must name the build — pass the SHA; the suffix is added for you. | +| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | -The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`. +The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`, where `_image_env.sh` always appends a `-livecodebench` suffix to `LCB_IMAGE_TAG` so the LCB image is self-identifying and never collides with the client image's bare `:` tag in a shared package. `push_image.sh` defaults the SHA part to the endpoints commit short SHA, so each build publishes an immutable `…/lcb-service:-livecodebench` — there is no moving `latest`/`release_v6` tag. The same SHA is also baked into the image as the `org.opencontainers.image.revision` label. #### Push (maintainer) Requires `docker login dhi.io` (base images) and `docker login` to your target registry first. +##### Updating the official repo image (`ghcr.io/mlcommons/endpoints`) + +Maintainers with `write:packages` on the **mlcommons** org publish the canonical image from the repo root. Authenticate to GHCR (a PAT / `GITHUB_TOKEN` with `write:packages`) and to `dhi.io` (hardened base images) first, then run the push script with the official registry + name: + ```bash -# Build (using HF_TOKEN as a build secret) and push: +echo | docker login ghcr.io -u --password-stdin +docker login dhi.io + +LCB_IMAGE_REGISTRY=ghcr.io/mlcommons LCB_IMAGE_NAME=endpoints HF_TOKEN= \ + ./src/inference_endpoint/evaluation/livecodebench/push_image.sh +``` + +This builds a multi-arch manifest and publishes `ghcr.io/mlcommons/endpoints:-livecodebench`. The `-livecodebench` suffix keeps it distinct from the client image's bare `:` tag in the same package, and the tag is immutable — re-pushing the same commit is refused unless you add `--force`. + +##### Publishing to your own registry + +```bash +# Build (using HF_TOKEN as a build secret) and push a MULTI-ARCH manifest +# (linux/amd64,linux/arm64) by default: LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= \ ./push_image.sh -# Or push an already-built local image without rebuilding: -LCB_IMAGE_REGISTRY=myregistry.com/team ./push_image.sh --no-build +# Or push an already-built local image without rebuilding (name the build explicitly): +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no-build ``` -**Cross-architecture builds.** To build for an architecture other than the host's (e.g. build `arm64` on an -`x86_64` node, or a multi-arch manifest), pass `--platform`: +Every build goes through `docker buildx` and pushes **straight to the registry**: buildx forces gzip layers so the +image is extractable by enroot/pyxis on SLURM (see [#467](https://github.com/mlcommons/endpoints/issues/467)), and a +multi-arch image cannot be loaded into the local docker store anyway. The script auto-creates a `docker-container` +buildx builder. `--no-build` is the one exception: it publishes a pre-built local `lcb-service:latest` (**host arch only**) by pushing it to a transient `staging-…` tag, verifying its layers, then promoting onto the pinned tag on success. Because a plain push cannot force gzip, a local image carrying zstd layers (containerd image store) is **rejected, not repaired** — rebuild via the buildx path (drop `--no-build`) to get gzip layers. `--no-build` also ignores `--platform`. + +Each build publishes an immutable `:-livecodebench` tag, so **re-pushing an existing `:-livecodebench` is refused** to protect it. Pass `--force` to overwrite deliberately (e.g. re-running a partially failed push): + +```bash +LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --force +``` + +**Single-arch / cross-arch builds.** Pass `--platform` to override the multi-arch default — e.g. a single arch (much +faster, no emulation) or a specific target: ```bash LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --platform linux/arm64 LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --platform linux/amd64,linux/arm64 ``` -Platform builds use `docker buildx` and push the image **straight to the registry** (a non-native image cannot be -loaded into the local docker store, so there is no `--no-build` for this path). The script auto-creates a -`docker-container` buildx builder. The target architecture must have **QEMU emulation registered on the host**, -a one-time step that needs `--privileged`: +Building an architecture other than the host's needs **QEMU emulation registered on the host**, a one-time step that +needs `--privileged`: ```bash docker run --privileged --rm tonistiigi/binfmt --install all ``` -> ⚠️ The dataset-generation step runs under emulation when cross-building, which is **much slower** than a native -> build (and still needs the same ~21 GiB peak). Prefer building natively on a host of the target architecture -> when one is available. +> ⚠️ The dataset-generation step runs under emulation for any non-host arch, which is **much slower** than native +> (and still needs the same ~21 GiB peak). Since multi-arch is the default, expect the non-host arch to build under +> emulation; pass `--platform ` for a quick single-arch image when that is enough. #### Pull (consumer / eval side) -No `HF_TOKEN` needed. By default the image is re-tagged locally as `lcb-service:latest`, so the -[hardened run command](#hardened-run-command) and the scorer's `ws://localhost:13835/evaluate` expectation work unchanged. +No `HF_TOKEN` needed. Set `LCB_IMAGE_TAG` to the build (short SHA) you want to pull. By default the image is re-tagged +locally as `lcb-service:latest`, so the [hardened run command](#hardened-run-command) and the scorer's +`ws://localhost:13835/evaluate` expectation work unchanged. ```bash -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh + +# Official image published by maintainers (matches the push recipe above). LCB_IMAGE_NAME +# must be set to `endpoints`, otherwise this resolves to ghcr.io/mlcommons/lcb-service (a +# different repo). Pass the SHA; the `-livecodebench` suffix is added for you. +LCB_IMAGE_REGISTRY=ghcr.io/mlcommons LCB_IMAGE_NAME=endpoints LCB_IMAGE_TAG= ./pull_image.sh ``` ### (Only if using enroot) Generating a .sqsh file for enroot @@ -240,8 +272,8 @@ The pull script can produce an enroot `.sqsh` from the pulled image with the `-- [enroot](https://github.com/NVIDIA/enroot/tree/main) (e.g. SLURM clusters): ```bash -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh # writes lcb_service.sqsh -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh out.sqsh # custom output path +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh # writes lcb_service.sqsh +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh out.sqsh # custom output path ``` This runs `enroot import --output dockerd://${LCB_IMAGE_REF}` on the just-pulled image. Running the service via enroot is diff --git a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh index ba615cb63..dd866f629 100644 --- a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh +++ b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh @@ -8,10 +8,12 @@ # Inputs (environment variables): # LCB_IMAGE_REGISTRY (required) registry + namespace, e.g. myregistry.com/team # LCB_IMAGE_NAME (optional) image repo name (default: lcb-service) -# LCB_IMAGE_TAG (optional) tag (default: release_v6) +# LCB_IMAGE_TAG (required) build id — push_image.sh defaults it to the endpoints short +# SHA. A "-livecodebench" suffix is always appended (below). # LCB_LOCAL_TAG (optional) local tag used by run/scorer (default: lcb-service:latest) # # Exports: +# LCB_IMAGE_TAG (with a "-livecodebench" suffix appended, idempotently) # LCB_IMAGE_REF = ${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG} # LCB_LOCAL_TAG (defaulted if unset) @@ -23,10 +25,26 @@ if [[ -z "${LCB_IMAGE_REGISTRY:-}" ]]; then fi LCB_IMAGE_NAME="${LCB_IMAGE_NAME:-lcb-service}" -# Defaults to the baked-in dataset version so the artifact is self-describing. -LCB_IMAGE_TAG="${LCB_IMAGE_TAG:-release_v6}" +# Images are tagged by the endpoints commit SHA (one immutable tag per build), so +# there is no channel default. push_image.sh sets this to the SHA automatically; +# a consumer pulling must name the specific build. +if [[ -z "${LCB_IMAGE_TAG:-}" ]]; then + echo "error: LCB_IMAGE_TAG is not set." >&2 + echo " push_image.sh defaults it to the endpoints commit SHA; for pull, set it to" >&2 + echo " the build you want, e.g. LCB_IMAGE_TAG=\$(git rev-parse --short HEAD)" >&2 + return 1 2>/dev/null || exit 1 +fi LCB_LOCAL_TAG="${LCB_LOCAL_TAG:-lcb-service:latest}" +# Every LCB image tag carries a "-livecodebench" component so the artifact is +# self-identifying and never collides with the client image's bare : tag when both +# live in one registry package. Applied here (the shared push/pull helper) so push and +# pull resolve the SAME ref: a consumer passing LCB_IMAGE_TAG= still pulls +# -livecodebench. Idempotent — a tag that already ends in -livecodebench is left as is. +if [[ "$LCB_IMAGE_TAG" != *-livecodebench ]]; then + LCB_IMAGE_TAG="${LCB_IMAGE_TAG}-livecodebench" +fi + # Strip any trailing slash on the registry to avoid a double slash in the ref. LCB_IMAGE_REF="${LCB_IMAGE_REGISTRY%/}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}" diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile index 83b1b79db..6989d929c 100644 --- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile +++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile @@ -63,6 +63,18 @@ COPY _server.py /app/server.py # Make lcb_serve.py available as a module ENV PYTHONPATH="/app" +# OCI image metadata: a static title/description that self-identifies this image (the +# LiveCodeBench evaluator service, distinct from the endpoints client image), plus +# provenance (source, and the endpoints repo commit via the ENDPOINTS_SHA build-arg). +# Recorded as config LABELs (not manifest annotations) so they survive the +# oci-mediatypes=false push and show in `docker inspect`. Declared after the COPYs so a new +# SHA only rebuilds this metadata layer, never the expensive dataset-generation stage above. +ARG ENDPOINTS_SHA=unknown +LABEL org.opencontainers.image.title="lcb-service" \ + org.opencontainers.image.description="LiveCodeBench evaluation service: a WebSocket judge (port 13835) that runs model-generated code against the LiveCodeBench dataset (release_v6, baked in). Used by the endpoints client's code_bench_scorer." \ + org.opencontainers.image.source="https://github.com/mlcommons/endpoints" \ + org.opencontainers.image.revision="${ENDPOINTS_SHA}" + # Launch the WebSocket server with long-running connection support # Default port 13835 # - timeout-keep-alive: Allow connections to stay open for hours diff --git a/src/inference_endpoint/evaluation/livecodebench/pull_image.sh b/src/inference_endpoint/evaluation/livecodebench/pull_image.sh index dc679d780..3b3127d2b 100755 --- a/src/inference_endpoint/evaluation/livecodebench/pull_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/pull_image.sh @@ -6,13 +6,13 @@ # lcb-service:latest so the hardened run command in README.md and the scorer's # ws://localhost:13835/evaluate expectation work unchanged. # -# Usage: -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh # also create lcb_service.sqsh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh out.sqsh # enroot import to out.sqsh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --no-local-tag # skip the lcb-service:latest tag +# Usage (images are tagged by short SHA, so LCB_IMAGE_TAG selects the build): +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh # also create lcb_service.sqsh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh out.sqsh # enroot import to out.sqsh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --no-local-tag # skip the lcb-service:latest tag # -# Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY required). +# Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY + LCB_IMAGE_TAG required). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index 3e92af49b..01d69c58a 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -5,27 +5,34 @@ # build time. Pushing the built image lets consumers pull-and-run with no HF_TOKEN # and no rebuild (see pull_image.sh). # +# The image is tagged -livecodebench — one immutable, self-identifying +# tag per build. The "-livecodebench" suffix is appended by _image_env.sh so the tag never +# collides with the client image's bare : in a shared registry package. LCB_IMAGE_TAG +# overrides the SHA part (the suffix is still appended). There is no moving channel tag. +# # Prerequisites: # - docker login dhi.io (base images are Docker Hardened Images) # - docker login (target for the push) # # Usage: # LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./push_image.sh --no-build # push existing local image +# Builds and pushes a MULTI-ARCH manifest (linux/amd64,linux/arm64) by default. +# ... --platform linux/arm64 # single arch (fast; no QEMU emulation) +# ... --platform linux/amd64,linux/arm64 # explicit multi-arch (the default) +# LCB_IMAGE_REGISTRY=... LCB_IMAGE_TAG= ./push_image.sh --no-build # push existing local image +# ... --force # overwrite an existing : tag (default: refuse) # -# Cross-architecture build (e.g. build arm64 on an x86 node, or a multi-arch manifest): -# LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh --platform linux/arm64 -# ... --platform linux/amd64,linux/arm64 # multi-arch manifest in one push -# (or set LCB_IMAGE_PLATFORM instead of the flag.) -# Platform builds use 'docker buildx' and push the image straight to the registry -# (a non-native image cannot be loaded into the local docker store). They require -# QEMU emulation registered on the host for the target arch, one time: -# docker run --privileged --rm tonistiigi/binfmt --install arm64 -# The dataset-generation step runs under emulation and is MUCH slower than native. +# All builds go through 'docker buildx' and push straight to the registry: buildx forces +# gzip layers so the image is enroot/pyxis-extractable (mlcommons/endpoints#467), and a +# multi-arch image cannot be loaded into the local docker store anyway. Building an arch +# other than the host's needs QEMU emulation registered once (needs --privileged): +# docker run --privileged --rm tonistiigi/binfmt --install all +# The dataset-generation step runs under emulation and is MUCH slower than native, so +# prefer --platform when a single-arch image is enough. # # Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY required). # HF_TOKEN required unless --no-build (passed as a BuildKit secret directly from the environment). -# LCB_IMAGE_PLATFORM optional target platform(s), e.g. linux/arm64 (default: host native). +# LCB_IMAGE_PLATFORM target platform(s), e.g. linux/arm64 (default: linux/amd64,linux/arm64). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -54,10 +61,16 @@ unsupported_platforms() { platform_supported() { [[ -z "$(unsupported_platforms "$1" "$2")" ]]; } NO_BUILD=0 -PLATFORM="${LCB_IMAGE_PLATFORM:-}" +FORCE=0 +# Multi-arch by default: one publish runs on amd64 and arm64, and the buildx path forces +# gzip layers so the image is enroot/pyxis-extractable (mlcommons/endpoints#467). Override +# with --platform for a single arch (e.g. --platform linux/arm64 skips the slow QEMU +# emulation of the non-host arch). +PLATFORM="${LCB_IMAGE_PLATFORM:-linux/amd64,linux/arm64}" while [[ $# -gt 0 ]]; do case "$1" in --no-build) NO_BUILD=1 ;; + --force) FORCE=1 ;; --platform) if [[ -z "${2:-}" || "$2" == --* ]]; then echo "error: --platform requires a value, e.g. linux/arm64" >&2 @@ -80,30 +93,110 @@ while [[ $# -gt 0 ]]; do shift done -# shellcheck source=_image_env.sh -source "${SCRIPT_DIR}/_image_env.sh" - # ---------------------------------------------------------------------------- -# Cross-architecture path: build with buildx and push directly to the registry. -# A non-native image cannot be `docker load`-ed into the local store, so buildx -# builds and pushes in a single step (no separate tag/push). +# Provenance: the endpoints repo commit this image is built from. It is both baked +# into the image config as an OCI label (see --build-arg below) AND used as the +# image tag — one immutable tag per build. A config LABEL (not a manifest +# --annotation) is deliberate: the buildx push sets oci-mediatypes=false, and +# Docker-media-type manifests have no annotations field, whereas config labels are +# representable in both formats and survive `docker inspect`. Scope the dirty check +# to this build context so unrelated edits elsewhere in the repo don't mark it -dirty. # ---------------------------------------------------------------------------- -if [[ -n "$PLATFORM" ]]; then +ENDPOINTS_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" +if [[ "$ENDPOINTS_SHA" != "unknown" ]] && ! git -C "$SCRIPT_DIR" diff --quiet HEAD -- "$SCRIPT_DIR" 2>/dev/null; then + ENDPOINTS_SHA="${ENDPOINTS_SHA}-dirty" +fi + +# Tag the image by the endpoints commit SHA (LCB_IMAGE_TAG overrides). The auto SHA +# tag is only trustworthy when the tree is clean AND we build now, so when the tag is +# not given explicitly, refuse the cases where : would misrepresent the image: +# - no git → nothing to name the image by +# - --no-build → pushes a pre-built local image whose baked revision label may be +# a different commit than HEAD; : would lie about provenance +# - dirty context → :-dirty is a moving, non-reproducible tag +# An explicit LCB_IMAGE_TAG is the escape hatch for all three. Set before sourcing so +# _image_env.sh builds LCB_IMAGE_REF from it. +if [[ -z "${LCB_IMAGE_TAG:-}" ]]; then + if [[ "$ENDPOINTS_SHA" == "unknown" ]]; then + echo "error: cannot determine a short SHA to tag the image (no git); set LCB_IMAGE_TAG." >&2 + exit 1 + fi if [[ "$NO_BUILD" -eq 1 ]]; then - echo "error: --platform builds the image, so it cannot be combined with --no-build." >&2 + echo "error: --no-build pushes a pre-built local image whose baked revision may not match" >&2 + echo " HEAD (${ENDPOINTS_SHA}); set LCB_IMAGE_TAG explicitly to name it." >&2 exit 1 fi + if [[ "$ENDPOINTS_SHA" == *-dirty ]]; then + echo "error: build context has uncommitted changes; :${ENDPOINTS_SHA} is a moving tag." >&2 + echo " Commit the changes, or set LCB_IMAGE_TAG explicitly to push anyway." >&2 + exit 1 + fi +fi +export LCB_IMAGE_TAG="${LCB_IMAGE_TAG:-$ENDPOINTS_SHA}" + +# shellcheck source=_image_env.sh +source "${SCRIPT_DIR}/_image_env.sh" +# shellcheck source=/dev/null +source "$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)/scripts/lib_registry.sh" + +# Refuse to overwrite an existing remote tag unless --force: the SHA tag is meant to be +# immutable, so a second push would silently replace a published image. Delegates to the +# shared ref_exists_in_registry (fail CLOSED — anything other than a confirmed "absent" +# blocks). --force skips the check entirely. +if [[ "$FORCE" -eq 0 ]]; then + rc=0; ref_exists_in_registry "$LCB_IMAGE_REF" || rc=$? + if [[ "$rc" -eq 0 ]]; then + echo "error: ${LCB_IMAGE_REF} already exists in the registry." >&2 + echo " The SHA tag is meant to be immutable; re-run with --force to overwrite it." >&2 + exit 1 + elif [[ "$rc" -ne 1 ]]; then + echo "error: could not verify whether ${LCB_IMAGE_REF} already exists (see above)." >&2 + echo " Fix registry access (or install docker buildx), or re-run with --force to skip this check." >&2 + exit 1 + fi +fi + +# ---------------------------------------------------------------------------- +# --no-build publishes a pre-built local image (host arch only): push to a staging tag, +# verify, then promote onto the pinned tag (a plain push can't be force-compressed, so +# the immutable : is only made reachable after the gzip check passes). +# Every other invocation builds with buildx and pushes straight to the registry: buildx +# forces gzip layers (enroot/pyxis-safe, #467) and a multi-arch image cannot be +# `docker load`-ed into the local store anyway, so there is no separate build/tag step. +# ---------------------------------------------------------------------------- +if [[ "$NO_BUILD" -eq 1 ]]; then + # A pre-built local image can carry zstd base layers (containerd image store) that + # enroot/pyxis cannot extract (#467), and this plain `docker push` applies no + # force-compression. Verify BEFORE the pinned : tag becomes reachable: push to a + # transient staging tag, assert its layers, then promote onto $LCB_IMAGE_REF only on + # success — so a failed verify never poisons the immutable tag (which the guard above + # then refuses to overwrite without --force). `imagetools create` copies layers by + # digest (no blob re-upload); --prefer-index=false makes the pin the SAME manifest + # digest that was verified (not a fresh 1-entry index), and it is re-asserted below. + STAGING_REF="${LCB_IMAGE_REF%:*}:staging-${LCB_IMAGE_TAG}" + echo ">> Tagging ${LCB_LOCAL_TAG} -> ${STAGING_REF} (staging)" + docker tag "$LCB_LOCAL_TAG" "$STAGING_REF" + + echo ">> Pushing ${STAGING_REF} for verification" + docker push "$STAGING_REF" + assert_gzip_layers "$STAGING_REF" || exit 1 + + echo ">> Verified; promoting ${STAGING_REF} -> ${LCB_IMAGE_REF}" + docker buildx imagetools create --prefer-index=false --tag "$LCB_IMAGE_REF" "$STAGING_REF" + assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 + echo " (transient ${STAGING_REF##*:} tag left in the registry; delete via your registry UI/API if desired.)" +else if [[ -z "${HF_TOKEN:-}" ]]; then - echo "error: HF_TOKEN is required to build the image." >&2 + echo "error: HF_TOKEN is required to build the image (or pass --no-build to push an existing one)." >&2 exit 1 fi if ! docker buildx version >/dev/null 2>&1; then - echo "error: 'docker buildx' is required for --platform builds but is not available." >&2 + echo "error: 'docker buildx' is required but is not available." >&2 exit 1 fi - # Ensure a docker-container builder exists (the default 'docker' driver cannot - # build non-native platforms). Creating it is safe and unprivileged. + # Ensure a docker-container builder exists (the default 'docker' driver can neither + # push nor build a non-host platform). Creating it is safe and unprivileged. if ! docker buildx inspect "$BUILDX_BUILDER" >/dev/null 2>&1; then echo ">> Creating buildx builder '${BUILDX_BUILDER}' (docker-container driver)" docker buildx create --name "$BUILDX_BUILDER" --driver docker-container >/dev/null @@ -124,47 +217,33 @@ if [[ -n "$PLATFORM" ]]; then echo "error: builder '${BUILDX_BUILDER}' cannot build: ${missing_platforms} — QEMU emulation for it is not registered on the host." >&2 echo " Register it once (needs --privileged), then re-run:" >&2 echo " docker run --privileged --rm tonistiigi/binfmt --install all" >&2 + echo " Or build a single arch to skip emulation: --platform linux/$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')" >&2 exit 1 fi - echo ">> Building ${LCB_IMAGE_REF} for ${PLATFORM} and pushing (buildx) ..." + echo ">> Building ${LCB_IMAGE_REF} for ${PLATFORM} (endpoints ${ENDPOINTS_SHA}) and pushing (buildx) ..." docker buildx build \ --builder "$BUILDX_BUILDER" \ --platform "$PLATFORM" \ -f "${SCRIPT_DIR}/lcb_serve.dockerfile" \ --secret id=HF_TOKEN,env=HF_TOKEN \ + --build-arg "ENDPOINTS_SHA=${ENDPOINTS_SHA}" \ -t "$LCB_IMAGE_REF" \ --provenance=false \ --output "type=image,push=true,compression=gzip,force-compression=true,oci-mediatypes=false" \ "$SCRIPT_DIR" -else - # ------------------------------------------------------------------------ - # Native path: plain docker build for the host arch, then tag and push. - # ------------------------------------------------------------------------ - if [[ "$NO_BUILD" -eq 0 ]]; then - if [[ -z "${HF_TOKEN:-}" ]]; then - echo "error: HF_TOKEN is required to build the image (or pass --no-build to push an existing one)." >&2 - exit 1 - fi - - echo ">> Building ${LCB_LOCAL_TAG} ..." - docker build \ - -f "${SCRIPT_DIR}/lcb_serve.dockerfile" \ - --secret id=HF_TOKEN,env=HF_TOKEN \ - -t "$LCB_LOCAL_TAG" \ - "$SCRIPT_DIR" - fi - - echo ">> Tagging ${LCB_LOCAL_TAG} -> ${LCB_IMAGE_REF}" - docker tag "$LCB_LOCAL_TAG" "$LCB_IMAGE_REF" - - echo ">> Pushing ${LCB_IMAGE_REF}" - docker push "$LCB_IMAGE_REF" + # Post-publish confirmation, not a gate: the build forces gzip layers so the pushed + # image is enroot-safe by construction (#467). Known caveat: a transient inspect flake + # here fails the job after the tag is live; re-run with --force to republish. (Only the + # --no-build path needs the staging→verify→promote dance, since it can't force gzip.) + assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 fi +echo +echo "Pushed: ${LCB_IMAGE_REF}" echo echo "Done. Consumers can now pull with:" echo " LCB_IMAGE_REGISTRY=${LCB_IMAGE_REGISTRY%/} \\" [[ "$LCB_IMAGE_NAME" != "lcb-service" ]] && echo " LCB_IMAGE_NAME=${LCB_IMAGE_NAME} \\" -[[ "$LCB_IMAGE_TAG" != "release_v6" ]] && echo " LCB_IMAGE_TAG=${LCB_IMAGE_TAG} \\" +echo " LCB_IMAGE_TAG=${LCB_IMAGE_TAG} \\" echo " ./pull_image.sh"