From 10d7fbede54e3ee0e0ecd06687831f756a0408c4 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:17:42 -0500 Subject: [PATCH 1/8] Testing workflow to push images Signed-off-by: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> --- .github/workflows/publish-image.yml | 82 +++++++++++ scripts/push_docker_image.sh | 208 ++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 .github/workflows/publish-image.yml create mode 100755 scripts/push_docker_image.sh diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 000000000..1474eb29a --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,82 @@ +# Manually-triggered build+push of the endpoints client image to GHCR. +# +# Run it from the Actions tab ("Run workflow"): pick the branch/tag in the UI (that +# selection is github.sha), or type an explicit SHA/branch/tag in the `ref` input. +# The image is pushed to ghcr.io/mlcommons/endpoints tagged with the short commit +# SHA and the ref name, using the automatic GITHUB_TOKEN (no stored secret needed). +name: Publish client image + +on: + workflow_dispatch: + inputs: + ref: + description: "Git ref to build (branch/tag/SHA). Blank = the ref selected above." + required: false + type: string + platforms: + description: "Target platform(s)" + required: true + default: linux/amd64 + type: choice + options: + - linux/amd64 + - linux/amd64,linux/arm64 + provision_dsr1: + description: "Bake in the DeepSeek-R1 accuracy evaluator (heavier; needs build-time network)" + required: true + default: "1" + type: choice + options: + - "1" + - "0" + no_cache: + description: "Build with --no-cache (clean/reproducible, slower)" + required: true + default: "true" + type: choice + options: + - "true" + - "false" + +# GITHUB_TOKEN needs packages:write to push to the org's GHCR package. +permissions: + contents: read + packages: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout selected ref + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Check out the ref *name* (not github.sha) so a local branch head exists + # and the script can resolve/tag it; blank input falls back to the + # dispatched branch/tag. Full history for `git rev-parse --short`. + ref: ${{ inputs.ref || github.ref_name }} + fetch-depth: 0 + + # Registers QEMU binfmt handlers so the docker-container builder can build + # the non-native arch when platforms includes linux/arm64. + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Logs in as the user who triggered the run; the password is this run's + # automatic GITHUB_TOKEN (scoped by the permissions block above). + - name: Log in to GitHub Container Registry + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + env: + ENDPOINTS_REF: ${{ inputs.ref || github.ref_name }} + PLATFORM: ${{ inputs.platforms }} + PROVISION_DSR1: ${{ inputs.provision_dsr1 }} + NO_CACHE: ${{ inputs.no_cache == 'true' && '1' || '0' }} + run: ./scripts/push_docker_image.sh diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh new file mode 100755 index 000000000..dde371e14 --- /dev/null +++ b/scripts/push_docker_image.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# push_docker_image.sh — build and push the endpoints client image to a registry. +# +# Builds scripts/Dockerfile.dev at a given git ref and pushes it to GitHub +# Container Registry (default: ghcr.io/mlcommons/endpoints), tagged with both the +# short commit SHA and the ref name so consumers can pin either. +# +# Prerequisites: +# - docker with buildx (the script provisions a docker-container builder; the +# default 'docker' driver can neither --push nor build multi-platform). +# - Authenticated to the target registry. Either run `docker login ghcr.io` +# yourself, or export GHCR_USER + GHCR_TOKEN (a GitHub PAT / GITHUB_TOKEN with +# write:packages) and this script logs in for you. +# +# Usage: +# ./scripts/push_docker_image.sh # build+push current HEAD (linux/amd64) +# ENDPOINTS_REF=v1.2.0 ./scripts/push_docker_image.sh # build+push a specific tag/branch/sha +# ./scripts/push_docker_image.sh --ref main # same, via flag +# ./scripts/push_docker_image.sh --multi-arch # linux/amd64,linux/arm64 manifest list +# ./scripts/push_docker_image.sh --platform linux/arm64 # single non-native arch +# ./scripts/push_docker_image.sh --cache # allow layer cache (default: --no-cache) +# ./scripts/push_docker_image.sh --multi-arch --no-push # dry build (validate both arches, publish nothing) +# ./scripts/push_docker_image.sh --allow-dirty # build HEAD despite uncommitted tracked changes +# +# Multi-arch note: --multi-arch (or --platform with a comma) pushes a manifest +# list usable on BOTH x86 and arm hosts; Docker pulls the matching arch. The +# non-native leg needs QEMU registered on the host once: +# docker run --privileged --rm tonistiigi/binfmt --install all +# Dockerfile.dev's DSR1 provisioning runs under emulation and is MUCH slower. +# +# Environment variables (flags take precedence): +# IMAGE full image ref w/o tag (default: ghcr.io/mlcommons/endpoints) +# ENDPOINTS_REF git ref to build (default: current HEAD; checked out if it differs) +# PLATFORM target platform(s) (default: linux/amd64) +# DOCKERFILE Dockerfile path (default: scripts/Dockerfile.dev) +# NO_CACHE 1=--no-cache, 0=cache (default: 1) +# ALLOW_DIRTY 1=build HEAD with uncommitted tracked changes (default: 0) +# PROVISION_DSR1 passthrough build-arg (default: Dockerfile default = 1) +# GHCR_USER/GHCR_TOKEN optional registry login (fallback: GITHUB_ACTOR/GITHUB_TOKEN) +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILDX_BUILDER="endpoints-buildx" + +usage() { + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//; /^set -euo/d' +} + +IMAGE="${IMAGE:-ghcr.io/mlcommons/endpoints}" +ENDPOINTS_REF="${ENDPOINTS_REF:-}" +PLATFORM="${PLATFORM:-linux/amd64}" +DOCKERFILE="${DOCKERFILE:-scripts/Dockerfile.dev}" +NO_CACHE="${NO_CACHE:-1}" +ALLOW_DIRTY="${ALLOW_DIRTY:-0}" +PUSH=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + --ref) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --ref requires a value" >&2; exit 1; } + ENDPOINTS_REF="$2"; shift ;; + --ref=*) ENDPOINTS_REF="${1#*=}" ;; + --image) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --image requires a value" >&2; exit 1; } + IMAGE="$2"; shift ;; + --image=*) IMAGE="${1#*=}" ;; + --platform) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --platform requires a value, e.g. linux/arm64" >&2; exit 1; } + PLATFORM="$2"; shift ;; + --platform=*) PLATFORM="${1#*=}" ;; + --multi-arch) PLATFORM="linux/amd64,linux/arm64" ;; + --no-cache) NO_CACHE=1 ;; + --cache) NO_CACHE=0 ;; + --no-push) PUSH=0 ;; + --allow-dirty) ALLOW_DIRTY=1 ;; + --dockerfile) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --dockerfile requires a value" >&2; exit 1; } + DOCKERFILE="$2"; shift ;; + --dockerfile=*) DOCKERFILE="${1#*=}" ;; + -h | --help) usage; exit 0 ;; + *) echo "error: unknown argument '$1'" >&2; usage; exit 1 ;; + esac + shift +done + +cd "$REPO_ROOT" + +if ! docker buildx version >/dev/null 2>&1; then + echo "error: 'docker buildx' is required but not available." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Optional registry login. Skipped silently if no token is provided (assumes the +# caller already ran `docker login`). Registry host is the first path segment of +# IMAGE (e.g. ghcr.io from ghcr.io/mlcommons/endpoints). +# --------------------------------------------------------------------------- +REGISTRY_HOST="${IMAGE%%/*}" +LOGIN_USER="${GHCR_USER:-${GITHUB_ACTOR:-}}" +LOGIN_TOKEN="${GHCR_TOKEN:-${GITHUB_TOKEN:-}}" +if [[ "$PUSH" == "1" && -n "$LOGIN_TOKEN" ]]; then + echo ">> Logging in to ${REGISTRY_HOST} as ${LOGIN_USER:-}" + echo "$LOGIN_TOKEN" | docker login "$REGISTRY_HOST" -u "${LOGIN_USER:-x-access-token}" --password-stdin +fi + +# --------------------------------------------------------------------------- +# Resolve the ref to build. If ENDPOINTS_REF is set and points at a different +# commit than the current HEAD, check it out (requiring a clean tree) and restore +# the original ref on exit so the working copy is left as we found it. +# --------------------------------------------------------------------------- +ORIGINAL_REF="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)" +[[ "$ORIGINAL_REF" == "HEAD" ]] && ORIGINAL_REF="$(git rev-parse HEAD)" + +restore_ref() { + if [[ -n "${DID_CHECKOUT:-}" && -n "$ORIGINAL_REF" ]]; then + echo ">> Restoring original ref ${ORIGINAL_REF}" + git checkout --quiet "$ORIGINAL_REF" + fi +} +trap restore_ref EXIT + +# Uncommitted changes to TRACKED files (staged or unstaged) make the working tree +# differ from HEAD. Untracked files are ignored here: they are not part of any commit +# and .dockerignore governs the build context. +tree_has_tracked_changes() { ! git diff --quiet HEAD 2>/dev/null; } + +if [[ -n "$ENDPOINTS_REF" ]]; then + # Fall back to the remote-tracking ref: after actions/checkout lands a detached + # SHA, a bare branch name like 'main' exists only as origin/main, so the plain + # resolve would fail and abort the CI default-dispatch path. + target_sha="$(git rev-parse --verify "${ENDPOINTS_REF}^{commit}" 2>/dev/null \ + || git rev-parse --verify "origin/${ENDPOINTS_REF}^{commit}" 2>/dev/null || true)" + [[ -n "$target_sha" ]] || { echo "error: '$ENDPOINTS_REF' is not a valid git ref." >&2; exit 1; } + if [[ "$target_sha" != "$(git rev-parse HEAD)" ]]; then + if tree_has_tracked_changes; then + echo "error: working tree has uncommitted changes to tracked files; commit/stash before building a different ref ($ENDPOINTS_REF)." >&2 + exit 1 + fi + echo ">> Checking out ${ENDPOINTS_REF}" + git checkout --quiet "$ENDPOINTS_REF" + DID_CHECKOUT=1 + fi +else + # Tag the built image with the current branch/ref name for readability. + ENDPOINTS_REF="$ORIGINAL_REF" +fi + +SHORT_SHA="$(git rev-parse --short HEAD)" +# Docker tags allow only [A-Za-z0-9_.-], must not start with '.' or '-', and cap at +# 128 chars. Map every other char (e.g. '/', '+', '~', '#') to '-' so a legitimate +# git ref name can't yield an invalid tag that fails `-t` only after the whole build. +REF_TAG="$(printf '%s' "$ENDPOINTS_REF" | tr -c 'A-Za-z0-9_.-' '-')" +[[ "$REF_TAG" == [A-Za-z0-9_]* ]] || REF_TAG="_${REF_TAG}" +REF_TAG="${REF_TAG:0:128}" + +# Building the working tree as-is (no distinct ref checked out): don't publish a +# :$SHORT_SHA tag whose '.' build context doesn't match commit $SHORT_SHA. +if [[ -z "${DID_CHECKOUT:-}" && "$ALLOW_DIRTY" != "1" ]] && tree_has_tracked_changes; then + dirty_msg="working tree has uncommitted changes to tracked files; :${SHORT_SHA} would not match commit ${SHORT_SHA}" + if [[ "$PUSH" == "1" ]]; then + echo "error: ${dirty_msg}." >&2 + echo " Commit/stash the changes, or pass --allow-dirty to publish anyway." >&2 + exit 1 + fi + echo ">> warning: ${dirty_msg} (dry run; nothing pushed)." >&2 +fi + +# --------------------------------------------------------------------------- +# Ensure a docker-container builder (supports --push and multi-platform). +# --------------------------------------------------------------------------- +if ! docker buildx inspect "$BUILDX_BUILDER" >/dev/null 2>&1; then + echo ">> Creating buildx builder '${BUILDX_BUILDER}' (docker-container driver)" + docker buildx create --name "$BUILDX_BUILDER" --driver docker-container >/dev/null +fi + +if [[ "$PLATFORM" == *,* ]]; then + echo ">> Multi-arch build for ${PLATFORM} — requires QEMU for non-native archs" + echo " (register once with: docker run --privileged --rm tonistiigi/binfmt --install all)" +fi + +BUILD_ARGS=() +[[ "$NO_CACHE" == "1" ]] && BUILD_ARGS+=(--no-cache) +[[ -n "${PROVISION_DSR1:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_DSR1=${PROVISION_DSR1}") +# With --no-push we build to cache and discard (validation only). A multi-platform +# result cannot be --load-ed into the local store, so no output flag is added. +[[ "$PUSH" == "1" ]] && BUILD_ARGS+=(--push) + +BUILD_VERB=$([[ "$PUSH" == "1" ]] && echo "and pushing" || echo "(dry run, --no-push)") +echo ">> Building ${IMAGE}:${SHORT_SHA} (ref: ${ENDPOINTS_REF}) for ${PLATFORM} ${BUILD_VERB}" +docker buildx build \ + --builder "$BUILDX_BUILDER" \ + --platform "$PLATFORM" \ + -f "$DOCKERFILE" \ + -t "${IMAGE}:${SHORT_SHA}" \ + -t "${IMAGE}:${REF_TAG}" \ + ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} \ + . + +echo +if [[ "$PUSH" == "1" ]]; then + echo "Done. Pushed:" + echo " ${IMAGE}:${SHORT_SHA}" + echo " ${IMAGE}:${REF_TAG}" + echo "Pull with:" + echo " docker pull ${IMAGE}:${SHORT_SHA}" +else + echo "Done. Dry build succeeded for ${PLATFORM} (nothing pushed)." +fi From 097ee8735b3734c53e114a15037ac355d2c9f843 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:14:47 -0500 Subject: [PATCH 2/8] Add metadata Signed-off-by: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> --- scripts/push_docker_image.sh | 30 +++++++++++++++++++ .../livecodebench/lcb_serve.dockerfile | 9 ++++++ .../evaluation/livecodebench/push_image.sh | 20 +++++++++++-- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh index dde371e14..465d00ee5 100755 --- a/scripts/push_docker_image.sh +++ b/scripts/push_docker_image.sh @@ -36,6 +36,7 @@ # NO_CACHE 1=--no-cache, 0=cache (default: 1) # ALLOW_DIRTY 1=build HEAD with uncommitted tracked changes (default: 0) # PROVISION_DSR1 passthrough build-arg (default: Dockerfile default = 1) +# IMAGE_DESCRIPTION GHCR package-page description (default: "endpoints client image (ref , commit )") # GHCR_USER/GHCR_TOKEN optional registry login (fallback: GITHUB_ACTOR/GITHUB_TOKEN) set -euo pipefail @@ -181,10 +182,39 @@ fi BUILD_ARGS=() [[ "$NO_CACHE" == "1" ]] && BUILD_ARGS+=(--no-cache) [[ -n "${PROVISION_DSR1:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_DSR1=${PROVISION_DSR1}") + +# Don't attach SLSA provenance attestations. buildx adds them by default on --push, +# which surfaces a spurious `unknown/unknown` entry on the GHCR package page and +# wraps even a single-arch build in an image index. Opting out yields a clean, +# single-manifest artifact; the source/revision/version annotations below preserve +# the "which commit built this" traceability the attestation would have carried. +BUILD_ARGS+=(--provenance=false) + # With --no-push we build to cache and discard (validation only). A multi-platform # result cannot be --load-ed into the local store, so no output flag is added. [[ "$PUSH" == "1" ]] && BUILD_ARGS+=(--push) +# OCI annotations for the GHCR package page (description + repo/commit provenance). +# --annotation requires the build to actually produce the component named by the +# level prefix, so gate on --push and pick the level by artifact shape: a multi-arch +# build pushes an image index (GHCR reads the description from the index); a +# single-arch build (provenance off, above) pushes a lone manifest, for which +# `index:` errors with "index annotations not supported for single platform export". +if [[ "$PUSH" == "1" ]]; then + if [[ "$PLATFORM" == *,* ]]; then + ANNOTATION_LEVEL="index" + else + ANNOTATION_LEVEL="manifest" + fi + IMAGE_DESCRIPTION="${IMAGE_DESCRIPTION:-endpoints client image (ref ${ENDPOINTS_REF}, commit ${SHORT_SHA})}" + BUILD_ARGS+=( + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.source=https://github.com/mlcommons/endpoints" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.revision=${SHORT_SHA}" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.version=${ENDPOINTS_REF}" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.description=${IMAGE_DESCRIPTION}" + ) +fi + BUILD_VERB=$([[ "$PUSH" == "1" ]] && echo "and pushing" || echo "(dry run, --no-push)") echo ">> Building ${IMAGE}:${SHORT_SHA} (ref: ${ENDPOINTS_REF}) for ${PLATFORM} ${BUILD_VERB}" docker buildx build \ diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile index 83b1b79db..a8f80c95a 100644 --- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile +++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile @@ -63,6 +63,15 @@ COPY _server.py /app/server.py # Make lcb_serve.py available as a module ENV PYTHONPATH="/app" +# Provenance: the endpoints repo commit this image was built from, supplied by +# push_image.sh via --build-arg. Recorded as a config LABEL (not a manifest +# annotation) so it survives the oci-mediatypes=false push and is visible in +# `docker inspect`. Declared after the COPYs so a new SHA only rebuilds this +# metadata layer, never the expensive dataset-generation stage above. +ARG ENDPOINTS_SHA=unknown +LABEL org.opencontainers.image.source="https://github.com/mlcommons/endpoints" \ + org.opencontainers.image.revision="${ENDPOINTS_SHA}" + # Launch the WebSocket server with long-running connection support # Default port 13835 # - timeout-keep-alive: Allow connections to stay open for hours diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index 3e92af49b..f9738d4f6 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -83,6 +83,20 @@ done # shellcheck source=_image_env.sh source "${SCRIPT_DIR}/_image_env.sh" +# ---------------------------------------------------------------------------- +# Provenance: the endpoints repo commit this image is built from, baked into the +# image config as an OCI label by both build paths below. A config LABEL (not a +# manifest --annotation) is deliberate: the buildx push sets oci-mediatypes=false, +# and Docker-media-type manifests have no annotations field, whereas config labels +# are representable in both formats and survive `docker inspect`. Scope the dirty +# check to this build context so unrelated edits elsewhere in the repo don't mark +# the image -dirty. +# ---------------------------------------------------------------------------- +ENDPOINTS_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" +if [[ "$ENDPOINTS_SHA" != "unknown" ]] && ! git -C "$SCRIPT_DIR" diff --quiet HEAD -- "$SCRIPT_DIR" 2>/dev/null; then + ENDPOINTS_SHA="${ENDPOINTS_SHA}-dirty" +fi + # ---------------------------------------------------------------------------- # Cross-architecture path: build with buildx and push directly to the registry. # A non-native image cannot be `docker load`-ed into the local store, so buildx @@ -127,12 +141,13 @@ if [[ -n "$PLATFORM" ]]; then exit 1 fi - echo ">> Building ${LCB_IMAGE_REF} for ${PLATFORM} and pushing (buildx) ..." + echo ">> Building ${LCB_IMAGE_REF} for ${PLATFORM} (endpoints ${ENDPOINTS_SHA}) and pushing (buildx) ..." docker buildx build \ --builder "$BUILDX_BUILDER" \ --platform "$PLATFORM" \ -f "${SCRIPT_DIR}/lcb_serve.dockerfile" \ --secret id=HF_TOKEN,env=HF_TOKEN \ + --build-arg "ENDPOINTS_SHA=${ENDPOINTS_SHA}" \ -t "$LCB_IMAGE_REF" \ --provenance=false \ --output "type=image,push=true,compression=gzip,force-compression=true,oci-mediatypes=false" \ @@ -147,10 +162,11 @@ else exit 1 fi - echo ">> Building ${LCB_LOCAL_TAG} ..." + echo ">> Building ${LCB_LOCAL_TAG} (endpoints ${ENDPOINTS_SHA}) ..." docker build \ -f "${SCRIPT_DIR}/lcb_serve.dockerfile" \ --secret id=HF_TOKEN,env=HF_TOKEN \ + --build-arg "ENDPOINTS_SHA=${ENDPOINTS_SHA}" \ -t "$LCB_LOCAL_TAG" \ "$SCRIPT_DIR" fi From da7d3a9119ef9d33c7550ed2a62555bae7637a57 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:31:35 -0500 Subject: [PATCH 3/8] More fixups. Signed-off-by: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> --- .../evaluation/livecodebench/README.md | 35 ++++---- .../evaluation/livecodebench/_image_env.sh | 13 ++- .../evaluation/livecodebench/pull_image.sh | 12 +-- .../evaluation/livecodebench/push_image.sh | 80 ++++++++++++++++--- 4 files changed, 105 insertions(+), 35 deletions(-) diff --git a/src/inference_endpoint/evaluation/livecodebench/README.md b/src/inference_endpoint/evaluation/livecodebench/README.md index a5b7fb5dd..07e6cf42a 100644 --- a/src/inference_endpoint/evaluation/livecodebench/README.md +++ b/src/inference_endpoint/evaluation/livecodebench/README.md @@ -182,14 +182,14 @@ consumer can **pull and run it with no `HF_TOKEN` and no rebuild**. Two scripts in this directory wrap the docker build/tag/push/pull steps. Both resolve the image reference from environment variables (shared via `_image_env.sh`): -| Variable | Required | Default | Meaning | -| -------------------- | -------- | -------------------- | --------------------------------------------------------------------------- | -| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | -| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | -| `LCB_IMAGE_TAG` | no | `release_v6` | tag; defaults to the baked-in dataset version so the tag is self-describing | -| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | +| Variable | Required | Default | Meaning | +| -------------------- | -------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | +| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | +| `LCB_IMAGE_TAG` | push: no · pull: yes | endpoints short SHA | image tag; `push_image.sh` defaults it to the endpoints commit SHA (one immutable tag per build). Pull must name the build. | +| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | -The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`. +The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`. `push_image.sh` sets `LCB_IMAGE_TAG` to the endpoints commit short SHA by default, so each build publishes an immutable `…/lcb-service:` — there is no moving `latest`/`release_v6` tag. The same SHA is also baked into the image as the `org.opencontainers.image.revision` label. #### Push (maintainer) @@ -200,8 +200,14 @@ Requires `docker login dhi.io` (base images) and `docker login` to your target r LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= \ ./push_image.sh -# Or push an already-built local image without rebuilding: -LCB_IMAGE_REGISTRY=myregistry.com/team ./push_image.sh --no-build +# Or push an already-built local image without rebuilding (name the build explicitly): +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no-build +``` + +Each build publishes an immutable `:` tag, so **re-pushing an existing `:` is refused** to protect it. Pass `--force` to overwrite deliberately (e.g. re-running a partially failed push): + +```bash +LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --force ``` **Cross-architecture builds.** To build for an architecture other than the host's (e.g. build `arm64` on an @@ -227,11 +233,12 @@ docker run --privileged --rm tonistiigi/binfmt --install all #### Pull (consumer / eval side) -No `HF_TOKEN` needed. By default the image is re-tagged locally as `lcb-service:latest`, so the -[hardened run command](#hardened-run-command) and the scorer's `ws://localhost:13835/evaluate` expectation work unchanged. +No `HF_TOKEN` needed. Set `LCB_IMAGE_TAG` to the build (short SHA) you want to pull. By default the image is re-tagged +locally as `lcb-service:latest`, so the [hardened run command](#hardened-run-command) and the scorer's +`ws://localhost:13835/evaluate` expectation work unchanged. ```bash -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh ``` ### (Only if using enroot) Generating a .sqsh file for enroot @@ -240,8 +247,8 @@ The pull script can produce an enroot `.sqsh` from the pulled image with the `-- [enroot](https://github.com/NVIDIA/enroot/tree/main) (e.g. SLURM clusters): ```bash -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh # writes lcb_service.sqsh -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh out.sqsh # custom output path +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh # writes lcb_service.sqsh +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh out.sqsh # custom output path ``` This runs `enroot import --output dockerd://${LCB_IMAGE_REF}` on the just-pulled image. Running the service via enroot is diff --git a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh index ba615cb63..32464b83e 100644 --- a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh +++ b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh @@ -8,7 +8,7 @@ # Inputs (environment variables): # LCB_IMAGE_REGISTRY (required) registry + namespace, e.g. myregistry.com/team # LCB_IMAGE_NAME (optional) image repo name (default: lcb-service) -# LCB_IMAGE_TAG (optional) tag (default: release_v6) +# LCB_IMAGE_TAG (required) tag — push_image.sh defaults it to the endpoints short SHA # LCB_LOCAL_TAG (optional) local tag used by run/scorer (default: lcb-service:latest) # # Exports: @@ -23,8 +23,15 @@ if [[ -z "${LCB_IMAGE_REGISTRY:-}" ]]; then fi LCB_IMAGE_NAME="${LCB_IMAGE_NAME:-lcb-service}" -# Defaults to the baked-in dataset version so the artifact is self-describing. -LCB_IMAGE_TAG="${LCB_IMAGE_TAG:-release_v6}" +# Images are tagged by the endpoints commit SHA (one immutable tag per build), so +# there is no channel default. push_image.sh sets this to the SHA automatically; +# a consumer pulling must name the specific build. +if [[ -z "${LCB_IMAGE_TAG:-}" ]]; then + echo "error: LCB_IMAGE_TAG is not set." >&2 + echo " push_image.sh defaults it to the endpoints commit SHA; for pull, set it to" >&2 + echo " the build you want, e.g. LCB_IMAGE_TAG=\$(git rev-parse --short HEAD)" >&2 + return 1 2>/dev/null || exit 1 +fi LCB_LOCAL_TAG="${LCB_LOCAL_TAG:-lcb-service:latest}" # Strip any trailing slash on the registry to avoid a double slash in the ref. diff --git a/src/inference_endpoint/evaluation/livecodebench/pull_image.sh b/src/inference_endpoint/evaluation/livecodebench/pull_image.sh index dc679d780..3b3127d2b 100755 --- a/src/inference_endpoint/evaluation/livecodebench/pull_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/pull_image.sh @@ -6,13 +6,13 @@ # lcb-service:latest so the hardened run command in README.md and the scorer's # ws://localhost:13835/evaluate expectation work unchanged. # -# Usage: -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh # also create lcb_service.sqsh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh out.sqsh # enroot import to out.sqsh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --no-local-tag # skip the lcb-service:latest tag +# Usage (images are tagged by short SHA, so LCB_IMAGE_TAG selects the build): +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh # also create lcb_service.sqsh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh out.sqsh # enroot import to out.sqsh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --no-local-tag # skip the lcb-service:latest tag # -# Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY required). +# Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY + LCB_IMAGE_TAG required). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index f9738d4f6..c83ba77ca 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -5,13 +5,17 @@ # build time. Pushing the built image lets consumers pull-and-run with no HF_TOKEN # and no rebuild (see pull_image.sh). # +# The image is tagged by the endpoints commit short SHA — one immutable tag per +# build. LCB_IMAGE_TAG overrides it. There is no moving channel tag. +# # Prerequisites: # - docker login dhi.io (base images are Docker Hardened Images) # - docker login (target for the push) # # Usage: # LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./push_image.sh --no-build # push existing local image +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no-build # push existing local image +# ... --force # overwrite an existing : tag (default: refuse) # # Cross-architecture build (e.g. build arm64 on an x86 node, or a multi-arch manifest): # LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh --platform linux/arm64 @@ -54,10 +58,12 @@ unsupported_platforms() { platform_supported() { [[ -z "$(unsupported_platforms "$1" "$2")" ]]; } NO_BUILD=0 +FORCE=0 PLATFORM="${LCB_IMAGE_PLATFORM:-}" while [[ $# -gt 0 ]]; do case "$1" in --no-build) NO_BUILD=1 ;; + --force) FORCE=1 ;; --platform) if [[ -z "${2:-}" || "$2" == --* ]]; then echo "error: --platform requires a value, e.g. linux/arm64" >&2 @@ -80,23 +86,71 @@ while [[ $# -gt 0 ]]; do shift done -# shellcheck source=_image_env.sh -source "${SCRIPT_DIR}/_image_env.sh" - # ---------------------------------------------------------------------------- -# Provenance: the endpoints repo commit this image is built from, baked into the -# image config as an OCI label by both build paths below. A config LABEL (not a -# manifest --annotation) is deliberate: the buildx push sets oci-mediatypes=false, -# and Docker-media-type manifests have no annotations field, whereas config labels -# are representable in both formats and survive `docker inspect`. Scope the dirty -# check to this build context so unrelated edits elsewhere in the repo don't mark -# the image -dirty. +# Provenance: the endpoints repo commit this image is built from. It is both baked +# into the image config as an OCI label (see --build-arg below) AND used as the +# image tag — one immutable tag per build. A config LABEL (not a manifest +# --annotation) is deliberate: the buildx push sets oci-mediatypes=false, and +# Docker-media-type manifests have no annotations field, whereas config labels are +# representable in both formats and survive `docker inspect`. Scope the dirty check +# to this build context so unrelated edits elsewhere in the repo don't mark it -dirty. # ---------------------------------------------------------------------------- ENDPOINTS_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" if [[ "$ENDPOINTS_SHA" != "unknown" ]] && ! git -C "$SCRIPT_DIR" diff --quiet HEAD -- "$SCRIPT_DIR" 2>/dev/null; then ENDPOINTS_SHA="${ENDPOINTS_SHA}-dirty" fi +# Tag the image by the endpoints commit SHA (LCB_IMAGE_TAG overrides). The auto SHA +# tag is only trustworthy when the tree is clean AND we build now, so when the tag is +# not given explicitly, refuse the cases where : would misrepresent the image: +# - no git → nothing to name the image by +# - --no-build → pushes a pre-built local image whose baked revision label may be +# a different commit than HEAD; : would lie about provenance +# - dirty context → :-dirty is a moving, non-reproducible tag +# An explicit LCB_IMAGE_TAG is the escape hatch for all three. Set before sourcing so +# _image_env.sh builds LCB_IMAGE_REF from it. +if [[ -z "${LCB_IMAGE_TAG:-}" ]]; then + if [[ "$ENDPOINTS_SHA" == "unknown" ]]; then + echo "error: cannot determine a short SHA to tag the image (no git); set LCB_IMAGE_TAG." >&2 + exit 1 + fi + if [[ "$NO_BUILD" -eq 1 ]]; then + echo "error: --no-build pushes a pre-built local image whose baked revision may not match" >&2 + echo " HEAD (${ENDPOINTS_SHA}); set LCB_IMAGE_TAG explicitly to name it." >&2 + exit 1 + fi + if [[ "$ENDPOINTS_SHA" == *-dirty ]]; then + echo "error: build context has uncommitted changes; :${ENDPOINTS_SHA} is a moving tag." >&2 + echo " Commit the changes, or set LCB_IMAGE_TAG explicitly to push anyway." >&2 + exit 1 + fi +fi +export LCB_IMAGE_TAG="${LCB_IMAGE_TAG:-$ENDPOINTS_SHA}" + +# shellcheck source=_image_env.sh +source "${SCRIPT_DIR}/_image_env.sh" + +# Refuse to overwrite an existing remote tag unless --force: the SHA tag is meant to +# be immutable, so a second push to the same ref would silently replace a published +# image. `imagetools inspect` queries the registry (no layer pull). Fail CLOSED: a +# clean exit means the tag exists (block); a non-zero exit is trusted as "absent" ONLY +# when the output confirms not-found. Any other failure (buildx plugin missing, auth +# denied, registry/network error) blocks too — otherwise the guard silently no-ops on +# exactly the hosts/creds where it can't verify, and immutability goes unenforced. +# --force skips the check entirely. +if [[ "$FORCE" -eq 0 ]]; then + if inspect_out="$(docker buildx imagetools inspect "$LCB_IMAGE_REF" 2>&1)"; then + echo "error: ${LCB_IMAGE_REF} already exists in the registry." >&2 + echo " The SHA tag is meant to be immutable; re-run with --force to overwrite it." >&2 + exit 1 + elif ! grep -qiE 'not found|manifest unknown|manifest_unknown|name_unknown|no such manifest' <<<"$inspect_out"; then + echo "error: could not verify whether ${LCB_IMAGE_REF} already exists:" >&2 + printf ' %s\n' "$inspect_out" >&2 + echo " Fix registry access (or install docker buildx), or re-run with --force to skip this check." >&2 + exit 1 + fi +fi + # ---------------------------------------------------------------------------- # Cross-architecture path: build with buildx and push directly to the registry. # A non-native image cannot be `docker load`-ed into the local store, so buildx @@ -178,9 +232,11 @@ else docker push "$LCB_IMAGE_REF" fi +echo +echo "Pushed: ${LCB_IMAGE_REF}" echo echo "Done. Consumers can now pull with:" echo " LCB_IMAGE_REGISTRY=${LCB_IMAGE_REGISTRY%/} \\" [[ "$LCB_IMAGE_NAME" != "lcb-service" ]] && echo " LCB_IMAGE_NAME=${LCB_IMAGE_NAME} \\" -[[ "$LCB_IMAGE_TAG" != "release_v6" ]] && echo " LCB_IMAGE_TAG=${LCB_IMAGE_TAG} \\" +echo " LCB_IMAGE_TAG=${LCB_IMAGE_TAG} \\" echo " ./pull_image.sh" From 75b84bd89582f88fe8a4bcdc1d2720c2d9b8bd94 Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:05:24 -0500 Subject: [PATCH 4/8] fix(livecodebench): force gzip layers on image push for enroot/pyxis (#467) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The published ghcr.io/mlcommons/endpoints:*-livecodebench image inherited zstd-compressed base layers from the dhi.io Docker Hardened Image base. enroot/ pyxis does not route the docker-namespaced zstd media type to a decompressor and hands the raw blob to tar, failing with "tar: This does not look like a tar archive" (mlcommons/endpoints#467). - scripts/lib_registry.sh: assert_gzip_layers — inspects the pushed manifest and fails on any non-gzip layer (walks a manifest list; docker + oci media types). - scripts/push_docker_image.sh: push via --output type=image,push=true,compression=gzip,force-compression=true so base layers are re-compressed to gzip; verify after push. - livecodebench/push_image.sh: verify layers after both the --platform and native push paths. verified a zstd build reproduces the tar error at pyxis import; the force-compression gzip build imports and runs cleanly. --- scripts/lib_registry.sh | 49 +++++++++++++++++++ scripts/push_docker_image.sh | 15 +++++- .../evaluation/livecodebench/push_image.sh | 6 +++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 scripts/lib_registry.sh diff --git a/scripts/lib_registry.sh b/scripts/lib_registry.sh new file mode 100644 index 000000000..3aa7b587c --- /dev/null +++ b/scripts/lib_registry.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# lib_registry.sh — shared registry helpers for the image push scripts. +# +# Not meant to be executed directly. `source` it from a push script. + +# assert_gzip_layers REF — fail unless every layer of REF (walking a manifest list) +# is gzip-compressed. enroot/pyxis routes a layer to its decompressor by media type +# and does not recognise the docker-namespaced zstd type +# (application/vnd.docker.image.rootfs.diff.tar.zstd) that buildx emits for zstd base +# layers under oci-mediatypes=false; it hands the raw blob to tar, which dies with +# "tar: This does not look like a tar archive" (mlcommons/endpoints#467). Reads registry +# metadata only (no blob pull), via `docker buildx imagetools inspect`. Fails CLOSED: +# any inspect error blocks the publish rather than silently passing. +assert_gzip_layers() { + local ref="$1" + docker buildx imagetools inspect "$ref" --raw >/dev/null 2>&1 \ + || { echo "error: cannot inspect ${ref} to verify layer compression." >&2; return 2; } + # --raw yields either an image index (has .manifests) or a single manifest (has + # .layers). Parse with python for robustness across both shapes and media-type + # namespaces (docker + oci); descend one level for a manifest list. + python3 - "$ref" <<'PY' +import sys, json, subprocess +ref = sys.argv[1] +base = ref.split("@", 1)[0] +def raw(r): + return json.loads(subprocess.check_output( + ["docker", "buildx", "imagetools", "inspect", r, "--raw"])) +def layer_types(man): + return [layer["mediaType"] for layer in man.get("layers", [])] +top = raw(ref) +types = [] +if top.get("manifests"): # image index / manifest list + for child in top["manifests"]: + plat = child.get("platform", {}) + if plat.get("os") == "unknown" or plat.get("architecture") == "unknown": + continue # skip attestation manifests + types += layer_types(raw(f"{base}@{child['digest']}")) +else: # single-arch manifest + types = layer_types(top) +bad = sorted({mt for mt in types if not mt.endswith("gzip")}) +if bad: + sys.stderr.write( + f"error: {ref} has non-gzip layer(s): {', '.join(bad)}\n" + " enroot/pyxis cannot extract these (mlcommons/endpoints#467).\n" + " Rebuild via the buildx --platform path, which forces gzip.\n") + sys.exit(1) +print(f">> Verified: all {len(types)} layers of {ref} are gzip (enroot-safe).") +PY +} diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh index 465d00ee5..82fea19e6 100755 --- a/scripts/push_docker_image.sh +++ b/scripts/push_docker_image.sh @@ -43,6 +43,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BUILDX_BUILDER="endpoints-buildx" +# shellcheck source=lib_registry.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib_registry.sh" + usage() { sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//; /^set -euo/d' } @@ -192,7 +195,14 @@ BUILD_ARGS+=(--provenance=false) # With --no-push we build to cache and discard (validation only). A multi-platform # result cannot be --load-ed into the local store, so no output flag is added. -[[ "$PUSH" == "1" ]] && BUILD_ARGS+=(--push) +# +# On push, force EVERY layer (including inherited base-image layers) to gzip so the +# artifact is extractable by enroot/pyxis on SLURM. Without force-compression, BuildKit +# copies base layers in their original codec (zstd for some bases) and enroot import +# fails with "tar: This does not look like a tar archive" (mlcommons/endpoints#467). +# type=image,push=true is the long form of --push (the two conflict); oci-mediatypes is +# left at its default so the --annotation provenance below still applies. +[[ "$PUSH" == "1" ]] && BUILD_ARGS+=(--output "type=image,push=true,compression=gzip,force-compression=true") # OCI annotations for the GHCR package page (description + repo/commit provenance). # --annotation requires the build to actually produce the component named by the @@ -226,6 +236,9 @@ docker buildx build \ ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} \ . +# Guard against silently publishing an image enroot/pyxis cannot extract (#467). +[[ "$PUSH" == "1" ]] && { assert_gzip_layers "${IMAGE}:${SHORT_SHA}" || exit 1; } + echo if [[ "$PUSH" == "1" ]]; then echo "Done. Pushed:" diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index c83ba77ca..2eddbec81 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -129,6 +129,8 @@ export LCB_IMAGE_TAG="${LCB_IMAGE_TAG:-$ENDPOINTS_SHA}" # shellcheck source=_image_env.sh source "${SCRIPT_DIR}/_image_env.sh" +# shellcheck source=/dev/null +source "$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)/scripts/lib_registry.sh" # Refuse to overwrite an existing remote tag unless --force: the SHA tag is meant to # be immutable, so a second push to the same ref would silently replace a published @@ -206,6 +208,7 @@ if [[ -n "$PLATFORM" ]]; then --provenance=false \ --output "type=image,push=true,compression=gzip,force-compression=true,oci-mediatypes=false" \ "$SCRIPT_DIR" + assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 else # ------------------------------------------------------------------------ # Native path: plain docker build for the host arch, then tag and push. @@ -230,6 +233,9 @@ else echo ">> Pushing ${LCB_IMAGE_REF}" docker push "$LCB_IMAGE_REF" + # The native docker push can preserve zstd base layers under the containerd image + # store; block a publish enroot/pyxis cannot extract (#467). + assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 fi echo From 28a464d5f41fa7057a13acf98f2d7e3b31d8d4f8 Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:52:30 -0500 Subject: [PATCH 5/8] Make multi-arch default Signed-off-by: arekay-nv <230885705+arekay-nv@users.noreply.github.com> --- .../evaluation/livecodebench/README.md | 24 ++--- .../evaluation/livecodebench/push_image.sh | 89 ++++++++----------- 2 files changed, 51 insertions(+), 62 deletions(-) diff --git a/src/inference_endpoint/evaluation/livecodebench/README.md b/src/inference_endpoint/evaluation/livecodebench/README.md index 07e6cf42a..7e97ed332 100644 --- a/src/inference_endpoint/evaluation/livecodebench/README.md +++ b/src/inference_endpoint/evaluation/livecodebench/README.md @@ -196,7 +196,8 @@ The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_ Requires `docker login dhi.io` (base images) and `docker login` to your target registry first. ```bash -# Build (using HF_TOKEN as a build secret) and push: +# Build (using HF_TOKEN as a build secret) and push a MULTI-ARCH manifest +# (linux/amd64,linux/arm64) by default: LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= \ ./push_image.sh @@ -204,32 +205,35 @@ LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= \ LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no-build ``` +Every build goes through `docker buildx` and pushes **straight to the registry**: buildx forces gzip layers so the +image is extractable by enroot/pyxis on SLURM (see [#467](https://github.com/mlcommons/endpoints/issues/467)), and a +multi-arch image cannot be loaded into the local docker store anyway. The script auto-creates a `docker-container` +buildx builder. `--no-build` is the one exception — it tag+pushes a pre-built local image for the host arch only. + Each build publishes an immutable `:` tag, so **re-pushing an existing `:` is refused** to protect it. Pass `--force` to overwrite deliberately (e.g. re-running a partially failed push): ```bash LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --force ``` -**Cross-architecture builds.** To build for an architecture other than the host's (e.g. build `arm64` on an -`x86_64` node, or a multi-arch manifest), pass `--platform`: +**Single-arch / cross-arch builds.** Pass `--platform` to override the multi-arch default — e.g. a single arch (much +faster, no emulation) or a specific target: ```bash LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --platform linux/arm64 LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --platform linux/amd64,linux/arm64 ``` -Platform builds use `docker buildx` and push the image **straight to the registry** (a non-native image cannot be -loaded into the local docker store, so there is no `--no-build` for this path). The script auto-creates a -`docker-container` buildx builder. The target architecture must have **QEMU emulation registered on the host**, -a one-time step that needs `--privileged`: +Building an architecture other than the host's needs **QEMU emulation registered on the host**, a one-time step that +needs `--privileged`: ```bash docker run --privileged --rm tonistiigi/binfmt --install all ``` -> ⚠️ The dataset-generation step runs under emulation when cross-building, which is **much slower** than a native -> build (and still needs the same ~21 GiB peak). Prefer building natively on a host of the target architecture -> when one is available. +> ⚠️ The dataset-generation step runs under emulation for any non-host arch, which is **much slower** than native +> (and still needs the same ~21 GiB peak). Since multi-arch is the default, expect the non-host arch to build under +> emulation; pass `--platform ` for a quick single-arch image when that is enough. #### Pull (consumer / eval side) diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index 2eddbec81..4ad678ab5 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -14,22 +14,23 @@ # # Usage: # LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh -# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no-build # push existing local image -# ... --force # overwrite an existing : tag (default: refuse) +# Builds and pushes a MULTI-ARCH manifest (linux/amd64,linux/arm64) by default. +# ... --platform linux/arm64 # single arch (fast; no QEMU emulation) +# ... --platform linux/amd64,linux/arm64 # explicit multi-arch (the default) +# LCB_IMAGE_REGISTRY=... LCB_IMAGE_TAG= ./push_image.sh --no-build # push existing local image +# ... --force # overwrite an existing : tag (default: refuse) # -# Cross-architecture build (e.g. build arm64 on an x86 node, or a multi-arch manifest): -# LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh --platform linux/arm64 -# ... --platform linux/amd64,linux/arm64 # multi-arch manifest in one push -# (or set LCB_IMAGE_PLATFORM instead of the flag.) -# Platform builds use 'docker buildx' and push the image straight to the registry -# (a non-native image cannot be loaded into the local docker store). They require -# QEMU emulation registered on the host for the target arch, one time: -# docker run --privileged --rm tonistiigi/binfmt --install arm64 -# The dataset-generation step runs under emulation and is MUCH slower than native. +# All builds go through 'docker buildx' and push straight to the registry: buildx forces +# gzip layers so the image is enroot/pyxis-extractable (mlcommons/endpoints#467), and a +# multi-arch image cannot be loaded into the local docker store anyway. Building an arch +# other than the host's needs QEMU emulation registered once (needs --privileged): +# docker run --privileged --rm tonistiigi/binfmt --install all +# The dataset-generation step runs under emulation and is MUCH slower than native, so +# prefer --platform when a single-arch image is enough. # # Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY required). # HF_TOKEN required unless --no-build (passed as a BuildKit secret directly from the environment). -# LCB_IMAGE_PLATFORM optional target platform(s), e.g. linux/arm64 (default: host native). +# LCB_IMAGE_PLATFORM target platform(s), e.g. linux/arm64 (default: linux/amd64,linux/arm64). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -59,7 +60,11 @@ platform_supported() { [[ -z "$(unsupported_platforms "$1" "$2")" ]]; } NO_BUILD=0 FORCE=0 -PLATFORM="${LCB_IMAGE_PLATFORM:-}" +# Multi-arch by default: one publish runs on amd64 and arm64, and the buildx path forces +# gzip layers so the image is enroot/pyxis-extractable (mlcommons/endpoints#467). Override +# with --platform for a single arch (e.g. --platform linux/arm64 skips the slow QEMU +# emulation of the non-host arch). +PLATFORM="${LCB_IMAGE_PLATFORM:-linux/amd64,linux/arm64}" while [[ $# -gt 0 ]]; do case "$1" in --no-build) NO_BUILD=1 ;; @@ -154,26 +159,32 @@ if [[ "$FORCE" -eq 0 ]]; then fi # ---------------------------------------------------------------------------- -# Cross-architecture path: build with buildx and push directly to the registry. -# A non-native image cannot be `docker load`-ed into the local store, so buildx -# builds and pushes in a single step (no separate tag/push). +# --no-build pushes a pre-built local image (host arch only) via a plain tag+push. +# Every other invocation builds with buildx and pushes straight to the registry: buildx +# forces gzip layers (enroot/pyxis-safe, #467) and a multi-arch image cannot be +# `docker load`-ed into the local store anyway, so there is no separate build/tag step. # ---------------------------------------------------------------------------- -if [[ -n "$PLATFORM" ]]; then - if [[ "$NO_BUILD" -eq 1 ]]; then - echo "error: --platform builds the image, so it cannot be combined with --no-build." >&2 - exit 1 - fi +if [[ "$NO_BUILD" -eq 1 ]]; then + echo ">> Tagging ${LCB_LOCAL_TAG} -> ${LCB_IMAGE_REF}" + docker tag "$LCB_LOCAL_TAG" "$LCB_IMAGE_REF" + + echo ">> Pushing ${LCB_IMAGE_REF}" + docker push "$LCB_IMAGE_REF" + # A pre-built local image can carry zstd base layers under the containerd image store; + # block a publish enroot/pyxis cannot extract (#467). + assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 +else if [[ -z "${HF_TOKEN:-}" ]]; then - echo "error: HF_TOKEN is required to build the image." >&2 + echo "error: HF_TOKEN is required to build the image (or pass --no-build to push an existing one)." >&2 exit 1 fi if ! docker buildx version >/dev/null 2>&1; then - echo "error: 'docker buildx' is required for --platform builds but is not available." >&2 + echo "error: 'docker buildx' is required but is not available." >&2 exit 1 fi - # Ensure a docker-container builder exists (the default 'docker' driver cannot - # build non-native platforms). Creating it is safe and unprivileged. + # Ensure a docker-container builder exists (the default 'docker' driver can neither + # push nor build a non-host platform). Creating it is safe and unprivileged. if ! docker buildx inspect "$BUILDX_BUILDER" >/dev/null 2>&1; then echo ">> Creating buildx builder '${BUILDX_BUILDER}' (docker-container driver)" docker buildx create --name "$BUILDX_BUILDER" --driver docker-container >/dev/null @@ -194,6 +205,7 @@ if [[ -n "$PLATFORM" ]]; then echo "error: builder '${BUILDX_BUILDER}' cannot build: ${missing_platforms} — QEMU emulation for it is not registered on the host." >&2 echo " Register it once (needs --privileged), then re-run:" >&2 echo " docker run --privileged --rm tonistiigi/binfmt --install all" >&2 + echo " Or build a single arch to skip emulation: --platform linux/$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')" >&2 exit 1 fi @@ -209,33 +221,6 @@ if [[ -n "$PLATFORM" ]]; then --output "type=image,push=true,compression=gzip,force-compression=true,oci-mediatypes=false" \ "$SCRIPT_DIR" assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 -else - # ------------------------------------------------------------------------ - # Native path: plain docker build for the host arch, then tag and push. - # ------------------------------------------------------------------------ - if [[ "$NO_BUILD" -eq 0 ]]; then - if [[ -z "${HF_TOKEN:-}" ]]; then - echo "error: HF_TOKEN is required to build the image (or pass --no-build to push an existing one)." >&2 - exit 1 - fi - - echo ">> Building ${LCB_LOCAL_TAG} (endpoints ${ENDPOINTS_SHA}) ..." - docker build \ - -f "${SCRIPT_DIR}/lcb_serve.dockerfile" \ - --secret id=HF_TOKEN,env=HF_TOKEN \ - --build-arg "ENDPOINTS_SHA=${ENDPOINTS_SHA}" \ - -t "$LCB_LOCAL_TAG" \ - "$SCRIPT_DIR" - fi - - echo ">> Tagging ${LCB_LOCAL_TAG} -> ${LCB_IMAGE_REF}" - docker tag "$LCB_LOCAL_TAG" "$LCB_IMAGE_REF" - - echo ">> Pushing ${LCB_IMAGE_REF}" - docker push "$LCB_IMAGE_REF" - # The native docker push can preserve zstd base layers under the containerd image - # store; block a publish enroot/pyxis cannot extract (#467). - assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 fi echo From 0ae7915898d0b0fce484ed4672eeac62f9c6d1b3 Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:40:19 -0500 Subject: [PATCH 6/8] feat(image-publish): self-identifying tags/labels + hardened publish guards Follow-up hardening for the image push/pull tooling (client + lcb-service): - lib_registry.sh: add ref_exists_in_registry, a fail-closed 0/1/2 registry existence probe. A two-stage match screens tooling/credential/permission errors to "indeterminate" BEFORE matching not-found phrasings, so a missing docker-credential helper ("executable file not found") can't be mistaken for an absent tag (which would let a push overwrite an immutable one). Handles GHCR ": not found" and ECR "name unknown ... does not exist". - push_docker_image.sh: add --force; refuse to overwrite an existing : unless --force. Blocks on any non-absent probe result (fail closed). - push_image.sh: --no-build now verifies layers on a transient staging tag then promotes onto the pinned tag only on success (verify-before-publish, so a zstd/#467 image never poisons the immutable pin). Promote with --prefer-index=false and re-assert the pin. Immutability guard reuses the shared ref_exists_in_registry. - _image_env.sh: always append an idempotent "-livecodebench" suffix to the LCB tag so the image is self-identifying and never collides with the client image's bare : in a shared registry package. Applied in the shared push/pull helper, so pull stays symmetric (consumers still pass the SHA). - Dockerfile.dev / lcb_serve.dockerfile: add OCI title/description LABELs so docker inspect self-identifies each image. Client label placed last (cache); its description is capability-neutral (PROVISION_DSR1/PROVISION_VBENCH may be 0). - README: document the tag scheme and add a maintainer recipe to publish the official ghcr.io/mlcommons/endpoints image. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/Dockerfile.dev | 11 ++++ scripts/lib_registry.sh | 30 ++++++++++ scripts/push_docker_image.sh | 24 ++++++++ .../evaluation/livecodebench/README.md | 30 +++++++--- .../evaluation/livecodebench/_image_env.sh | 13 ++++- .../livecodebench/lcb_serve.dockerfile | 15 +++-- .../evaluation/livecodebench/push_image.sh | 56 +++++++++++-------- 7 files changed, 143 insertions(+), 36 deletions(-) diff --git a/scripts/Dockerfile.dev b/scripts/Dockerfile.dev index 2751dc830..f008a4b29 100644 --- a/scripts/Dockerfile.dev +++ b/scripts/Dockerfile.dev @@ -102,3 +102,14 @@ RUN if [ "${PROVISION_VBENCH}" = "1" ]; then \ else \ echo "PROVISION_VBENCH=${PROVISION_VBENCH}: skipping VBench provisioning" ; \ fi + +# OCI image metadata so `docker inspect` self-identifies this image (the endpoints benchmark +# client, distinct from the lcb-service evaluator image). Declared last so a wording change +# only rebuilds this tiny config layer, not the apt/uv/DSR1/VBench stages above. +# push_docker_image.sh additionally stamps dynamic source/revision/version/description +# annotations on the pushed manifest. The description is deliberately capability-neutral: +# PROVISION_DSR1 / PROVISION_VBENCH can be 0, so it names the evaluators without asserting +# they are present in every build. +LABEL org.opencontainers.image.title="inference-endpoint" \ + org.opencontainers.image.description="MLPerf inference endpoint benchmarking client (the inference-endpoint CLI); DeepSeek-R1 and VBench accuracy evaluators are provisioned by default, toggled by PROVISION_DSR1 / PROVISION_VBENCH." \ + org.opencontainers.image.source="https://github.com/mlcommons/endpoints" diff --git a/scripts/lib_registry.sh b/scripts/lib_registry.sh index 3aa7b587c..bd2e784bf 100644 --- a/scripts/lib_registry.sh +++ b/scripts/lib_registry.sh @@ -47,3 +47,33 @@ if bad: print(f">> Verified: all {len(types)} layers of {ref} are gzip (enroot-safe).") PY } + +# ref_exists_in_registry REF — probe whether REF is already published, reading registry +# metadata only (no blob pull) via `docker buildx imagetools inspect`. Return codes: +# 0 present +# 1 definitely absent (registry reported not-found) +# 2 indeterminate (auth / network / tooling error) — output echoed to stderr +# Callers enforcing an immutable tag MUST treat 2 as "block" (fail CLOSED): never +# overwrite when existence can't be verified, or the guard silently no-ops on exactly +# the hosts/creds where it can't check. +ref_exists_in_registry() { + local ref="$1" out + if out="$(docker buildx imagetools inspect "$ref" 2>&1)"; then + return 0 + fi + # A missing binary / credential-helper / permission failure also contains "not found" + # ("docker-credential-xxx: executable file not found") but is NOT an absent manifest — + # classifying it as absent would let a push overwrite an immutable tag. Screen these to + # indeterminate FIRST so the not-found match below can't fire on them (fail CLOSED). + if grep -qiE 'executable file not found|command not found|no such file or directory|permission denied|credential' <<<"$out"; then + printf '%s\n' "$out" >&2 + return 2 + fi + # Registry "absent" phrasings: GHCR (`: not found`), OCI/Docker distribution + # (`manifest unknown`, `name unknown`), and ECR (`name unknown … does not exist`). + if grep -qiE 'not found|manifest unknown|manifest_unknown|name[ _]unknown|no such manifest|does not exist' <<<"$out"; then + return 1 + fi + printf '%s\n' "$out" >&2 + return 2 +} diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh index 82fea19e6..525df32f1 100755 --- a/scripts/push_docker_image.sh +++ b/scripts/push_docker_image.sh @@ -21,6 +21,7 @@ # ./scripts/push_docker_image.sh --cache # allow layer cache (default: --no-cache) # ./scripts/push_docker_image.sh --multi-arch --no-push # dry build (validate both arches, publish nothing) # ./scripts/push_docker_image.sh --allow-dirty # build HEAD despite uncommitted tracked changes +# ./scripts/push_docker_image.sh --force # overwrite an existing : tag (default: refuse) # # Multi-arch note: --multi-arch (or --platform with a comma) pushes a manifest # list usable on BOTH x86 and arm hosts; Docker pulls the matching arch. The @@ -57,6 +58,7 @@ DOCKERFILE="${DOCKERFILE:-scripts/Dockerfile.dev}" NO_CACHE="${NO_CACHE:-1}" ALLOW_DIRTY="${ALLOW_DIRTY:-0}" PUSH=1 +FORCE=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -77,6 +79,7 @@ while [[ $# -gt 0 ]]; do --cache) NO_CACHE=0 ;; --no-push) PUSH=0 ;; --allow-dirty) ALLOW_DIRTY=1 ;; + --force) FORCE=1 ;; --dockerfile) [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --dockerfile requires a value" >&2; exit 1; } DOCKERFILE="$2"; shift ;; @@ -169,6 +172,27 @@ if [[ -z "${DID_CHECKOUT:-}" && "$ALLOW_DIRTY" != "1" ]] && tree_has_tracked_cha echo ">> warning: ${dirty_msg} (dry run; nothing pushed)." >&2 fi +# --------------------------------------------------------------------------- +# Refuse to overwrite an already-published : tag unless --force. The SHA tag is a +# stable pin (consumers `docker pull …:`), and SHORT_SHA is not a function of the +# build inputs the workflow varies (platform, PROVISION_DSR1, cache), so a second push +# of the same commit would silently replace a published image. Reads registry metadata +# only; fail CLOSED (an indeterminate probe blocks too). The moving : tag is +# intentionally not guarded — it is meant to move. +# --------------------------------------------------------------------------- +if [[ "$PUSH" == "1" && "$FORCE" != "1" ]]; then + rc=0; ref_exists_in_registry "${IMAGE}:${SHORT_SHA}" || rc=$? + if [[ "$rc" -eq 0 ]]; then + echo "error: ${IMAGE}:${SHORT_SHA} already exists in the registry." >&2 + echo " The SHA tag is a stable pin; re-run with --force to overwrite it." >&2 + exit 1 + elif [[ "$rc" -ne 1 ]]; then + echo "error: could not verify whether ${IMAGE}:${SHORT_SHA} already exists (see above)." >&2 + echo " Fix registry access/login, or re-run with --force to skip this check." >&2 + exit 1 + fi +fi + # --------------------------------------------------------------------------- # Ensure a docker-container builder (supports --push and multi-platform). # --------------------------------------------------------------------------- diff --git a/src/inference_endpoint/evaluation/livecodebench/README.md b/src/inference_endpoint/evaluation/livecodebench/README.md index 7e97ed332..8416c8490 100644 --- a/src/inference_endpoint/evaluation/livecodebench/README.md +++ b/src/inference_endpoint/evaluation/livecodebench/README.md @@ -182,19 +182,35 @@ consumer can **pull and run it with no `HF_TOKEN` and no rebuild**. Two scripts in this directory wrap the docker build/tag/push/pull steps. Both resolve the image reference from environment variables (shared via `_image_env.sh`): -| Variable | Required | Default | Meaning | -| -------------------- | -------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | -| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | -| `LCB_IMAGE_TAG` | push: no · pull: yes | endpoints short SHA | image tag; `push_image.sh` defaults it to the endpoints commit SHA (one immutable tag per build). Pull must name the build. | -| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | +| Variable | Required | Default | Meaning | +| -------------------- | -------------------- | ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | +| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | +| `LCB_IMAGE_TAG` | push: no · pull: yes | `-livecodebench` | image tag; `push_image.sh` defaults the SHA part to the endpoints commit SHA and `_image_env.sh` always appends a `-livecodebench` suffix (one immutable, self-identifying tag per build). Pull must name the build — pass the SHA; the suffix is added for you. | +| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | -The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`. `push_image.sh` sets `LCB_IMAGE_TAG` to the endpoints commit short SHA by default, so each build publishes an immutable `…/lcb-service:` — there is no moving `latest`/`release_v6` tag. The same SHA is also baked into the image as the `org.opencontainers.image.revision` label. +The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`, where `_image_env.sh` always appends a `-livecodebench` suffix to `LCB_IMAGE_TAG` so the LCB image is self-identifying and never collides with the client image's bare `:` tag in a shared package. `push_image.sh` defaults the SHA part to the endpoints commit short SHA, so each build publishes an immutable `…/lcb-service:-livecodebench` — there is no moving `latest`/`release_v6` tag. The same SHA is also baked into the image as the `org.opencontainers.image.revision` label. #### Push (maintainer) Requires `docker login dhi.io` (base images) and `docker login` to your target registry first. +##### Updating the official repo image (`ghcr.io/mlcommons/endpoints`) + +Maintainers with `write:packages` on the **mlcommons** org publish the canonical image from the repo root. Authenticate to GHCR (a PAT / `GITHUB_TOKEN` with `write:packages`) and to `dhi.io` (hardened base images) first, then run the push script with the official registry + name: + +```bash +echo | docker login ghcr.io -u --password-stdin +docker login dhi.io + +LCB_IMAGE_REGISTRY=ghcr.io/mlcommons LCB_IMAGE_NAME=endpoints HF_TOKEN= \ + ./src/inference_endpoint/evaluation/livecodebench/push_image.sh +``` + +This builds a multi-arch manifest and publishes `ghcr.io/mlcommons/endpoints:-livecodebench`. The `-livecodebench` suffix keeps it distinct from the client image's bare `:` tag in the same package, and the tag is immutable — re-pushing the same commit is refused unless you add `--force`. + +##### Publishing to your own registry + ```bash # Build (using HF_TOKEN as a build secret) and push a MULTI-ARCH manifest # (linux/amd64,linux/arm64) by default: diff --git a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh index 32464b83e..dd866f629 100644 --- a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh +++ b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh @@ -8,10 +8,12 @@ # Inputs (environment variables): # LCB_IMAGE_REGISTRY (required) registry + namespace, e.g. myregistry.com/team # LCB_IMAGE_NAME (optional) image repo name (default: lcb-service) -# LCB_IMAGE_TAG (required) tag — push_image.sh defaults it to the endpoints short SHA +# LCB_IMAGE_TAG (required) build id — push_image.sh defaults it to the endpoints short +# SHA. A "-livecodebench" suffix is always appended (below). # LCB_LOCAL_TAG (optional) local tag used by run/scorer (default: lcb-service:latest) # # Exports: +# LCB_IMAGE_TAG (with a "-livecodebench" suffix appended, idempotently) # LCB_IMAGE_REF = ${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG} # LCB_LOCAL_TAG (defaulted if unset) @@ -34,6 +36,15 @@ if [[ -z "${LCB_IMAGE_TAG:-}" ]]; then fi LCB_LOCAL_TAG="${LCB_LOCAL_TAG:-lcb-service:latest}" +# Every LCB image tag carries a "-livecodebench" component so the artifact is +# self-identifying and never collides with the client image's bare : tag when both +# live in one registry package. Applied here (the shared push/pull helper) so push and +# pull resolve the SAME ref: a consumer passing LCB_IMAGE_TAG= still pulls +# -livecodebench. Idempotent — a tag that already ends in -livecodebench is left as is. +if [[ "$LCB_IMAGE_TAG" != *-livecodebench ]]; then + LCB_IMAGE_TAG="${LCB_IMAGE_TAG}-livecodebench" +fi + # Strip any trailing slash on the registry to avoid a double slash in the ref. LCB_IMAGE_REF="${LCB_IMAGE_REGISTRY%/}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}" diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile index a8f80c95a..6989d929c 100644 --- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile +++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile @@ -63,13 +63,16 @@ COPY _server.py /app/server.py # Make lcb_serve.py available as a module ENV PYTHONPATH="/app" -# Provenance: the endpoints repo commit this image was built from, supplied by -# push_image.sh via --build-arg. Recorded as a config LABEL (not a manifest -# annotation) so it survives the oci-mediatypes=false push and is visible in -# `docker inspect`. Declared after the COPYs so a new SHA only rebuilds this -# metadata layer, never the expensive dataset-generation stage above. +# OCI image metadata: a static title/description that self-identifies this image (the +# LiveCodeBench evaluator service, distinct from the endpoints client image), plus +# provenance (source, and the endpoints repo commit via the ENDPOINTS_SHA build-arg). +# Recorded as config LABELs (not manifest annotations) so they survive the +# oci-mediatypes=false push and show in `docker inspect`. Declared after the COPYs so a new +# SHA only rebuilds this metadata layer, never the expensive dataset-generation stage above. ARG ENDPOINTS_SHA=unknown -LABEL org.opencontainers.image.source="https://github.com/mlcommons/endpoints" \ +LABEL org.opencontainers.image.title="lcb-service" \ + org.opencontainers.image.description="LiveCodeBench evaluation service: a WebSocket judge (port 13835) that runs model-generated code against the LiveCodeBench dataset (release_v6, baked in). Used by the endpoints client's code_bench_scorer." \ + org.opencontainers.image.source="https://github.com/mlcommons/endpoints" \ org.opencontainers.image.revision="${ENDPOINTS_SHA}" # Launch the WebSocket server with long-running connection support diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index 4ad678ab5..a07bfa220 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -5,8 +5,10 @@ # build time. Pushing the built image lets consumers pull-and-run with no HF_TOKEN # and no rebuild (see pull_image.sh). # -# The image is tagged by the endpoints commit short SHA — one immutable tag per -# build. LCB_IMAGE_TAG overrides it. There is no moving channel tag. +# The image is tagged -livecodebench — one immutable, self-identifying +# tag per build. The "-livecodebench" suffix is appended by _image_env.sh so the tag never +# collides with the client image's bare : in a shared registry package. LCB_IMAGE_TAG +# overrides the SHA part (the suffix is still appended). There is no moving channel tag. # # Prerequisites: # - docker login dhi.io (base images are Docker Hardened Images) @@ -137,42 +139,52 @@ source "${SCRIPT_DIR}/_image_env.sh" # shellcheck source=/dev/null source "$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)/scripts/lib_registry.sh" -# Refuse to overwrite an existing remote tag unless --force: the SHA tag is meant to -# be immutable, so a second push to the same ref would silently replace a published -# image. `imagetools inspect` queries the registry (no layer pull). Fail CLOSED: a -# clean exit means the tag exists (block); a non-zero exit is trusted as "absent" ONLY -# when the output confirms not-found. Any other failure (buildx plugin missing, auth -# denied, registry/network error) blocks too — otherwise the guard silently no-ops on -# exactly the hosts/creds where it can't verify, and immutability goes unenforced. -# --force skips the check entirely. +# Refuse to overwrite an existing remote tag unless --force: the SHA tag is meant to be +# immutable, so a second push would silently replace a published image. Delegates to the +# shared ref_exists_in_registry (fail CLOSED — anything other than a confirmed "absent" +# blocks). --force skips the check entirely. if [[ "$FORCE" -eq 0 ]]; then - if inspect_out="$(docker buildx imagetools inspect "$LCB_IMAGE_REF" 2>&1)"; then + rc=0; ref_exists_in_registry "$LCB_IMAGE_REF" || rc=$? + if [[ "$rc" -eq 0 ]]; then echo "error: ${LCB_IMAGE_REF} already exists in the registry." >&2 echo " The SHA tag is meant to be immutable; re-run with --force to overwrite it." >&2 exit 1 - elif ! grep -qiE 'not found|manifest unknown|manifest_unknown|name_unknown|no such manifest' <<<"$inspect_out"; then - echo "error: could not verify whether ${LCB_IMAGE_REF} already exists:" >&2 - printf ' %s\n' "$inspect_out" >&2 + elif [[ "$rc" -ne 1 ]]; then + echo "error: could not verify whether ${LCB_IMAGE_REF} already exists (see above)." >&2 echo " Fix registry access (or install docker buildx), or re-run with --force to skip this check." >&2 exit 1 fi fi # ---------------------------------------------------------------------------- -# --no-build pushes a pre-built local image (host arch only) via a plain tag+push. +# --no-build publishes a pre-built local image (host arch only): push to a staging tag, +# verify, then promote onto the pinned tag (a plain push can't be force-compressed, so +# the immutable : is only made reachable after the gzip check passes). # Every other invocation builds with buildx and pushes straight to the registry: buildx # forces gzip layers (enroot/pyxis-safe, #467) and a multi-arch image cannot be # `docker load`-ed into the local store anyway, so there is no separate build/tag step. # ---------------------------------------------------------------------------- if [[ "$NO_BUILD" -eq 1 ]]; then - echo ">> Tagging ${LCB_LOCAL_TAG} -> ${LCB_IMAGE_REF}" - docker tag "$LCB_LOCAL_TAG" "$LCB_IMAGE_REF" - - echo ">> Pushing ${LCB_IMAGE_REF}" - docker push "$LCB_IMAGE_REF" - # A pre-built local image can carry zstd base layers under the containerd image store; - # block a publish enroot/pyxis cannot extract (#467). + # A pre-built local image can carry zstd base layers (containerd image store) that + # enroot/pyxis cannot extract (#467), and this plain `docker push` applies no + # force-compression. Verify BEFORE the pinned : tag becomes reachable: push to a + # transient staging tag, assert its layers, then promote onto $LCB_IMAGE_REF only on + # success — so a failed verify never poisons the immutable tag (which the guard above + # then refuses to overwrite without --force). `imagetools create` copies layers by + # digest (no blob re-upload); --prefer-index=false makes the pin the SAME manifest + # digest that was verified (not a fresh 1-entry index), and it is re-asserted below. + STAGING_REF="${LCB_IMAGE_REF%:*}:staging-${LCB_IMAGE_TAG}" + echo ">> Tagging ${LCB_LOCAL_TAG} -> ${STAGING_REF} (staging)" + docker tag "$LCB_LOCAL_TAG" "$STAGING_REF" + + echo ">> Pushing ${STAGING_REF} for verification" + docker push "$STAGING_REF" + assert_gzip_layers "$STAGING_REF" || exit 1 + + echo ">> Verified; promoting ${STAGING_REF} -> ${LCB_IMAGE_REF}" + docker buildx imagetools create --prefer-index=false --tag "$LCB_IMAGE_REF" "$STAGING_REF" assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 + echo " (transient ${STAGING_REF##*:} tag left in the registry; delete via your registry UI/API if desired.)" else if [[ -z "${HF_TOKEN:-}" ]]; then echo "error: HF_TOKEN is required to build the image (or pass --no-build to push an existing one)." >&2 From 9d749906250ea57513958064567357a884a96fca Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:10:26 -0500 Subject: [PATCH 7/8] fix(image-publish): close registry-probe fail-open + add CI force input Address review-council findings on the image publish tooling: - lib_registry.sh: ref_exists_in_registry no longer classifies auth-hidden-as- not-found as absent. Registries that answer an unauthorized/private repo with "repository does not exist or may require 'docker login': denied" previously matched the not-found phrasing and returned "absent" (rc=1), letting a push- only / split-scope credential overwrite an immutable tag. Auth/denied/401/403 phrasings are now screened to indeterminate (rc=2) BEFORE the not-found match; GHCR ": not found" and ECR "name unknown" still resolve to absent. - publish-image.yml: add a `force` workflow_dispatch input (forwarded as a fixed --force flag via env, never interpolating the input into the command) so a SHA can be re-published (e.g. a different arch, or a failed run); document that platform/DSR1/cache are not part of the tag. - push_docker_image.sh / push_image.sh: document that the buildx-path assert_gzip_layers is a post-publish confirmation (the build forces gzip, so the pushed image is enroot-safe by construction), and that a transient inspect flake fails the job after the tag is live (re-run with --force). Reword the client guard comment: the : gate covers the whole build (both tags). - README: correct the pinned tag to :-livecodebench; add an official pull example (LCB_IMAGE_NAME=endpoints, else it resolves to a different repo); document that --no-build stages+verifies+promotes, is host-arch only, ignores --platform, and rejects (does not repair) zstd local images. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/publish-image.yml | 12 +++++++++++- scripts/lib_registry.sh | 15 ++++++++++----- scripts/push_docker_image.sh | 14 +++++++++++--- .../evaluation/livecodebench/README.md | 9 +++++++-- .../evaluation/livecodebench/push_image.sh | 4 ++++ 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 1474eb29a..7f4af0e81 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -37,6 +37,14 @@ on: options: - "true" - "false" + force: + description: "Overwrite an existing : tag. platform/DSR1/cache are NOT part of the tag, so the first successful publish of a SHA wins; use force to re-publish a SHA (e.g. a different arch, or a failed run)." + required: true + default: "false" + type: choice + options: + - "false" + - "true" # GITHUB_TOKEN needs packages:write to push to the org's GHCR package. permissions: @@ -79,4 +87,6 @@ jobs: PLATFORM: ${{ inputs.platforms }} PROVISION_DSR1: ${{ inputs.provision_dsr1 }} NO_CACHE: ${{ inputs.no_cache == 'true' && '1' || '0' }} - run: ./scripts/push_docker_image.sh + # Fixed flag or empty — the input value itself is never interpolated into the command. + FORCE_FLAG: ${{ inputs.force == 'true' && '--force' || '' }} + run: ./scripts/push_docker_image.sh $FORCE_FLAG diff --git a/scripts/lib_registry.sh b/scripts/lib_registry.sh index bd2e784bf..a3f2b1965 100644 --- a/scripts/lib_registry.sh +++ b/scripts/lib_registry.sh @@ -61,11 +61,16 @@ ref_exists_in_registry() { if out="$(docker buildx imagetools inspect "$ref" 2>&1)"; then return 0 fi - # A missing binary / credential-helper / permission failure also contains "not found" - # ("docker-credential-xxx: executable file not found") but is NOT an absent manifest — - # classifying it as absent would let a push overwrite an immutable tag. Screen these to - # indeterminate FIRST so the not-found match below can't fire on them (fail CLOSED). - if grep -qiE 'executable file not found|command not found|no such file or directory|permission denied|credential' <<<"$out"; then + # Two error classes contain not-found-ish phrasing but are NOT an absent manifest, and + # classifying either as absent would let a push overwrite an immutable tag: + # - tooling failures: "docker-credential-xxx: executable file not found" + # - authorization hidden as absence: many registries (e.g. Docker Hub) answer an + # unauthorized/private repo with "repository does not exist or may require 'docker + # login': denied", which also matches "does not exist" below. + # Screen both to indeterminate FIRST so the not-found match can't fire on them (fail + # CLOSED). A truly-absent public tag (GHCR ": not found", ECR "name unknown") has + # none of these phrases and still resolves to rc=1. + if grep -qiE 'executable file not found|command not found|no such file or directory|permission denied|credential|denied|unauthorized|forbidden|requires? .*(login|auth)|may require|401|403|authentication' <<<"$out"; then printf '%s\n' "$out" >&2 return 2 fi diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh index 525df32f1..3d0db73b9 100755 --- a/scripts/push_docker_image.sh +++ b/scripts/push_docker_image.sh @@ -177,8 +177,10 @@ fi # stable pin (consumers `docker pull …:`), and SHORT_SHA is not a function of the # build inputs the workflow varies (platform, PROVISION_DSR1, cache), so a second push # of the same commit would silently replace a published image. Reads registry metadata -# only; fail CLOSED (an indeterminate probe blocks too). The moving : tag is -# intentionally not guarded — it is meant to move. +# only; fail CLOSED (an indeterminate probe blocks too). This gate covers the whole build: +# if : exists, BOTH it and the moving : tag are refused until --force, so +# refreshing : for an already-published commit needs --force (the CI workflow +# exposes a `force` input for exactly this). # --------------------------------------------------------------------------- if [[ "$PUSH" == "1" && "$FORCE" != "1" ]]; then rc=0; ref_exists_in_registry "${IMAGE}:${SHORT_SHA}" || rc=$? @@ -260,7 +262,13 @@ docker buildx build \ ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} \ . -# Guard against silently publishing an image enroot/pyxis cannot extract (#467). +# Guard against silently publishing an image enroot/pyxis cannot extract (#467). This runs +# AFTER --push, but the build above forces every layer to gzip (compression=gzip, +# force-compression=true), so the pushed image is enroot-safe by construction — this is a +# post-publish confirmation, not a gate. Known caveat: a transient registry-inspect flake +# here fails the job after the tag is already live; the immutable-tag guard then blocks a +# plain retry, so re-run with --force to republish. (Verify-before-publish would need the +# staging→promote dance the LCB --no-build path uses; not worth it when gzip is forced.) [[ "$PUSH" == "1" ]] && { assert_gzip_layers "${IMAGE}:${SHORT_SHA}" || exit 1; } echo diff --git a/src/inference_endpoint/evaluation/livecodebench/README.md b/src/inference_endpoint/evaluation/livecodebench/README.md index 8416c8490..3899717ab 100644 --- a/src/inference_endpoint/evaluation/livecodebench/README.md +++ b/src/inference_endpoint/evaluation/livecodebench/README.md @@ -224,9 +224,9 @@ LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no- Every build goes through `docker buildx` and pushes **straight to the registry**: buildx forces gzip layers so the image is extractable by enroot/pyxis on SLURM (see [#467](https://github.com/mlcommons/endpoints/issues/467)), and a multi-arch image cannot be loaded into the local docker store anyway. The script auto-creates a `docker-container` -buildx builder. `--no-build` is the one exception — it tag+pushes a pre-built local image for the host arch only. +buildx builder. `--no-build` is the one exception: it publishes a pre-built local `lcb-service:latest` (**host arch only**) by pushing it to a transient `staging-…` tag, verifying its layers, then promoting onto the pinned tag on success. Because a plain push cannot force gzip, a local image carrying zstd layers (containerd image store) is **rejected, not repaired** — rebuild via the buildx path (drop `--no-build`) to get gzip layers. `--no-build` also ignores `--platform`. -Each build publishes an immutable `:` tag, so **re-pushing an existing `:` is refused** to protect it. Pass `--force` to overwrite deliberately (e.g. re-running a partially failed push): +Each build publishes an immutable `:-livecodebench` tag, so **re-pushing an existing `:-livecodebench` is refused** to protect it. Pass `--force` to overwrite deliberately (e.g. re-running a partially failed push): ```bash LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --force @@ -259,6 +259,11 @@ locally as `lcb-service:latest`, so the [hardened run command](#hardened-run-com ```bash LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh + +# Official image published by maintainers (matches the push recipe above). LCB_IMAGE_NAME +# must be set to `endpoints`, otherwise this resolves to ghcr.io/mlcommons/lcb-service (a +# different repo). Pass the SHA; the `-livecodebench` suffix is added for you. +LCB_IMAGE_REGISTRY=ghcr.io/mlcommons LCB_IMAGE_NAME=endpoints LCB_IMAGE_TAG= ./pull_image.sh ``` ### (Only if using enroot) Generating a .sqsh file for enroot diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index a07bfa220..01d69c58a 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -232,6 +232,10 @@ else --provenance=false \ --output "type=image,push=true,compression=gzip,force-compression=true,oci-mediatypes=false" \ "$SCRIPT_DIR" + # Post-publish confirmation, not a gate: the build forces gzip layers so the pushed + # image is enroot-safe by construction (#467). Known caveat: a transient inspect flake + # here fails the job after the tag is live; re-run with --force to republish. (Only the + # --no-build path needs the staging→verify→promote dance, since it can't force gzip.) assert_gzip_layers "$LCB_IMAGE_REF" || exit 1 fi From 6f4581ccf6c95c980ad8f1e90f35f8efd4a622a0 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:44:55 -0500 Subject: [PATCH 8/8] feat(image-publish): add provision_vbench toggle to script + workflow push_docker_image.sh forwarded only PROVISION_DSR1, so PROVISION_VBENCH=0 was silently ignored and the VBench (WAN 2.2) scorer was always baked in. Forward PROVISION_VBENCH as a build-arg symmetrically with PROVISION_DSR1, and expose a provision_vbench choice input in the publish-image workflow. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/publish-image.yml | 9 +++++++++ scripts/push_docker_image.sh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 7f4af0e81..b4b5e8598 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -29,6 +29,14 @@ on: options: - "1" - "0" + provision_vbench: + description: "Bake in the WAN 2.2 VBench accuracy scorer (heavier; downloads AMT/RAFT/RAM weights at build time)" + required: true + default: "1" + type: choice + options: + - "1" + - "0" no_cache: description: "Build with --no-cache (clean/reproducible, slower)" required: true @@ -86,6 +94,7 @@ jobs: ENDPOINTS_REF: ${{ inputs.ref || github.ref_name }} PLATFORM: ${{ inputs.platforms }} PROVISION_DSR1: ${{ inputs.provision_dsr1 }} + PROVISION_VBENCH: ${{ inputs.provision_vbench }} NO_CACHE: ${{ inputs.no_cache == 'true' && '1' || '0' }} # Fixed flag or empty — the input value itself is never interpolated into the command. FORCE_FLAG: ${{ inputs.force == 'true' && '--force' || '' }} diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh index 3d0db73b9..d02b044e7 100755 --- a/scripts/push_docker_image.sh +++ b/scripts/push_docker_image.sh @@ -37,6 +37,7 @@ # NO_CACHE 1=--no-cache, 0=cache (default: 1) # ALLOW_DIRTY 1=build HEAD with uncommitted tracked changes (default: 0) # PROVISION_DSR1 passthrough build-arg (default: Dockerfile default = 1) +# PROVISION_VBENCH passthrough build-arg (default: Dockerfile default = 1) # IMAGE_DESCRIPTION GHCR package-page description (default: "endpoints client image (ref , commit )") # GHCR_USER/GHCR_TOKEN optional registry login (fallback: GITHUB_ACTOR/GITHUB_TOKEN) set -euo pipefail @@ -211,6 +212,7 @@ fi BUILD_ARGS=() [[ "$NO_CACHE" == "1" ]] && BUILD_ARGS+=(--no-cache) [[ -n "${PROVISION_DSR1:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_DSR1=${PROVISION_DSR1}") +[[ -n "${PROVISION_VBENCH:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_VBENCH=${PROVISION_VBENCH}") # Don't attach SLSA provenance attestations. buildx adds them by default on --push, # which surfaces a spurious `unknown/unknown` entry on the GHCR package page and