From 4696bcd1306012439c7f05e3b3d51305e38751b0 Mon Sep 17 00:00:00 2001 From: Adam Page Date: Sun, 5 Jul 2026 11:20:39 -0500 Subject: [PATCH 1/4] chore(devcontainer): add helper script for driving dev container from outside sessions Agent sessions without Remote-Containers support (e.g. running outside the dev container) need a reliable way to run commands in the existing GPU/uv environment. Add scripts/devcontainer.sh wrapping the devcontainer CLI: - up/status/down for lifecycle management - exec/shell for running commands with correct user, cwd, and remoteEnv (GPU/CUDA vars), which plain 'docker exec' silently drops - auto-resolves the main worktree checkout the container was built against, so it works correctly even when invoked from a linked 'git worktree' Document the workflow in AGENTS.md under Tooling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 6 +++ scripts/devcontainer.sh | 117 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100755 scripts/devcontainer.sh diff --git a/AGENTS.md b/AGENTS.md index be82167..43e7f15 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,12 @@ Three layers, kept deliberately separate (see `docs/experiment-architecture.md` ## Tooling - We primarily develop in a dev container and use `uv` for project and package management. +- If your agent session runs *outside* the dev container (e.g. no Remote-Containers support), don't run training/tests on the bare host. Use `scripts/devcontainer.sh` to drive the existing container instead: + - `scripts/devcontainer.sh up` — bring the container up (safe to call anytime; no-op if already running). + - `scripts/devcontainer.sh exec -- ` — run a command inside the container with correct user, cwd, and `remoteEnv` (GPU/CUDA vars). Prefer this over raw `docker exec`, which silently drops `remoteEnv`. + - `scripts/devcontainer.sh shell` — open an interactive shell in the container. + - `scripts/devcontainer.sh status` / `down` — check or stop the container. + - The script auto-resolves the main worktree checkout the container was built against, so it works correctly even when invoked from a linked `git worktree`. ## Project Management - GitHub Project board: https://github.com/orgs/AmbiqAI/projects/36 diff --git a/scripts/devcontainer.sh b/scripts/devcontainer.sh new file mode 100755 index 0000000..73ff7db --- /dev/null +++ b/scripts/devcontainer.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Helper for driving the project's dev container from an agent/session that is +# running *outside* the container (e.g. a VS Code Remote-Containers window +# already has it open, or you're on a plain host shell / git worktree). +# +# Why this exists: +# - `devcontainer` CLI looks up a running container by matching the +# *workspace folder path* baked into its labels at creation time. If you +# run it from a git worktree (a different path than the one the +# container was originally created for), it fails with +# "Dev container not found" even though the container is running. +# - Plain `docker exec` works from anywhere, but silently drops the +# `remoteEnv` values from devcontainer.json (e.g. TF_FORCE_GPU_ALLOW_GROWTH, +# CUDA LD_LIBRARY_PATH additions) since it doesn't know about them. +# +# This script always resolves the *main* worktree checkout (the one the dev +# container was actually built against) and passes `--container-id` so +# commands work correctly regardless of which worktree/directory you're in. +# +# Usage: +# scripts/devcontainer.sh up # bring the container up (idempotent) +# scripts/devcontainer.sh id # print the running container id +# scripts/devcontainer.sh exec -- [args] # run a command in the container +# scripts/devcontainer.sh shell # open an interactive shell +# scripts/devcontainer.sh down # stop the container +# scripts/devcontainer.sh status # show container state +# +# Examples: +# scripts/devcontainer.sh exec -- uv run pytest tests/test_two_stage.py -q +# scripts/devcontainer.sh exec -- uv run python scripts/train_rvq_prior.py --help + +set -euo pipefail + +# Resolve the main worktree path: the dev container is created against the +# primary checkout, not any linked `git worktree add` copy. `git worktree +# list` always prints the main worktree first. +resolve_main_worktree() { + if git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git -C "$(dirname "${BASH_SOURCE[0]}")" worktree list --porcelain \ + | awk '/^worktree /{print $2; exit}' + fi +} + +MAIN_WORKSPACE="${DEVCONTAINER_WORKSPACE:-$(resolve_main_worktree)}" +if [[ -z "${MAIN_WORKSPACE}" ]]; then + echo "error: could not resolve main worktree path; set DEVCONTAINER_WORKSPACE" >&2 + exit 1 +fi + +container_id() { + docker ps -q --filter "label=devcontainer.local_folder=${MAIN_WORKSPACE}" +} + +cmd_up() { + devcontainer up --workspace-folder "${MAIN_WORKSPACE}" +} + +cmd_id() { + local id + id="$(container_id)" + if [[ -z "${id}" ]]; then + echo "error: no running dev container for ${MAIN_WORKSPACE} (run 'up' first)" >&2 + exit 1 + fi + echo "${id}" +} + +cmd_exec() { + local id + id="$(cmd_id)" + devcontainer exec --container-id "${id}" --workspace-folder "${MAIN_WORKSPACE}" -- "$@" +} + +cmd_shell() { + local id + id="$(cmd_id)" + devcontainer exec --container-id "${id}" --workspace-folder "${MAIN_WORKSPACE}" -- bash +} + +cmd_down() { + local id + id="$(container_id)" + if [[ -z "${id}" ]]; then + echo "no running dev container for ${MAIN_WORKSPACE}" + return 0 + fi + docker stop "${id}" +} + +cmd_status() { + local id + id="$(container_id)" + if [[ -z "${id}" ]]; then + echo "stopped: ${MAIN_WORKSPACE}" + return 0 + fi + docker ps --filter "id=${id}" --format 'table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Names}}' +} + +subcommand="${1:-}" +[[ $# -gt 0 ]] && shift || true + +case "${subcommand}" in + up) cmd_up ;; + id) cmd_id ;; + exec) + [[ "${1:-}" == "--" ]] && shift + cmd_exec "$@" + ;; + shell) cmd_shell ;; + down) cmd_down ;; + status) cmd_status ;; + *) + echo "usage: $(basename "$0") {up|id|exec -- |shell|down|status}" >&2 + exit 1 + ;; +esac From e7dc8902045a05a99d497df8a277dd44b2bbf90b Mon Sep 17 00:00:00 2001 From: Adam Page Date: Sun, 5 Jul 2026 12:22:24 -0500 Subject: [PATCH 2/4] feat(devcontainer): add gpu-check/gpu-recover to the dev-container helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nvidia-smi succeeding inside a container doesn't guarantee a CUDA context can actually be created — after host-side kernel/driver churn (e.g. an unattended-upgrades kernel update rebuilding the NVIDIA DKMS module without a reboot), a long-running container can end up with stale device bindings that nvidia-smi doesn't catch. - gpu-check: nvidia-smi + a real TensorFlow CUDA context init. - gpu-recover: try a cheap 'docker restart' first (re-runs the NVIDIA container-runtime hook), fall back to a full stop + devcontainer up. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 1 + scripts/devcontainer.sh | 66 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 43e7f15..d555307 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,6 +53,7 @@ Three layers, kept deliberately separate (see `docs/experiment-architecture.md` - `scripts/devcontainer.sh exec -- ` — run a command inside the container with correct user, cwd, and `remoteEnv` (GPU/CUDA vars). Prefer this over raw `docker exec`, which silently drops `remoteEnv`. - `scripts/devcontainer.sh shell` — open an interactive shell in the container. - `scripts/devcontainer.sh status` / `down` — check or stop the container. + - `scripts/devcontainer.sh gpu-check` — verify the GPU is *actually* usable (real CUDA context, not just `nvidia-smi`, which can report healthy while training still fails). `gpu-recover` auto-heals (cheap `docker restart` first, full recreate as fallback). - The script auto-resolves the main worktree checkout the container was built against, so it works correctly even when invoked from a linked `git worktree`. ## Project Management diff --git a/scripts/devcontainer.sh b/scripts/devcontainer.sh index 73ff7db..912232d 100755 --- a/scripts/devcontainer.sh +++ b/scripts/devcontainer.sh @@ -24,10 +24,25 @@ # scripts/devcontainer.sh shell # open an interactive shell # scripts/devcontainer.sh down # stop the container # scripts/devcontainer.sh status # show container state +# scripts/devcontainer.sh gpu-check # verify the GPU is actually usable (not just nvidia-smi) +# scripts/devcontainer.sh gpu-recover # auto-heal a broken GPU: restart, else full recreate # # Examples: # scripts/devcontainer.sh exec -- uv run pytest tests/test_two_stage.py -q # scripts/devcontainer.sh exec -- uv run python scripts/train_rvq_prior.py --help +# +# Why gpu-check/gpu-recover instead of just running `nvidia-smi`: +# `nvidia-smi` inside the container only proves the NVML library can talk to +# the driver — it does NOT prove a CUDA context can actually be created on +# the container's own device file descriptors. After a host-side kernel/ +# driver churn event (e.g. an unattended-upgrades kernel update rebuilding +# the NVIDIA DKMS module without a reboot), `nvidia-smi` can keep working +# while real framework calls (TensorFlow/PyTorch CUDA init) fail inside an +# already-running container. gpu-check does a real minimal CUDA init via +# TensorFlow so it catches that failure mode; gpu-recover tries the cheap +# fix first (`docker restart`, which re-runs the NVIDIA container-runtime +# hook and re-binds fresh device state) before falling back to a full +# down+up (which recreates the container from scratch). set -euo pipefail @@ -97,6 +112,53 @@ cmd_status() { docker ps --filter "id=${id}" --format 'table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Names}}' } +cmd_gpu_check() { + local id + id="$(cmd_id)" + echo "-- nvidia-smi (NVML-level check) --" + if ! devcontainer exec --container-id "${id}" --workspace-folder "${MAIN_WORKSPACE}" -- nvidia-smi --query-gpu=name,memory.used --format=csv,noheader; then + echo "gpu-check: FAILED at nvidia-smi (driver/NVML not visible in container)" >&2 + return 1 + fi + echo "-- CUDA context init via TensorFlow (real usability check) --" + if ! devcontainer exec --container-id "${id}" --workspace-folder "${MAIN_WORKSPACE}" -- \ + uv run python -c " +import tensorflow as tf +gpus = tf.config.list_physical_devices('GPU') +assert gpus, 'no GPU visible to TensorFlow' +with tf.device('/GPU:0'): + x = tf.constant([1.0, 2.0]) + tf.constant([3.0, 4.0]) + x.numpy() +print('CUDA context OK:', gpus) +" 2>&1 | tail -20; then + echo "gpu-check: FAILED — nvidia-smi worked but a real CUDA context could not be created." >&2 + echo "This is the 'stale device binding' failure mode; run '$(basename "$0") gpu-recover'." >&2 + return 1 + fi + echo "gpu-check: OK" +} + +cmd_gpu_recover() { + local id + id="$(container_id)" + if [[ -z "${id}" ]]; then + echo "no running container; bringing one up..." + cmd_up + return + fi + echo "Attempting cheap recovery: docker restart (re-runs the NVIDIA container-runtime hook)..." + docker restart "${id}" >/dev/null + sleep 3 + if cmd_gpu_check; then + echo "gpu-recover: fixed via docker restart." + return 0 + fi + echo "docker restart did not fix it; falling back to full stop + devcontainer up (recreates the container)..." + docker stop "${id}" >/dev/null 2>&1 || true + cmd_up + cmd_gpu_check +} + subcommand="${1:-}" [[ $# -gt 0 ]] && shift || true @@ -110,8 +172,10 @@ case "${subcommand}" in shell) cmd_shell ;; down) cmd_down ;; status) cmd_status ;; + gpu-check) cmd_gpu_check ;; + gpu-recover) cmd_gpu_recover ;; *) - echo "usage: $(basename "$0") {up|id|exec -- |shell|down|status}" >&2 + echo "usage: $(basename "$0") {up|id|exec -- |shell|down|status|gpu-check|gpu-recover}" >&2 exit 1 ;; esac From 689a3f913c7d2bb5e0f7404dd525b02a57bbea82 Mon Sep 17 00:00:00 2001 From: Adam Page Date: Mon, 6 Jul 2026 08:33:57 -0500 Subject: [PATCH 3/4] fix: address PR review findings on devcontainer helper script Addresses findings from a dedicated code-review pass on PR #63: - resolve_main_worktree: the awk parser used field-splitting ($2) to extract the worktree path from 'git worktree list --porcelain' output, which silently truncates any workspace path containing spaces. Now strips only the 'worktree ' prefix and prints the rest of the line verbatim. - container_id(): now guards against multiple containers matching the same workspace-folder label (e.g. a leftover container from a previous 'up'), failing loudly instead of feeding a multi-line ID string into --container-id/docker stop and producing confusing downstream errors. - gpu-recover: the 'no running container' cold-start path now runs gpu-check after bringing the container up, matching the other two recovery paths. Previously it returned cmd_up's exit status directly, which only proves the container was created, not that the underlying GPU/CUDA issue gpu-recover exists to fix is actually resolved. Verified: bash -n syntax check, dogfooded up/status/id/exec/gpu-check against the real shared container, and isolated unit checks for the awk path-with-spaces fix and the multi-container guard's three cases (multi-line/single/empty). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/devcontainer.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/devcontainer.sh b/scripts/devcontainer.sh index 912232d..1399716 100755 --- a/scripts/devcontainer.sh +++ b/scripts/devcontainer.sh @@ -51,8 +51,11 @@ set -euo pipefail # list` always prints the main worktree first. resolve_main_worktree() { if git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + # Strip only the `worktree ` prefix and print the rest of the line + # verbatim (not `{print $2}`, which splits on whitespace and would + # silently truncate a workspace path containing spaces). git -C "$(dirname "${BASH_SOURCE[0]}")" worktree list --porcelain \ - | awk '/^worktree /{print $2; exit}' + | awk '/^worktree /{sub(/^worktree /, ""); print; exit}' fi } @@ -63,7 +66,15 @@ if [[ -z "${MAIN_WORKSPACE}" ]]; then fi container_id() { - docker ps -q --filter "label=devcontainer.local_folder=${MAIN_WORKSPACE}" + local ids + ids="$(docker ps -q --filter "label=devcontainer.local_folder=${MAIN_WORKSPACE}")" + if [[ -n "${ids}" ]] && [[ "$(wc -l <<< "${ids}")" -gt 1 ]]; then + echo "error: multiple running containers match workspace ${MAIN_WORKSPACE}:" >&2 + echo "${ids}" >&2 + echo "Disambiguate manually with --container-id, or stop the extras." >&2 + exit 1 + fi + echo "${ids}" } cmd_up() { @@ -144,6 +155,12 @@ cmd_gpu_recover() { if [[ -z "${id}" ]]; then echo "no running container; bringing one up..." cmd_up + # Bringing the container up only proves it was created — it says + # nothing about whether the GPU/CUDA issue gpu-recover exists to fix + # is actually resolved. Verify with gpu-check, same as the other two + # recovery paths below, so a caller checking this function's exit + # code gets a real answer instead of a false "success". + cmd_gpu_check return fi echo "Attempting cheap recovery: docker restart (re-runs the NVIDIA container-runtime hook)..." From bea2eeb1d9512b1bba4feda5ca39fef828c1b584 Mon Sep 17 00:00:00 2001 From: Adam Page Date: Mon, 6 Jul 2026 16:15:09 -0500 Subject: [PATCH 4/4] fix: gpu-recover fallback now actually recreates the container Codex review on PR #63 correctly identified that the 'full recreate' fallback in gpu-recover did not actually recreate anything: devcontainer up finds and restarts an existing (even stopped) container for a given workspace rather than creating a fresh one, so 'docker stop' + 'devcontainer up' just restarted the same container with its stale device bindings intact -- contradicting the script's own comment claiming a from-scratch recreate. Fix: docker rm the container (in addition to stop) before calling cmd_up, so devcontainer up has no existing container to find and must create a new one. Verified the normal up/gpu-check path still works correctly against the real shared container (including the ordinary restart-a-stopped-container case, which is the behavior this fix deliberately does NOT change for cmd_up itself -- only gpu-recover's last-resort fallback now forces a true recreate). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/devcontainer.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/devcontainer.sh b/scripts/devcontainer.sh index 1399716..7693737 100755 --- a/scripts/devcontainer.sh +++ b/scripts/devcontainer.sh @@ -41,8 +41,8 @@ # already-running container. gpu-check does a real minimal CUDA init via # TensorFlow so it catches that failure mode; gpu-recover tries the cheap # fix first (`docker restart`, which re-runs the NVIDIA container-runtime -# hook and re-binds fresh device state) before falling back to a full -# down+up (which recreates the container from scratch). +# hook and re-binds fresh device state) before falling back to removing +# and recreating the container entirely via `devcontainer up`. set -euo pipefail @@ -170,8 +170,14 @@ cmd_gpu_recover() { echo "gpu-recover: fixed via docker restart." return 0 fi - echo "docker restart did not fix it; falling back to full stop + devcontainer up (recreates the container)..." + echo "docker restart did not fix it; falling back to a full recreate (stop + remove + devcontainer up)..." + # `docker stop` alone is not enough: `devcontainer up` finds and restarts + # an existing (even stopped) container for this workspace rather than + # creating a fresh one, so the stale device bindings this fallback exists + # to clear would otherwise survive. Remove the container so `up` has no + # choice but to create a new one from the image. docker stop "${id}" >/dev/null 2>&1 || true + docker rm "${id}" >/dev/null 2>&1 || true cmd_up cmd_gpu_check }