diff --git a/.gitignore b/.gitignore index cfdda8806..55548cd41 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,9 @@ dist/ # SPDX report project.spdx +# MuJoCo writes this to the working directory, not next to the model. +MUJOCO_LOG.TXT + # robotic_grounding source bundle (cloned locally / in CI from # jiwenc-nv/v2d:retargeter; never committed -- repopulated deterministically # from deps/v2d/version.txt). The Teleop wheel build copies this subtree @@ -66,3 +69,12 @@ deps/v2d/wheels/ # Runtime device files injected by MCP server infrastructure .mcp.json + +# SO-101 leader-gripper assets, fetched by scripts/fetch-so-arm.sh into the +# package's assets directory, so only the authored wrapper XML is tracked. +# +# Keep this rule HERE, not in examples/mujoco_xr/.gitignore: scikit-build-core +# resolves .gitignore against the project root, so a rule there would strip the +# meshes out of the wheel too. +/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/* +!/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml diff --git a/CMakeLists.txt b/CMakeLists.txt index 188304954..85f6b8a85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,7 @@ if(BUILD_EXAMPLES) add_subdirectory(examples/haptic_feedback) if(BUILD_VIZ) add_subdirectory(examples/camera_viz/tests) + add_subdirectory(examples/mujoco_xr) endif() elseif(BUILD_EXAMPLE_TELEOP_ROS2) add_subdirectory(examples/teleop_ros2) diff --git a/docs/source/getting_started/build_from_source/index.rst b/docs/source/getting_started/build_from_source/index.rst index 6238274fa..fc9bee4c2 100644 --- a/docs/source/getting_started/build_from_source/index.rst +++ b/docs/source/getting_started/build_from_source/index.rst @@ -26,6 +26,25 @@ Prerequisites - **uv** for Python dependency management and managed Python - **Internet connection** for downloading dependencies via CMake FetchContent +.. note:: + **Optional — only needed to build the Televiz visualization module,** ``BUILD_VIZ``. + ``BUILD_VIZ`` is auto-detected: it defaults to ``ON`` when all three of the following are + found at configure time and to ``OFF`` otherwise, so a core-only source build still + configures on a machine without them. Watch the + ``-- BUILD_VIZ: (Vulkan=... CUDAToolkit=... glslang=...)`` configure line to see + which one is missing. + + - **Vulkan headers + loader** — ``libvulkan-dev`` on Linux, the LunarG SDK on Windows. + - **CUDA Toolkit** (cudart at link time) — ``nvidia-cuda-toolkit`` or the official NVIDIA + installer. + - **glslangValidator** for compiling shaders to SPIR-V — ``glslang-tools`` on Linux, + ``brew install glslang`` on macOS; ships with the Vulkan SDK on Windows. + + ``BUILD_VIZ=ON`` also pulls in GLFW, whose CMake uses ``pkg_check_modules()`` — install + ``pkg-config`` as well, or the configure fails before viz is reached. Most users do not + need any of this: ``pip install isaacteleop`` already ships the compiled ``isaacteleop.viz`` + module. See `Other Build options`_ for the full option table. + .. _one-time-setup: One time setup diff --git a/examples/mujoco_xr/CMakeLists.txt b/examples/mujoco_xr/CMakeLists.txt new file mode 100644 index 000000000..0a74178b0 --- /dev/null +++ b/examples/mujoco_xr/CMakeLists.txt @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Orchestrator for the MuJoCo XR example. Defines no target of its own. +# +# Configured two ways, and the branches below turn on which: +# +# IN-TREE add_subdirectory'd from the root. Builds the extension in place +# for ctest and a bare `pytest`, and installs nothing. +# STANDALONE top-level, as scikit-build-core drives it for +# `uv pip install ./examples/mujoco_xr`. Sets up for itself what +# root scope provided: Python3_EXECUTABLE, pybind11, output path. +# +# SKBUILD is not the discriminator: a plain `cmake -B build examples/mujoco_xr` +# has none of the root scope either and must fail for the same reasons. + +cmake_minimum_required(VERSION 3.20) + +# CMAKE_SOURCE_DIR is set before any project() call, so this is legal here. +# PROJECT_IS_TOP_LEVEL is not -- it needs the project() this branch is deciding +# whether to make. +if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(_mujoco_xr_standalone TRUE) +else() + set(_mujoco_xr_standalone FALSE) +endif() + +if(_mujoco_xr_standalone) + project(mujoco_xr LANGUAGES CXX) + + # Inherited from the root build in the in-tree case; restated because cpp/ + # compiles as C++20 either way. + set(CMAKE_CXX_STANDARD 20) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + + # Development.Module, not the bare `Development` that + # cmake/SetupPython.cmake:83 uses under SKBUILD: `Development` also demands + # libpython, which the manylinux and uv-managed interpreters that install + # this wheel frequently do not ship. + find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module) + + # From the PEP-517 build environment, not FetchContent: an isolated wheel + # build should not clone from GitHub, and nothing pybind11-typed crosses + # this module's boundary. scikit-build-core puts the build env's + # site-packages on CMAKE_PREFIX_PATH, which is what finds it. + find_package(pybind11 CONFIG REQUIRED) +endif() + +# There is deliberately no `option(BUILD_EXAMPLE_MUJOCO_XR ...)`: the probe +# below is the whole gate, and an option on top would report ON while the +# example was skipped. + +# ============================================================================== +# MuJoCo, discovered through the build interpreter's wheel +# ============================================================================== +# No find_package(mujoco): the wheel is the only supported source and is what +# the Python side loads at runtime, so one libmujoco serves both languages. + +# In-tree only in practice: standalone, find_package(Python3 REQUIRED) above has +# already failed the configure. +if(NOT DEFINED Python3_EXECUTABLE) + message(STATUS "mujoco_xr: skipped (Python3_EXECUTABLE is not defined)") + return() +endif() + +# Not hardcoded: the pyproject.toml files are read and cross-checked against the +# installed version, so drift fails the configure rather than surfacing as an +# ImportError on the headset. MATCHALL because pyproject.toml carries the pin +# twice and those must agree -- and because it would match a version written in +# prose too, which is why those comments never restate the number. +set(_mujoco_pin_files "pyproject.toml" "tests/pyproject.toml") +set(_mujoco_pins "") +set(_mujoco_pin_labels "") +foreach(_pin_file IN LISTS _mujoco_pin_files) + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/${_pin_file}" _pin_file_text) + string(REGEX MATCHALL "mujoco==[0-9][0-9a-zA-Z._-]*" _pin_matches "${_pin_file_text}") + if(NOT _pin_matches) + message(FATAL_ERROR "mujoco_xr: ${_pin_file} declares no `mujoco==` pin; " + "this file reads that pin instead of hardcoding one.") + endif() + foreach(_pin_match IN LISTS _pin_matches) + # MATCHALL populates no CMAKE_MATCH_, so strip the fixed prefix. + string(REPLACE "mujoco==" "" _pin_version "${_pin_match}") + list(APPEND _mujoco_pins "${_pin_version}") + list(APPEND _mujoco_pin_labels "${_pin_file}") + endforeach() +endforeach() +list(GET _mujoco_pins 0 _mujoco_declared_pin) + +# Before the probe, because the cross-check below only runs on a machine that +# has mujoco. Deduplicated into a COPY: _mujoco_pins must keep one entry per +# pin, or the ZIP_LISTS cross-check silently stops checking the later ones. +set(_mujoco_distinct_pins "${_mujoco_pins}") +list(REMOVE_DUPLICATES _mujoco_distinct_pins) +list(LENGTH _mujoco_distinct_pins _mujoco_distinct_pin_count) +if(NOT _mujoco_distinct_pin_count EQUAL 1) + message(FATAL_ERROR "mujoco_xr: the mujoco pins disagree -- pyproject.toml (both " + "build-system.requires and project.dependencies) and " + "tests/pyproject.toml must all name the SAME mujoco version " + "(exactly one libmujoco may be loaded in the process). " + "Found: ${_mujoco_pins} in ${_mujoco_pin_labels}") +endif() + +execute_process( + COMMAND "${Python3_EXECUTABLE}" -c + "import mujoco, os; print(mujoco.__version__); print(os.path.dirname(mujoco.__file__))" + OUTPUT_VARIABLE _mujoco_probe + ERROR_VARIABLE _mujoco_probe_err + RESULT_VARIABLE _mujoco_probe_rc + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(NOT _mujoco_probe_rc EQUAL 0) + # Fatal standalone, where the build IS the example: returning would emit a + # valid wheel with no _mujoco_xr*.so in it, and an unexplained ImportError. + if(_mujoco_xr_standalone) + message(FATAL_ERROR "mujoco_xr: '${Python3_EXECUTABLE}' cannot import mujoco, so the " + "extension cannot be compiled and this wheel would contain no " + "_mujoco_xr*.so at all. pyproject.toml's build-system.requires " + "declares it -- either allow build isolation to install it, or " + "pre-install it into this interpreter: " + "uv pip install --python ${Python3_EXECUTABLE} " + "\"mujoco==${_mujoco_declared_pin}\"") + endif() + # In-tree, the one message between a green build and a silently uncompiled + # example, so it names the exact command. Configure CREATES that + # interpreter, hence the re-configure step in the README. + message(STATUS "mujoco_xr: skipped -- '${Python3_EXECUTABLE}' cannot import mujoco. " + "Install it and re-run cmake --preset with: " + "uv pip install --python ${Python3_EXECUTABLE} \"mujoco==${_mujoco_declared_pin}\"") + return() +endif() + +string(REPLACE "\n" ";" _mujoco_probe_lines "${_mujoco_probe}") +list(GET _mujoco_probe_lines 0 _mujoco_version) +list(GET _mujoco_probe_lines 1 _mujoco_dir) +string(STRIP "${_mujoco_version}" _mujoco_version) +string(STRIP "${_mujoco_dir}" _mujoco_dir) + +# After the probe, so a machine with no mujoco gets the skip message rather than +# a pin complaint. Every pin matters: this module links the installed version's +# SONAME while the pyproject pins decide what the wheel build, the app and ctest +# each resolve -- and exactly one libmujoco may be loaded. +foreach(_pin_file _pin IN ZIP_LISTS _mujoco_pin_labels _mujoco_pins) + # STREQUAL, not VERSION_EQUAL: the regex admits PEP-440 suffixes and + # VERSION_EQUAL discards them, reporting EQUAL for exactly the strings that + # need distinguishing. A pin is an exact string. + if(NOT _pin STREQUAL "${_mujoco_version}") + message(FATAL_ERROR + "mujoco_xr: ${_pin_file} pins mujoco==${_pin}, but '${Python3_EXECUTABLE}' has " + "${_mujoco_version}. The C++ module and the Python app must load ONE libmujoco. Either " + "install the declared pin (uv pip install --python ${Python3_EXECUTABLE} " + "\"mujoco==${_pin}\") or update EVERY pin -- pyproject.toml carries it twice " + "(build-system.requires and project.dependencies) and tests/pyproject.toml once -- " + "to ${_mujoco_version}.") + endif() +endforeach() + +file(GLOB _mujoco_libs "${_mujoco_dir}/libmujoco.so.*") +list(LENGTH _mujoco_libs _mujoco_lib_count) +if(NOT _mujoco_lib_count EQUAL 1) + message(FATAL_ERROR "mujoco_xr: expected exactly one libmujoco.so.* in ${_mujoco_dir}, " + "found ${_mujoco_lib_count}: ${_mujoco_libs}") +endif() +# Lowercase on purpose: hand-set variables read through inherited scope, not the +# find_package output MUJOCO_LIBRARY / MUJOCO_INCLUDE_DIR would imply. +list(GET _mujoco_libs 0 _mujoco_library) +set(_mujoco_include_dir "${_mujoco_dir}/include") +if(NOT EXISTS "${_mujoco_include_dir}/mujoco/mujoco.h") + message(FATAL_ERROR "mujoco_xr: ${_mujoco_include_dir}/mujoco/mujoco.h is missing " + "(is this a source checkout rather than a wheel?)") +endif() + +# The line to grep for: a green build does not imply this example compiled. +message(STATUS "mujoco_xr: ON (mujoco=${_mujoco_version} lib=${_mujoco_library})") + +# Handed down explicitly: ${CMAKE_SOURCE_DIR} means different things in the two +# configures, and "../" is not allowed in CMake paths here. +set(_mujoco_xr_root "${CMAKE_CURRENT_SOURCE_DIR}") + +add_subdirectory(cpp) + +# BUILD_TESTING is undefined standalone, so ctest entries are in-tree only -- +# which is right: they need the in-place extension and the repo's isaacteleop. +if(BUILD_TESTING) + add_subdirectory(tests) +endif() + +# No install() rules here, deliberately: the wheel is the only run path, and +# standalone it is cpp/CMakeLists.txt's install(TARGETS) that fills it. diff --git a/examples/mujoco_xr/README.md b/examples/mujoco_xr/README.md new file mode 100644 index 000000000..db291c99b --- /dev/null +++ b/examples/mujoco_xr/README.md @@ -0,0 +1,447 @@ + + +# MuJoCo XR + +A MuJoCo scene rendered stereoscopically into an Isaac Teleop Televiz XR +session, with an SO-101 leader gripper locked to the operator's right hand. + +Single process, single thread, **one** OpenXR session: + +``` +VizSession(kXr) ──get_oxr_handles()──▶ TeleopSession + │ │ + │ recommended resolution │ controller grip poses + ▼ ▼ +mjr_render ─blit─▶ flip + depth-invert ─glReadPixels─▶ PBO ═CUDA═▶ submit() +``` + +That is the thesis: `VizSession` (rendering) and `TeleopSession` (input) share +one OpenXR session via `get_oxr_handles()`, and **MuJoCo's own renderer** +reaches `ProjectionLayer.submit()` by CUDA pointer with no copy through host +memory. Nothing else in this repository does that. + +**`cpp/` is a readback, not a renderer.** `mjr_render` draws into MuJoCo's +offscreen framebuffer; `cpp/gl_readback.cpp` blits that into a sampleable pair, +runs one fullscreen pass, and reads the result into a pixel-pack buffer that +CUDA imports. Every step stays in video memory. The GL half of `cpp/` — `gl.*`, +`gl_readback.*`, `gl_functions.inc` — is ~700 lines of it, and owns no shading, +no meshes and no camera maths beyond six frustum numbers. + +The trick is which CUDA entry point is used. `cudaGraphicsGLRegisterImage` +registers no depth format and no multisampled renderbuffer, and +`mjrContext.offDepthStencil` is both — that is the wall a naive port hits. +`cudaGraphicsGLRegisterBuffer` has neither restriction, and `glReadPixels` into +a bound `GL_PIXEL_PACK_BUFFER` is a device-to-device transfer, so the CUDA-linear +buffer `submit()` already wants falls straight out of it. + +The fullscreen pass exists for two conversions, both of which are silent +failures on anything short of a headset: + +| | MuJoCo writes | ProjectionLayer is promised | +|---|---|---| +| row 0 | bottom (`glClipControl(GL_LOWER_LEFT, ...)`) | top | +| depth | reverse Z, near → 1 (`GL_GEQUAL`, `glClearDepth(0)`) | near → 0 | + +The depth line is `1.0 - d`, which is exactly what MuJoCo's own +`mjr_readPixels` does on the CPU (`flipDepthIfRequired`, render_gl2.c); doing it +in the shader is what keeps the host out of the loop. + +**What this costs.** One blit, one fullscreen pass and one pack-buffer copy per +eye per frame, all in VRAM, against a path that already copies image → staging +buffer → mailbox array → swapchain. **What it buys:** every geom type, the scene +XML's materials, lights, shadows and reflections, and MuJoCo's own mesh +handling. + +`_mujoco_xr` links `libmujoco`, so this example ships as its own wheel rather +than inside `isaacteleop` — otherwise that wheel's contents would depend on +whether the build host happened to have `mujoco` installed. Exactly one +`libmujoco` may be loaded in the process, because `mjModel*` / `mjData*` +addresses cross the pybind boundary; `__init__.py` imports `mujoco` before the +extension and asserts both report the same version. + +## Status — read this before anything else + +| | | +|---|---| +| **Covered by tests** | [`ctest -L mujoco_xr`](#tests) — the frame conventions, the frustum, the clock, the ghost overlay and its jaw channel, all pure CPU; **plus `test_readback.py`, which drives the real GPU path** (mjr_render → blit → flip/invert → PBO → CUDA). That one needs CUDA-OpenGL interop, so it wants a discrete NVIDIA GPU; it skips loudly elsewhere. **Measured on Jetson/Tegra it skips**, because `cudaGLGetDevices` reports the EGL context on no CUDA device — so a green `ctest` there does *not* mean the GPU path ran. | +| **Never executed anywhere** | **The XR half** — everything downstream of the readback. See [Not verified anywhere](#not-verified-anywhere-in-ci-or-on-a-developer-desktop). | +| **Wrong by construction until calibrated** | The workspace translation, for any scene that adds static content — see [Frames](#frames-cppframeshpp). The shipped ghost-only scene does not show it. | + +Nothing in `.github/workflows/` installs `mujoco`, so the example is never +configured and **not one of its tests has ever run in CI**. Green means one +developer ran it locally. Wiring examples into CI is +[NVIDIA/IsaacTeleop#880](https://github.com/NVIDIA/IsaacTeleop/issues/880). + +## Scope + +Renderer + MuJoCo + rig, and one scene: `assets/scene.xml` — an **SO-101 +leader gripper ghost** locked to the right controller's grip pose, and nothing +else. No table, no blocks, no ground plane: this is an AR scene and passthrough +is the background. + +The ghost is not decoration. It is a real mesh assembly (4 fetched STLs, so it +exercises the `mjGEOM_MESH` path), and locking it to the hand makes the *grip* +calibration visible — whether the tool sits in the hand the way a hand holds +one. It cannot show a wrong `cpp/frames.hpp`: those constants place it, and the +eye pose reaches MuJoCo world through the same ones, so they cancel and the +ghost lands in the hand whatever they say. Only static content shows them, and +the shipped scene has none. + +**Its trigger is driven by the shipped `SO101GripperRetargeter`, as a graph +edge** — the retargeter is a `BaseRetargeter` node inside `_build_pipeline()`, +not a library call beside it, and its closedness output reaches `mjData` and +therefore the screen. There is no robot in the scene, so the jaw it drives is +the operator's own trigger; that is enough to show the edge is live, and the +SO-101 that will read the same output arrives with the scene catalogue. + +Two calibrations, and they are different in kind. `cpp/frames.hpp` is a +*convention* fixed by two specs and cannot be wrong at runtime. +`_QUAT_GRIP_FROM_GHOST` / `_POS_GRIP_FROM_GHOST` in `app.py` are a *measurement* +of how a hand holds a tool, taken on a headset and checkable nowhere else. See +[Frames](#frames-cppframeshpp). + +## Build + +**This example is its own wheel, and the wheel is the only way to run it.** + +```bash +uv pip install "isaacteleop[cloudxr]" --find-links=./install/wheels/ # THIS checkout, not PyPI +uv pip install ./examples/mujoco_xr # same environment +python -m isaacteleop_examples.mujoco_xr # needs a headset +``` + +Both wheels must land in **one** environment, and that is the environment +[`rigs/mujoco_xr.yaml`](../../rigs/mujoco_xr.yaml) runs from. `uv pip install` +compiles the extension through scikit-build-core and does not read the CMake +build tree at all. + +You need `uv`, CMake ≥ 3.20, a C++ compiler, CUDA, and the OpenGL headers +(`libgl-dev` on Debian/Ubuntu — `cuda_gl_interop.h` includes `` +unconditionally, so this is CUDA's requirement as much as ours). No Vulkan and +no `glslangValidator`: the readback shader is a string the driver compiles at +runtime. Nothing is *linked* against OpenGL either — `cpp/gl.hpp` takes the +enums and the `PFNGL...PROC` typedefs from ``, which declares no +symbol, and resolves the 45 entry points in `cpp/gl_functions.inc` through the +platform `GetProcAddress` against the context `mujoco.GLContext` created. +Running the app additionally needs a GPU with EGL + CUDA and a headset. **Build +isolation does not cover the non-Python half of that list**: on a host missing +CUDA or the GL headers the install fails *inside* the isolated PEP-517 build, +with the CMake or compiler error wrapped in backend output. + +**On a multi-GPU host, set `MUJOCO_EGL_DEVICE_ID`.** The OpenGL context has to +land on the same card viz picked, and nothing makes that happen by default — +`MUJOCO_EGL_DEVICE_ID` indexes EGL devices, which need not agree with CUDA's +ordering. The renderer checks at construction and names both device numbers +rather than render into the wrong card's memory. + +**`pip install -e` is not supported.** An editable install redirects the package +back to the source tree, which is exactly where the in-tree CMake build drops +*its* `_mujoco_xr*.so` — you would silently import that one instead, and the +wrong `.so` imports fine right up until `mjModel*` crosses the boundary. To +iterate, `uv pip install --reinstall-package isaacteleop-examples-mujoco-xr +./examples/mujoco_xr` (the CMake cache persists via `build-dir`, so it stays +incremental). `--reinstall-package` rather than a bare reinstall because the +version is fixed at `0.0.0`, so `uv` would otherwise skip the rebuild. + +### The in-tree CMake build, which is a separate thing + +The example is **also** wired into the root build, and that path is what +[`ctest`](#tests) runs against: it builds `_mujoco_xr*.so` in place beside +`python/isaacteleop_examples/mujoco_xr/__init__.py` and installs nothing. + +So the extension is compiled twice — once here for `ctest`, once by +scikit-build-core for the wheel, whose ABI tag comes from whichever interpreter +installs it. That is a deliberate trade: collapsing it means either shipping the +root build's tree as a wheel with no ABI tag, or dropping the in-tree `ctest` +path. It collapses for real the day `ctest` runs against the *installed* wheel, +which needs a locally published `isaacteleop` to resolve against — the one on +PyPI is a different build from the viz in this checkout. + +Steps 1 and 3 are the same command, and the repetition is not decorative: on a +fresh clone the interpreter in step 2 does not exist until configure creates it, +and **the mujoco probe runs at configure time**, so it has to run again once the +wheel is there. + +```bash +# 1. Configure once to create the build venv. This first pass necessarily +# reports `-- mujoco_xr: skipped ...` — expected, not a failure. +cmake --preset py3.12 -DBUILD_VIZ=ON + +# 2. Install mujoco into the interpreter configure just created. `python -m pip` +# does not work: that venv has no pip. +uv pip install --python build/cmake-cpython-312/teleop_build_venv/bin/python "mujoco==3.11.0" + +# 3. Re-configure. NOW the probe finds mujoco and the example is added. +cmake --preset py3.12 -DBUILD_VIZ=ON + +# 4. Build. There is no `cmake --install` step for this example. +cmake --build --preset py3.12 --parallel +``` + +A green build does **not** mean this example compiled. The reliable check: + +```bash +cmake --preset py3.12 -DBUILD_VIZ=ON 2>&1 | grep '^-- mujoco_xr:' +``` + +The `ON` line names the exact `libmujoco.so.*` that was linked. There is no +`BUILD_EXAMPLE_MUJOCO_XR` flag — the gate is `BUILD_VIZ` plus whether `mujoco` +is importable from the interpreter CMake resolved. + +**The same trap applies to the ctest list.** `tests/CMakeLists.txt` globs +`test_*.py` at configure time, so adding or deleting a test file leaves the +entry list stale until you re-run step 3. + +## Run + +```bash +python -m isaacteleop_examples.mujoco_xr --help # includes CloudXRLauncher's flags +``` + +Through the rig, which starts the CloudXR runtime alongside the app, from the +repository root: + +```bash +python -m isaacteleop.rig rigs/mujoco_xr.yaml +``` + +`{python}` in the rig expands to the interpreter you launch it with, so both +wheels have to be installed *there* — not in the build venv, which has no +`isaacteleop`. Picking up the wrong venv is silent, so check before you start: + +```bash +python -c "import sys, isaacteleop; from isaacteleop_examples import mujoco_xr; print(sys.executable, isaacteleop.__file__, mujoco_xr.__file__)" +``` + +Both packages must come from the same `site-packages`; the app's startup log +prints the `isaacteleop:` line for the same reason. Against a runtime you +started yourself: + +```bash +python -m isaacteleop.cloudxr --accept-eula # one terminal +python -m isaacteleop_examples.mujoco_xr --no-launch-cloudxr-runtime # another +``` + +`--no-launch-cloudxr-runtime` is not cosmetic: omitting it makes the app start +its own runtime, which is right when nothing else has and fatal when something +has (the runtime is a host singleton on WSS port 48322). If no runtime is +running and you pass it anyway, the failure comes out of `VizSession.create` as +an OpenXR error before any of this example's code runs — **no `[mujoco_xr]` +lines at all** is the tell. + +There is one scene and no flag to change it: `assets/scene.xml` is package data +beside the module, and editing it is how you load something else. There is no +desktop or headless display mode; without a headset the verification path is +[`ctest -L mujoco_xr`](#tests). + +## Conventions you can break + +### Frames (`cpp/frames.hpp`) + +`R_mj_from_xr = Rz(-90) * Rx(+90)`. XR `-Z` → MuJoCo `+x`, XR `+Y` → MuJoCo +`+z`, XR `+X` → MuJoCo `-y`. Testable definition: a point 1 m in front of the +operator at eye height `h` lands at MuJoCo `(+1, 0, h)` before the workspace +translation. `tests/test_frames.py` checks exactly that. It deliberately differs +from `examples/cloudxr_mujoco_teleop/visualize_poses_mujoco_example.py`, which +applies `Rx(+90)` only (XR-forward → MuJoCo `+y`, not REP-103). + +**`kTransMjFromXr` is the lever, and it is a calibration that is routinely +wrong.** `(-1.0, 0.0, -0.73)`, two independent terms: `x` is operator standoff +(the base sits ~1 m in front of the operator), `z` is a floor datum — MuJoCo +`z = 0` is a work surface 0.73 m above the physical floor. That `z` is only +right against a floor-origin reference space, and the session does not ask for +one: viz's default origin is the headset's start pose, i.e. head height. A +scene that puts static content on the work surface owns re-tuning it. +**Neither term may be zeroed.** + +It places static content only. The ghost goes out through `mj_from_xr` and the +eye pose goes out through the same transform, so both constants cancel on it and +the shipped scene — which is the ghost and nothing else — is blind to a wrong +value. Judging one means a scene with something world-locked in it. + +There is no recentre keypress and no runtime override: changing the datum means +editing the constant and rebuilding (~8 s). The procedure is to stand where you +intend to work, start the app on such a scene, read the `frames:` line in the +startup log, compare the virtual surface against the real one, and adjust `z`. A +`--workspace-offset` flag was considered and rejected: a Python-side offset +applied to one of the two conversions and not the other would move the gripper +and leave the scene put, which is precisely the symptom this example exists to +disambiguate. + +### Where the ghost sits on the hand (`app.py`) + +A *second* calibration, and a different kind: `_EULER_GRIP_FROM_GHOST_DEG` and +`_POS_GRIP_FROM_GHOST` place the leader gripper on the operator's hand. Without +them the gripper's body origin — the follower's `gripper` datum, up at the wrist +— lands on the grip pose, so the tool hangs off the hand at an arbitrary angle. + +**These are measured on a headset, not derived.** That is the whole provenance: +it is a claim about how a gripper should look in a hand that is actually holding +a *controller*, and nothing headless can settle it. + +A mesh-derived version was tried first and hardware overruled it. It mapped the +handle loop's principal axis onto the fist axis, the loop's centroid onto the +palm, and the jaw assembly forward of the knuckles — i.e. it assumed the hand +goes *through* the loop, the way it would on the real leader device. Measured +against the shipped values, that model puts the loop centroid 56 mm from the +palm and not straddling it at all. The premise was wrong: you are gripping a +controller, so where the loop falls is a question about the controller in the +hand, not about the loop. + +The mesh geometry is still worth knowing when reading the numbers. +`Handle_SO101` is a closed **loop**, not a bar; the jaw assembly sits off to one +side of it, and the jaws run **60.7°** off the loop's long axis. The OpenXR +**grip** frame they are expressed in (`grip/pose`, not `aim/pose`) is `−Z` little +finger → thumb, `+X` into the palm, `+Y` forward through the knuckles. + +**To re-tune.** The rotation is degrees, intrinsic X-then-Y-then-Z — the same +convention as a MuJoCo `euler=` attribute, pinned by a test against a compiled +model rather than asserted here. Change one angle, `uv pip install +--reinstall-package isaacteleop-examples-mujoco-xr ./examples/mujoco_xr`, +relaunch: `Rz` spins the gripper about its own long axis, `Rx` / `Ry` tilt it in +the hand, and `_POS_GRIP_FROM_GHOST` slides it along the grip axes if the angle +is right but the placement is not. **No test asserts a posture**, deliberately — +they cover the machinery, so re-tuning cannot turn them red. The one that +matters asserts the ghost is *rigidly attached* to the grip frame, which is +true of any calibration and false if the correction is composed on the wrong +side. + +**A trap worth keeping even though the derivation is retired.** MuJoCo rewrites +every mesh into its inertial frame, so recovering an STL's own axes needs +`mesh_pos` / `mesh_quat`. Skip that and you get the *handle's* axis back instead +of the jaws', which is self-consistent, passes an axis-only check, and is wrong +by 60°. The shank's own principal axis is no substitute either — it is a +near-isotropic blob (σ₀/σ₁ = 1.26), so its principal direction is noise. + +### Scene assets + +Every geom type draws, and the XML's materials, lights, shadows and +reflections are live — this is `mjr_render`, so the scene file means what the +MuJoCo docs say it means. + +**The lighting knob that matters is ambient, not diffuse.** `scene.xml` sets +``. Ambient is direction-independent, so it is a *floor* +on how dark a surface can get; diffuse is what carries shape. MuJoCo's own +defaults are why this scene read as dark — not because they are dim overall, but +because the floor under them is 0.1. Measured over the ghost from three +directions, as a share of its material albedo: + +| headlight (amb / diff / spec) | shades | dimmest | mean | below ⅓ albedo | above albedo | +|---|---|---|---|---|---| +| `0.1 / 0.4 / 0.5` (MuJoCo default) | 437 | 0.10 | 0.25 | **94.0%** | 0% | +| `0.4 / 0.4 / 0.3` (shipped) | 372 | **0.40** | 0.55 | 0% | 0% | + +The trade is explicit: the shipped values give up some tonal range — 372 +distinct shades against the default's 437 — to buy a hard floor. The dimmest +pixel is 0.40 of albedo, which is the ambient term exactly. + +That floor earns its keep twice over. It bounds MuJoCo's smeared crease normals +— one averaged normal per welded vertex, and `render_gl3.c` lights one-sided, so +a face corner pointing away from its own triangle (11.4% of them on +`wrist_roll`) lands on ambient rather than on black, which is a tonal wobble +instead of shattered facets. And it bounds shadows the same way, so +`mjRND_SHADOW` needs no attention. + +Specular is the one term that spends *outside* that budget: it is additive and +white rather than scaled by the material `rgba`, and it is gated by the material +as much as the light — `leader_gripper.xml` declares neither `specular` nor +`shininess`, so MuJoCo's defaults (0.5 and 0.5) apply and the effective highlight +is `0.3 × 0.5`. Ambient plus diffuse comes to 0.8, and that 0.2 of headroom is +what absorbs it: no pixel exceeds the albedo at these values. Raising either +term without lowering the other is what would start clipping. + +**The remaining defect: the headlight is not head-mounted here.** +`mjv_updateScene` bakes it into `mjvScene.lights[0]` from the `mjvCamera` it is +passed, and this app passes a fixed `mjv_defaultFreeCamera` and only overwrites +`mjvScene.camera` afterwards. It is a directional light fixed in MuJoCo world by +`model.vis.global_.azimuth` / `elevation`, and it never follows the head. The +ambient floor makes that survivable rather than correct — the ghost stays legible +at every hand orientation, but which side of it is lit depends on where in the +room the hand is, not on where the operator is looking. Two ways to fix it +properly: write `mjvScene.lights[]` in `render()` after the cameras (`mjr_render` +reads the array as you leave it, and `dir` is the camera's `forward`, un-negated +— that measures 0.71 mean against the 0.55 above, and independent of hand pose), +or give the scene its own `` elements, which `mjv_updateScene` does place +correctly. + +The ghost's four STLs are **fetched, not vendored** — 2.3 MB of binary in a +source tree is a poor trade when upstream publishes them at a stable commit, and +Git LFS made every clone pay for them. Run it once, then reinstall, because they +are package data: + +```bash +examples/mujoco_xr/scripts/fetch-so-arm.sh # from the repository root +uv pip install --reinstall-package isaacteleop-examples-mujoco-xr ./examples/mujoco_xr +``` + +Nothing fetches at build time: an isolated PEP-517 wheel build must not reach +the network, so the app fails at startup naming the script and `test_ghost.py` +**skips** with the same reason. Downloads are checksum-verified against a pinned +commit — a silently substituted mesh renders as a broken gripper rather than an +error, which has already cost a debugging session. + +The script also pulls `so101_new_calib.urdf`, which is where the trigger's hinge +and its 0..100° travel come from, so it is on disk to check them against. Three of the four +meshes are leader-specific print parts; the fourth is the **STS3215 servo**, +shared with the follower. It is not decoration — `wrist_roll` is a C-shaped +bracket that wraps the servo, so without it the assembly has an open notch where +the motor belongs and reads as a broken asset. + +It declares **two** mocap bodies — the gripper and its trigger — because the +trigger articulates; a jointed child of a mocap body would be a dynamic joint +that `mj_step` integrates gravity into, and a mocap body is kinematic by +construction. + +The ghost is **opaque**, and `test_ghost.py` asserts it. That removes the +draw-order constraint (at alpha 1.0 the depth test decides everything) and the +ghost-writes-depth-into-the-reprojection-buffer concern. A scene that puts a +robot under the ghost and drops the alpha back takes both on again: `mjv_updateScene` +emits in geom-id order, so the `` must come **last**. Nothing asserts +that ordering today — it only matters below alpha 1.0, so the test belongs with +the scene that needs it. + +**Pass MuJoCo an absolute scene path.** Measured on mujoco 3.11.0, a *relative* +model path mis-composes the mesh paths of an ``d file in a +subdirectory and fails with `Error opening file ''`. +`DEFAULT_SCENE` in `app.py` is absolute for this reason. + +## Tests + +```bash +ctest --test-dir build/cmake-cpython-312 -L mujoco_xr --output-on-failure +``` + +| file | covers | +|---|---| +| `test_frames.py` | the XR→MuJoCo axis map and quaternion order | +| `test_projection.py` | the mjvGLCamera frustum (that it is the fov projected onto the near plane, and that the half-width is set so mjr_render's aspect fallback stays off) and the standard-Z depth contract | +| `test_app_helpers.py` | the NaN-safe `dt` clamp, the zeroed-`predicted_display_time` guard, and that the first-frame frustum assertion passes on the real thing and fires on each way it can go wrong | +| `test_readback.py` | **the GPU path**: that something is drawn at all, that row 0 is the top of the operator's view and the image is not mirrored, that the depth handed to `submit()` is standard Z with the background at exactly 1.0, and that the two eyes carry parallax of the right sign. Skips with a reason when there is no GPU | +| `test_ghost.py` | the overlay: that the ghost is opaque, collision-free and carries no mass, that both its bodies are kinematic mocap bodies with no joint anywhere, that the four leader parts form one assembly with sub-mm gaps at the bolted joints and the servo seated in its bracket, that the print STLs are scaled from millimetres and the servo is not, that the ghost is *rigidly attached* to the grip frame whatever the calibration, that squeezing swings the trigger monotonically from the URDF joint's upper limit to its authored zero without driving the lever through the body, that the shipped `SO101GripperRetargeter` really is the thing driving that channel (built as a real pipeline and fed synthetic DeviceIO snapshots), and that an untracked controller freezes the whole gripper rather than parking it at the scene origin | + +All but `test_readback.py` run on a CPU with no GPU, no headset, no CloudXR +runtime and no window system; keep it that way, because a permanently-skipping +test reports green while covering nothing. `test_readback.py` is the deliberate +exception: what it covers is otherwise invisible until someone is wearing a +headset, and it needs no headset itself. + +## Not verified anywhere in CI or on a developer desktop + +**Everything downstream of the readback.** `ProjectionLayer.submit()`, the +frame loop that sequences it, OpenXR session sharing via `oxr_handles`, whether +the runtime accepts the depth layer, and **controllers on a shared session** — +none of it is executed by any test or on any machine here. `test_readback.py` +covers the render and the CUDA hand-off and stops at `submit()`. How the ghost +*looks* is unverified too, and so is the grip-to-gripper calibration, by +construction — `tests/test_ghost.py` pins the *machinery* and leaves the shipped +constants free to be tuned. + +Controllers on a shared session have no precedent elsewhere in this repository: +`xrAttachSessionActionSets` is legal once per `XrSession`, Teleop sidesteps it +with `XR_NVX1_action_context`, and the one existing shared-session example +(`examples/oglo_tactile`) exercises only Hand and Head trackers, which use no +actions. Treat that as the likeliest first-run blocker. diff --git a/examples/mujoco_xr/cpp/CMakeLists.txt b/examples/mujoco_xr/cpp/CMakeLists.txt new file mode 100644 index 000000000..962cebe89 --- /dev/null +++ b/examples/mujoco_xr/cpp/CMakeLists.txt @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# The pybind11 module `_mujoco_xr` -- MuJoCo's own OpenGL renderer, read back +# into CUDA-visible buffers for viz::ProjectionLayer. +# +# Protocol-only linkage: no viz:: target, because __cuda_array_interface__ is +# the whole interface, and for the same reason this pybind11 need not be the one +# the root build FetchContents. Do not pin them together. +# +# No OpenGL on the link line either -- gl.hpp resolves through the platform +# GetProcAddress, and libGL here would be a second dispatch path for one +# context. Its HEADERS are already implied by CUDA::cudart. +# +# Inherited-scope inputs, all set by ../CMakeLists.txt: +# _mujoco_library, _mujoco_include_dir, _mujoco_xr_root, _mujoco_xr_standalone + +cmake_minimum_required(VERSION 3.20) + +find_package(CUDAToolkit REQUIRED) + +pybind11_add_module(mujoco_xr_py + mujoco_xr_bindings.cpp + gl.cpp + gl_readback.cpp + scene_renderer.cpp + frames.hpp + gl.hpp + gl_functions.inc + glcamera.hpp + gl_readback.hpp + scene_renderer.hpp +) + +target_include_directories(mujoco_xr_py + PRIVATE + "${_mujoco_include_dir}" +) + +target_link_libraries(mujoco_xr_py + PRIVATE + # Matches viz_core: no runtime libcudart.so on a driver-only machine. + CUDA::cudart_static + # gl.cpp's loader needs dlopen/dlsym. + ${CMAKE_DL_LIBS} + "${_mujoco_library}" +) + +target_compile_options(mujoco_xr_py PRIVATE -Wall -Wextra) + +set_target_properties(mujoco_xr_py PROPERTIES + OUTPUT_NAME "_mujoco_xr" + # No RPATH, deliberately -- do not "fix" this. __init__.py imports `mujoco` + # first, so the already-loaded library satisfies our NEEDED entry. An RPATH + # would silently load a second libmujoco and hand mjModel* across two + # copies; without one, a mismatch is a clean ImportError. + BUILD_WITH_INSTALL_RPATH ON + INSTALL_RPATH "" +) + +# ============================================================================== +# Where the .so goes, and it is a different place in each configure +# ============================================================================== +if(_mujoco_xr_standalone) + # CMAKE_INSTALL_PREFIX is scikit-build-core's platlib staging root, so the + # DESTINATION must spell the namespace too -- drop `isaacteleop_examples/` + # and the .so lands outside the package. `sdist.exclude` in + # ../pyproject.toml is what stops a stale in-place .so shipping alongside + # it, and that breakage is intermittent: only a cross-ABI build ships two. + install(TARGETS mujoco_xr_py + LIBRARY DESTINATION isaacteleop_examples/mujoco_xr + ) +else() + # In-tree: drop the .so beside __init__.py, which is what tests/conftest.py + # reaches by prepending python/ to sys.path. ${_mujoco_xr_root} rather than + # ${CMAKE_SOURCE_DIR}, which is this directory in the standalone configure. + # This copies and never removes, so a renamed module leaves a stale .so. + set_target_properties(mujoco_xr_py PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${_mujoco_xr_root}/python/isaacteleop_examples/mujoco_xr" + ) +endif() diff --git a/examples/mujoco_xr/cpp/frames.hpp b/examples/mujoco_xr/cpp/frames.hpp new file mode 100644 index 000000000..369fd2fe5 --- /dev/null +++ b/examples/mujoco_xr/cpp/frames.hpp @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// The app's only XR<->MuJoCo frame crossing; Python reaches it through the +// pybind module rather than re-deriving it. Rules: +// +// - Quaternions cross as xyzw (OpenXR, and so GRIP_ORIENTATION); MuJoCo is +// wxyz and viz::Pose3D a third spelling. Reorder on every crossing, name +// every variable q_xyzw or q_wxyz, fill boundary structs field by field. +// - R_mj_from_xr = Rz(-90deg) * Rx(+90deg): XR -Z -> MJ +x, +Y -> +z, +// +X -> -y. p_mj = R * p_xr + t; q_mj = q_mj_from_xr (x) q_xr. +// - Name conversions `_from_` so they read by adjacency +// (p_mj = mj_from_xr_pos(p_xr)), never the A_T_B robotics form. + +#include + +#include + +namespace mujoco_xr +{ + +// A handedness convention, fixed by the two specs (OpenXR is y-up / +// -z-forward, MuJoCo is REP-103 z-up), so it cannot be wrong at runtime. If a +// scene's static content appears rotated 90 degrees, this is the bug; the ghost +// cannot show it, because it is placed through this transform and the eye pose +// goes into MuJoCo world through the same one, so it cancels. +// +// Deliberately diverges from +// examples/cloudxr_mujoco_teleop/visualize_poses_mujoco_example.py, which +// applies Rx(+90) only and maps XR-forward to MuJoCo +y, which is not REP-103. +// Do not "fix" this constant to match it. +inline constexpr std::array kQuatMjFromXr = { 0.5, 0.5, -0.5, -0.5 }; // wxyz + +// A workspace calibration, routinely wrong, and it places static scene content +// only -- it cancels on the ghost, so the shipped scene cannot show it. Two +// independent terms, and zeroing either is a bug: +// x = -1.0 operator standoff, independent of the reference space. +// z = -0.73 floor datum: MuJoCo z=0 is a work surface 0.73 m above the floor. +// Only right against a floor-origin reference space, which this +// session does not ask for -- viz's origin is the headset's start +// pose. A scene with static content owns re-tuning it. +inline constexpr std::array kTransMjFromXr = { -1.0, 0.0, -0.73 }; + +// XR (xyzw) -> MuJoCo world (wxyz). The app's only quaternion crossing. +inline std::array mj_from_xr_quat(const std::array& q_xyzw) +{ + const mjtNum q_wxyz[4] = { q_xyzw[3], q_xyzw[0], q_xyzw[1], q_xyzw[2] }; // reorder + std::array out{}; + mju_mulQuat(out.data(), kQuatMjFromXr.data(), q_wxyz); + return out; +} + +// XR reference-space point -> MuJoCo world point: R * p + t. +inline std::array mj_from_xr_pos(const std::array& p_xr) +{ + std::array out{}; + mju_rotVecQuat(out.data(), p_xr.data(), kQuatMjFromXr.data()); + for (int i = 0; i < 3; ++i) + { + out[i] += kTransMjFromXr[i]; + } + return out; +} + +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/cpp/gl.cpp b/examples/mujoco_xr/cpp/gl.cpp new file mode 100644 index 000000000..39a7b5414 --- /dev/null +++ b/examples/mujoco_xr/cpp/gl.cpp @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "gl.hpp" + +#include +#include +#include + +namespace mujoco_xr +{ +namespace gl +{ + +#define MUJOCO_XR_GL(name, upper) PFNGL##upper##PROC name = nullptr; +#include "gl_functions.inc" +#undef MUJOCO_XR_GL + +namespace +{ + +using ProcLoader = void* (*)(const char*); + +bool loaded_ = false; + +// RTLD_NOLOAD first: we want the copy the process already loaded, since that is +// the one `mujoco.GLContext` made a context on -- a second copy resolves against +// a dispatch table with no current context. Plain dlopen as a fallback. +void* open_already_loaded(const char* soname) +{ + void* handle = dlopen(soname, RTLD_LAZY | RTLD_NOLOAD); + if (handle == nullptr) + { + handle = dlopen(soname, RTLD_LAZY); + } + return handle; +} + +// eglGetProcAddress / glXGetProcAddress, whichever this process has: both +// return the libglvnd stub for the calling thread's current context, so either +// serves whatever MUJOCO_GL selected. EGL first because headless is this +// example's only mode. +ProcLoader find_proc_loader() +{ + static constexpr struct + { + const char* soname; + const char* symbol; + } kCandidates[] = { + { "libEGL.so.1", "eglGetProcAddress" }, + { "libGLX.so.0", "glXGetProcAddressARB" }, + { "libGL.so.1", "glXGetProcAddressARB" }, + { "libGL.so.1", "glXGetProcAddress" }, + }; + for (const auto& candidate : kCandidates) + { + void* handle = open_already_loaded(candidate.soname); + if (handle == nullptr) + { + continue; + } + if (void* sym = dlsym(handle, candidate.symbol)) + { + return reinterpret_cast(sym); + } + } + throw std::runtime_error( + "mujoco_xr: found neither eglGetProcAddress nor glXGetProcAddress. The OpenGL " + "context must be created (mujoco.GLContext) BEFORE the renderer, on this thread."); +} + +// Deduces the pointer type from the target, so no caller repeats a cast. +template +void resolve(ProcLoader get_proc, Fn& out, const char* name) +{ + void* sym = get_proc(name); + if (sym == nullptr) + { + throw std::runtime_error(std::string("mujoco_xr: OpenGL entry point ") + name + + " is unavailable. Either no context is current on this thread, or it is older " + "than OpenGL 3.3."); + } + out = reinterpret_cast(sym); +} + +} // namespace + +void load() +{ + if (loaded_) + { + return; + } + const ProcLoader get_proc = find_proc_loader(); + +#define MUJOCO_XR_GL(name, upper) resolve(get_proc, name, "gl" #name); +#include "gl_functions.inc" +#undef MUJOCO_XR_GL + + loaded_ = true; +} + +bool loaded() +{ + return loaded_; +} + +void check(const char* what) +{ + GLenum first = GL_NO_ERROR; + for (GLenum err = GetError(); err != GL_NO_ERROR; err = GetError()) + { + if (first == GL_NO_ERROR) + { + first = err; + } + } + if (first == GL_NO_ERROR) + { + return; + } + static const char kHex[] = "0123456789abcdef"; + std::string code = "0x"; + for (int shift = 12; shift >= 0; shift -= 4) + { + code.push_back(kHex[(first >> shift) & 0xF]); + } + throw std::runtime_error("mujoco_xr: OpenGL error " + code + " during " + what); +} + +} // namespace gl +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/cpp/gl.hpp b/examples/mujoco_xr/cpp/gl.hpp new file mode 100644 index 000000000..1dfb2233b --- /dev/null +++ b/examples/mujoco_xr/cpp/gl.hpp @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// The OpenGL entry points this module calls (gl_functions.inc), resolved at +// runtime against the context `mujoco.GLContext` created. Nothing links libGL: +// glcorearb.h supplies the enums and PFNGL...PROC typedefs but declares no +// function, GL_GLEXT_PROTOTYPES being undefined. It costs no build dependency +// either -- it ships beside , which cuda_gl_interop.h always includes. + +#include + +namespace mujoco_xr +{ +namespace gl +{ + +#define MUJOCO_XR_GL(name, upper) extern PFNGL##upper##PROC name; +#include "gl_functions.inc" +#undef MUJOCO_XR_GL + +// Resolves every entry point above against the CURRENT context. Idempotent. +// Throws naming the first unresolvable one, which means either no current +// context or one older than OpenGL 3.3. +void load(); + +// Whether load() succeeded. Teardown paths need it: they are reached with +// nothing loaded when construction threw. +bool loaded(); + +// Throws naming `what` if glGetError() is set. Drains the queue either way, so +// one stale error cannot fail every later check. +void check(const char* what); + +} // namespace gl +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/cpp/gl_functions.inc b/examples/mujoco_xr/cpp/gl_functions.inc new file mode 100644 index 000000000..83bb29164 --- /dev/null +++ b/examples/mujoco_xr/cpp/gl_functions.inc @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Every OpenGL entry point this module calls: the name we call it by, then the +// spelling its glcorearb.h PFNGL...PROC typedef uses. +// +// No include guard, deliberately: gl.hpp and gl.cpp include this three times +// over with MUJOCO_XR_GL defined differently -- declare, define, load -- so no +// two lists can drift and leave an entry point null at the first call. + +MUJOCO_XR_GL(Enable, ENABLE) +MUJOCO_XR_GL(Disable, DISABLE) +MUJOCO_XR_GL(GetError, GETERROR) +MUJOCO_XR_GL(Viewport, VIEWPORT) +MUJOCO_XR_GL(PixelStorei, PIXELSTOREI) +MUJOCO_XR_GL(GetIntegerv, GETINTEGERV) +MUJOCO_XR_GL(ReadPixels, READPIXELS) +MUJOCO_XR_GL(DrawArrays, DRAWARRAYS) +MUJOCO_XR_GL(GenTextures, GENTEXTURES) +MUJOCO_XR_GL(DeleteTextures, DELETETEXTURES) +MUJOCO_XR_GL(BindTexture, BINDTEXTURE) +MUJOCO_XR_GL(TexImage2D, TEXIMAGE2D) +MUJOCO_XR_GL(TexParameteri, TEXPARAMETERI) +MUJOCO_XR_GL(ActiveTexture, ACTIVETEXTURE) +MUJOCO_XR_GL(GenFramebuffers, GENFRAMEBUFFERS) +MUJOCO_XR_GL(DeleteFramebuffers, DELETEFRAMEBUFFERS) +MUJOCO_XR_GL(BindFramebuffer, BINDFRAMEBUFFER) +MUJOCO_XR_GL(FramebufferTexture2D, FRAMEBUFFERTEXTURE2D) +MUJOCO_XR_GL(CheckFramebufferStatus, CHECKFRAMEBUFFERSTATUS) +MUJOCO_XR_GL(GetFramebufferAttachmentParameteriv, GETFRAMEBUFFERATTACHMENTPARAMETERIV) +MUJOCO_XR_GL(BlitFramebuffer, BLITFRAMEBUFFER) +MUJOCO_XR_GL(DrawBuffers, DRAWBUFFERS) +MUJOCO_XR_GL(ReadBuffer, READBUFFER) +MUJOCO_XR_GL(GenBuffers, GENBUFFERS) +MUJOCO_XR_GL(DeleteBuffers, DELETEBUFFERS) +MUJOCO_XR_GL(BindBuffer, BINDBUFFER) +MUJOCO_XR_GL(BufferData, BUFFERDATA) +MUJOCO_XR_GL(GenVertexArrays, GENVERTEXARRAYS) +MUJOCO_XR_GL(DeleteVertexArrays, DELETEVERTEXARRAYS) +MUJOCO_XR_GL(BindVertexArray, BINDVERTEXARRAY) +MUJOCO_XR_GL(CreateShader, CREATESHADER) +MUJOCO_XR_GL(ShaderSource, SHADERSOURCE) +MUJOCO_XR_GL(CompileShader, COMPILESHADER) +MUJOCO_XR_GL(GetShaderiv, GETSHADERIV) +MUJOCO_XR_GL(GetShaderInfoLog, GETSHADERINFOLOG) +MUJOCO_XR_GL(DeleteShader, DELETESHADER) +MUJOCO_XR_GL(CreateProgram, CREATEPROGRAM) +MUJOCO_XR_GL(AttachShader, ATTACHSHADER) +MUJOCO_XR_GL(LinkProgram, LINKPROGRAM) +MUJOCO_XR_GL(GetProgramiv, GETPROGRAMIV) +MUJOCO_XR_GL(GetProgramInfoLog, GETPROGRAMINFOLOG) +MUJOCO_XR_GL(DeleteProgram, DELETEPROGRAM) +MUJOCO_XR_GL(UseProgram, USEPROGRAM) +MUJOCO_XR_GL(GetUniformLocation, GETUNIFORMLOCATION) +MUJOCO_XR_GL(Uniform1i, UNIFORM1I) diff --git a/examples/mujoco_xr/cpp/gl_readback.cpp b/examples/mujoco_xr/cpp/gl_readback.cpp new file mode 100644 index 000000000..a3840421c --- /dev/null +++ b/examples/mujoco_xr/cpp/gl_readback.cpp @@ -0,0 +1,398 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "gl_readback.hpp" + +#include +#include +#include +#include +#include + +namespace mujoco_xr +{ + +using namespace gl; + +namespace +{ + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + { + throw std::runtime_error(std::string("mujoco_xr: ") + what + " failed: " + cudaGetErrorString(err)); + } +} + +// gl_VertexID -> one viewport-covering triangle, so there is no vertex buffer. +constexpr const char* kVertexSource = R"glsl( +#version 330 core +out vec2 vUv; +void main() +{ + vec2 p = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); + vUv = p; + gl_Position = vec4(p * 2.0 - 1.0, 0.0, 1.0); +} +)glsl"; + +// The two conversions the XR layer needs, in the one place they happen. +constexpr const char* kFragmentSource = R"glsl( +#version 330 core +uniform sampler2D uColor; +uniform sampler2D uDepth; +in vec2 vUv; +layout(location = 0) out vec4 oColor; +layout(location = 1) out float oDepth; +void main() +{ + vec2 uv = vec2(vUv.x, 1.0 - vUv.y); // GL bottom-up -> XR top-down + oColor = texture(uColor, uv); + oDepth = 1.0 - texture(uDepth, uv).r; // mjr_render reverse Z -> near 0, far 1 +} +)glsl"; + +GLuint compile(GLenum stage, const char* source) +{ + const GLuint shader = CreateShader(stage); + ShaderSource(shader, 1, &source, nullptr); + CompileShader(shader); + GLint ok = GL_FALSE; + GetShaderiv(shader, GL_COMPILE_STATUS, &ok); + if (ok != static_cast(GL_TRUE)) + { + GLint len = 0; + GetShaderiv(shader, GL_INFO_LOG_LENGTH, &len); + std::string log(static_cast(len > 0 ? len : 1), '\0'); + GetShaderInfoLog(shader, len, nullptr, log.data()); + DeleteShader(shader); + throw std::runtime_error("mujoco_xr: readback shader failed to compile: " + log); + } + return shader; +} + +GLuint make_texture(GLenum internal_format, GLenum format, GLenum type, uint32_t width, uint32_t height) +{ + GLuint tex = 0; + GenTextures(1, &tex); + BindTexture(GL_TEXTURE_2D, tex); + TexImage2D(GL_TEXTURE_2D, 0, static_cast(internal_format), static_cast(width), + static_cast(height), 0, format, type, nullptr); + TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + BindTexture(GL_TEXTURE_2D, 0); + return tex; +} + +// A pack buffer and its CUDA registration. ReadOnly: submit() only reads it. +void make_pbo(size_t size_bytes, GLuint* out_pbo, void** out_resource) +{ + GenBuffers(1, out_pbo); + BindBuffer(GL_PIXEL_PACK_BUFFER, *out_pbo); + BufferData(GL_PIXEL_PACK_BUFFER, static_cast(size_bytes), nullptr, GL_STREAM_READ); + BindBuffer(GL_PIXEL_PACK_BUFFER, 0); + check("pixel pack buffer allocation"); + + cudaGraphicsResource_t res = nullptr; + check_cuda(cudaGraphicsGLRegisterBuffer(&res, *out_pbo, cudaGraphicsRegisterFlagsReadOnly), + "cudaGraphicsGLRegisterBuffer"); + *out_resource = res; +} + +// `src_fbo`'s depth format, as the triple a matching texture needs. +struct DepthFormat +{ + GLenum internal_format; + GLenum format; + GLenum type; +}; + +// Restores the draw-framebuffer binding this object was constructed under. +struct ScopedDrawFramebuffer +{ + GLint previous = 0; + + ScopedDrawFramebuffer() + { + GetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, &previous); + } + ~ScopedDrawFramebuffer() + { + BindFramebuffer(GL_FRAMEBUFFER, static_cast(previous)); + } + + ScopedDrawFramebuffer(const ScopedDrawFramebuffer&) = delete; + ScopedDrawFramebuffer& operator=(const ScopedDrawFramebuffer&) = delete; +}; + +DepthFormat depth_format_of(GLuint src_fbo) +{ + BindFramebuffer(GL_READ_FRAMEBUFFER, src_fbo); + GLint component_type = 0; + GetFramebufferAttachmentParameteriv( + GL_READ_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_FRAMEBUFFER_ATTACHMENT_COMPONENT_TYPE, &component_type); + BindFramebuffer(GL_READ_FRAMEBUFFER, 0); + check("querying the MuJoCo offscreen depth format"); + + if (static_cast(component_type) == GL_FLOAT) + { + return { GL_DEPTH32F_STENCIL8, GL_DEPTH_STENCIL, GL_FLOAT_32_UNSIGNED_INT_24_8_REV }; + } + return { GL_DEPTH24_STENCIL8, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8 }; +} + +} // namespace + +Readback::~Readback() +{ + destroy(); +} + +void Readback::build_program() +{ + const GLuint vs = compile(GL_VERTEX_SHADER, kVertexSource); + const GLuint fs = compile(GL_FRAGMENT_SHADER, kFragmentSource); + program_ = CreateProgram(); + AttachShader(program_, vs); + AttachShader(program_, fs); + LinkProgram(program_); + DeleteShader(vs); + DeleteShader(fs); + + GLint ok = GL_FALSE; + GetProgramiv(program_, GL_LINK_STATUS, &ok); + if (ok != static_cast(GL_TRUE)) + { + GLint len = 0; + GetProgramiv(program_, GL_INFO_LOG_LENGTH, &len); + std::string log(static_cast(len > 0 ? len : 1), '\0'); + GetProgramInfoLog(program_, len, nullptr, log.data()); + throw std::runtime_error("mujoco_xr: readback program failed to link: " + log); + } + + UseProgram(program_); + Uniform1i(GetUniformLocation(program_, "uColor"), 0); + Uniform1i(GetUniformLocation(program_, "uDepth"), 1); + UseProgram(0); + check("readback program setup"); +} + +void Readback::create(uint32_t width, uint32_t height, uint32_t view_count, GLuint src_fbo) +{ + if (width == 0 || height == 0 || view_count == 0) + { + throw std::invalid_argument("mujoco_xr: readback needs a non-empty size and at least one view"); + } + load(); + width_ = width; + height_ = height; + + const ScopedDrawFramebuffer restore_binding; + + build_program(); + GenVertexArrays(1, &vao_); + + const DepthFormat depth = depth_format_of(src_fbo); + views_.resize(view_count); + for (View& v : views_) + { + v.blit_color = make_texture(GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, width, height); + v.blit_depth = make_texture(depth.internal_format, depth.format, depth.type, width, height); + GenFramebuffers(1, &v.blit_fbo); + BindFramebuffer(GL_FRAMEBUFFER, v.blit_fbo); + FramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, v.blit_color, 0); + FramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_TEXTURE_2D, v.blit_depth, 0); + if (CheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + { + throw std::runtime_error("mujoco_xr: the readback blit framebuffer is incomplete"); + } + + v.out_color = make_texture(GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, width, height); + v.out_depth = make_texture(GL_R32F, GL_RED, GL_FLOAT, width, height); + GenFramebuffers(1, &v.out_fbo); + BindFramebuffer(GL_FRAMEBUFFER, v.out_fbo); + FramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, v.out_color, 0); + FramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT1, GL_TEXTURE_2D, v.out_depth, 0); + if (CheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + { + throw std::runtime_error("mujoco_xr: the readback output framebuffer is incomplete"); + } + + const size_t pixels = static_cast(width) * height; + make_pbo(pixels * 4, &v.color_pbo, &v.color_resource); + make_pbo(pixels * sizeof(float), &v.depth_pbo, &v.depth_resource); + } + check("readback resource creation"); +} + +void Readback::unmap(View& v, bool throw_on_error) +{ + if (!v.mapped) + { + return; + } + cudaGraphicsResource_t resources[2] = { static_cast(v.color_resource), + static_cast(v.depth_resource) }; + const cudaError_t err = cudaGraphicsUnmapResources(2, resources, nullptr); + v.mapped = false; + v.color_device_ptr = nullptr; + v.depth_device_ptr = nullptr; + if (throw_on_error) + { + check_cuda(err, "cudaGraphicsUnmapResources"); + } +} + +void Readback::capture(uint32_t view, GLuint src_fbo) +{ + if (view >= views_.size()) + { + throw std::out_of_range("mujoco_xr: readback view index out of range"); + } + View& v = views_[view]; + // glReadPixels below is graphics access, illegal while CUDA holds them. + unmap(v, /*throw_on_error=*/true); + + // mjr_render draws into whatever is bound when it is called. Leaving our + // own framebuffer bound sends the NEXT frame to it, and the symptom is an + // empty image with no GL error anywhere. + const ScopedDrawFramebuffer restore_binding; + + const GLint w = static_cast(width_); + const GLint h = static_cast(height_); + + BindFramebuffer(GL_READ_FRAMEBUFFER, src_fbo); + BindFramebuffer(GL_DRAW_FRAMEBUFFER, v.blit_fbo); + BlitFramebuffer(0, 0, w, h, 0, 0, w, h, GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT, GL_NEAREST); + + // MuJoCo leaves these on, and none may touch a pass that only moves pixels. + // Not restored: mjr_render's initGL3 sets all four again every frame. + Disable(GL_DEPTH_TEST); + Disable(GL_CULL_FACE); + Disable(GL_BLEND); + Disable(GL_SCISSOR_TEST); + + BindFramebuffer(GL_FRAMEBUFFER, v.out_fbo); + const GLenum targets[2] = { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1 }; + DrawBuffers(2, targets); + Viewport(0, 0, w, h); + UseProgram(program_); + ActiveTexture(GL_TEXTURE0); + BindTexture(GL_TEXTURE_2D, v.blit_color); + ActiveTexture(GL_TEXTURE1); + BindTexture(GL_TEXTURE_2D, v.blit_depth); + BindVertexArray(vao_); + DrawArrays(GL_TRIANGLES, 0, 3); + BindVertexArray(0); + UseProgram(0); + + // Tight rows: VizBuffer's pitch is width * bpp, and the 4-byte default + // alignment would pad an odd-width RGBA8 row. + PixelStorei(GL_PACK_ALIGNMENT, 1); + + ReadBuffer(GL_COLOR_ATTACHMENT0); + BindBuffer(GL_PIXEL_PACK_BUFFER, v.color_pbo); + ReadPixels(0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + + ReadBuffer(GL_COLOR_ATTACHMENT1); + BindBuffer(GL_PIXEL_PACK_BUFFER, v.depth_pbo); + ReadPixels(0, 0, w, h, GL_RED, GL_FLOAT, nullptr); + + BindBuffer(GL_PIXEL_PACK_BUFFER, 0); + check("readback capture"); +} + +void Readback::map() +{ + for (View& v : views_) + { + if (v.mapped) + { + continue; + } + // Orders itself after this thread's GL work, so no glFinish is needed. + cudaGraphicsResource_t resources[2] = { static_cast(v.color_resource), + static_cast(v.depth_resource) }; + check_cuda(cudaGraphicsMapResources(2, resources, nullptr), "cudaGraphicsMapResources"); + v.mapped = true; + + size_t size = 0; + check_cuda(cudaGraphicsResourceGetMappedPointer(&v.color_device_ptr, &size, resources[0]), + "cudaGraphicsResourceGetMappedPointer(color)"); + check_cuda(cudaGraphicsResourceGetMappedPointer(&v.depth_device_ptr, &size, resources[1]), + "cudaGraphicsResourceGetMappedPointer(depth)"); + } +} + +const Readback::View& Readback::at(uint32_t view) const +{ + if (view >= views_.size()) + { + throw std::out_of_range("mujoco_xr: readback view index out of range"); + } + const View& v = views_[view]; + if (!v.mapped) + { + throw std::runtime_error("mujoco_xr: no rendered frame for this view yet -- call render() first"); + } + return v; +} + +void* Readback::color_ptr(uint32_t view) const +{ + return at(view).color_device_ptr; +} + +void* Readback::depth_ptr(uint32_t view) const +{ + return at(view).depth_device_ptr; +} + +void Readback::destroy() +{ + for (View& v : views_) + { + unmap(v, /*throw_on_error=*/false); + if (v.color_resource != nullptr) + { + (void)cudaGraphicsUnregisterResource(static_cast(v.color_resource)); + } + if (v.depth_resource != nullptr) + { + (void)cudaGraphicsUnregisterResource(static_cast(v.depth_resource)); + } + // Only if the entry points were ever resolved. The caller owns the + // ordering: Renderer.close() before mujoco.GLContext.free(). + if (loaded()) + { + const GLuint buffers[2] = { v.color_pbo, v.depth_pbo }; + DeleteBuffers(2, buffers); + const GLuint framebuffers[2] = { v.blit_fbo, v.out_fbo }; + DeleteFramebuffers(2, framebuffers); + const GLuint textures[4] = { v.blit_color, v.blit_depth, v.out_color, v.out_depth }; + DeleteTextures(4, textures); + } + } + views_.clear(); + + if (loaded()) + { + if (vao_ != 0) + { + DeleteVertexArrays(1, &vao_); + vao_ = 0; + } + if (program_ != 0) + { + DeleteProgram(program_); + program_ = 0; + } + } + width_ = 0; + height_ = 0; +} + +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/cpp/gl_readback.hpp b/examples/mujoco_xr/cpp/gl_readback.hpp new file mode 100644 index 000000000..36472e4a4 --- /dev/null +++ b/examples/mujoco_xr/cpp/gl_readback.hpp @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// MuJoCo's offscreen framebuffer -> the CUDA-linear buffers +// viz::ProjectionLayer.submit() consumes, with no host round trip. Stage by +// stage in README.md. +// +// Use cudaGraphicsGLRegisterBUFFER, never RegisterImage: RegisterImage takes no +// depth format and no multisampled renderbuffer, and mjrContext.offDepthStencil +// is both. + +#include "gl.hpp" + +#include +#include + +namespace mujoco_xr +{ + +class Readback +{ +public: + Readback() = default; + ~Readback(); + + Readback(const Readback&) = delete; + Readback& operator=(const Readback&) = delete; + + // `src_fbo` is mjrContext.offFBO. Needed here, not just in capture(): + // glBlitFramebuffer rejects a depth blit between differing formats, so the + // blit target is matched to whichever depth format MuJoCo chose. + void create(uint32_t width, uint32_t height, uint32_t view_count, GLuint src_fbo); + void destroy(); + + // Blit, convert and read back one view. Unmaps that view first, so a + // pointer from color_ptr()/depth_ptr() lives only until the next capture(). + void capture(uint32_t view, GLuint src_fbo); + + // Map every view into CUDA. Once, after the frame's last capture(). + void map(); + + void* color_ptr(uint32_t view) const; + void* depth_ptr(uint32_t view) const; + + uint32_t width() const + { + return width_; + } + uint32_t height() const + { + return height_; + } + +private: + struct View + { + GLuint blit_fbo = 0; + GLuint blit_color = 0; // RGBA8 texture + GLuint blit_depth = 0; // DEPTH24_STENCIL8 texture + GLuint out_fbo = 0; + GLuint out_color = 0; // RGBA8 texture + GLuint out_depth = 0; // R32F texture + GLuint color_pbo = 0; + GLuint depth_pbo = 0; + void* color_resource = nullptr; // cudaGraphicsResource_t + void* depth_resource = nullptr; + void* color_device_ptr = nullptr; + void* depth_device_ptr = nullptr; + bool mapped = false; + }; + + void build_program(); + // throw_on_error false on the teardown path, which is reached from a + // destructor with the GL context possibly already gone. + void unmap(View& v, bool throw_on_error); + const View& at(uint32_t view) const; + + uint32_t width_ = 0; + uint32_t height_ = 0; + GLuint program_ = 0; + GLuint vao_ = 0; + std::vector views_; +}; + +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/cpp/glcamera.hpp b/examples/mujoco_xr/cpp/glcamera.hpp new file mode 100644 index 000000000..0a3be2351 --- /dev/null +++ b/examples/mujoco_xr/cpp/glcamera.hpp @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// One XR view's asymmetric fov -> the mjvGLCamera frustum fields MuJoCo builds +// its projection from. Free functions, no GPU and no MuJoCo state, so +// tests/test_projection.py can pin the convention headless. +// +// `frustum_width` is a HALF-width, and at 0 mjr_render derives the horizontal +// extent from the viewport aspect instead (render_gl3.c setView), which renders +// something plausible from a fov carrying nothing. Always set it; reject a fov +// that would leave it 0. mjvisualize.h calls the field "not used for +// rendering", which is wrong as of 3.11.0. + +#include +#include +#include + +namespace mujoco_xr +{ + +// MuJoCo's own spelling and units: extents on the near plane, half_width +// symmetric about center. +struct Frustum +{ + float center = 0.0f; + float half_width = 0.0f; + float bottom = 0.0f; + float top = 0.0f; + float near_z = 0.0f; + float far_z = 0.0f; +}; + +// fov_lrud is (left, right, up, down) radians -- viz::Fov's and XrFovf's field +// order, with left and down normally negative. No y flip here: OpenGL clip +// space is y-up like the fov, and the flip happens once, on readback. +inline Frustum frustum_from_fov(const std::array& fov_lrud, float near_z, float far_z) +{ + if (!(near_z > 0.0f) || !(far_z > near_z)) + { + throw std::invalid_argument("mujoco_xr: need 0 < near_z < far_z"); + } + const float left = near_z * std::tan(fov_lrud[0]); + const float right = near_z * std::tan(fov_lrud[1]); + const float top = near_z * std::tan(fov_lrud[2]); + const float bottom = near_z * std::tan(fov_lrud[3]); + + Frustum f; + f.center = 0.5f * (right + left); + f.half_width = 0.5f * (right - left); + f.bottom = bottom; + f.top = top; + f.near_z = near_z; + f.far_z = far_z; + + // A default-constructed viz::Fov is all zeros, and a zero half_width is + // exactly what turns the aspect-ratio fallback on. Refuse it. + if (!(f.half_width > 0.0f) || !(f.top > f.bottom)) + { + throw std::invalid_argument( + "mujoco_xr: degenerate fov -- angle_right must exceed angle_left and angle_up must exceed " + "angle_down. An all-zero fov means FrameInfo.views was never filled."); + } + return f; +} + +// A view-space distance -> the depth handed to ProjectionLayer.submit(): +// standard Z, near -> 0, far -> 1. NOT what MuJoCo writes -- mjr_render is +// reverse Z (glClipControl ZERO_TO_ONE, GL_GEQUAL, glClearDepth(0)), so the two +// differ by exactly `1 - d`, the subtraction in gl_readback.cpp's shader. +inline float submitted_depth(float distance, float near_z, float far_z) +{ + return far_z * (distance - near_z) / (distance * (far_z - near_z)); +} + +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/cpp/mujoco_xr_bindings.cpp b/examples/mujoco_xr/cpp/mujoco_xr_bindings.cpp new file mode 100644 index 000000000..3f9ec6626 --- /dev/null +++ b/examples/mujoco_xr/cpp/mujoco_xr_bindings.cpp @@ -0,0 +1,249 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// pybind11 entry point for `mujoco_xr._mujoco_xr`. +// +// Nothing typed crosses this boundary in either direction. viz::Pose3D / Fov +// are registered in `_viz` and not castable here (this module links no viz +// target), so poses and fovs cross as flat float arrays; mjModel / mjData cross +// as integer addresses, Python owning them and C++ owning mjvScene / mjrContext. + +#include "frames.hpp" +#include "glcamera.hpp" +#include "scene_renderer.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace mujoco_xr +{ +namespace +{ + +namespace py = pybind11; + +// A non-owning view of one CUDA-mapped pack buffer, shaped for viz's +// `cuda_array_to_viz_buffer`: kRGBA8 -> "|u1" (H, W, 4), kD32F -> "(cfg, reinterpret_cast(model_address)); + } + + SceneRenderer& get() + { + if (!renderer_) + { + throw std::runtime_error("mujoco_xr: renderer has been closed"); + } + return *renderer_; + } + + void close() + { + renderer_.reset(); + } + +private: + std::unique_ptr renderer_; +}; + +CudaImageView image_view(SceneRenderer& r, int view, bool is_depth) +{ + if (view < 0 || static_cast(view) >= r.view_count()) + { + throw std::out_of_range("mujoco_xr: view index out of range"); + } + const Readback& rb = r.readback(); + const auto index = static_cast(view); + void* ptr = is_depth ? rb.depth_ptr(index) : rb.color_ptr(index); + return CudaImageView{ reinterpret_cast(ptr), rb.width(), rb.height(), is_depth }; +} + +} // namespace +} // namespace mujoco_xr + +PYBIND11_MODULE(_mujoco_xr, m) +{ + namespace py = pybind11; + using namespace pybind11::literals; + + m.doc() = "MuJoCo's OpenGL renderer, read back into CUDA for Isaac Teleop's Televiz ProjectionLayer."; + + m.def( + "mujoco_version", []() { return std::string(mj_versionString()); }, + "The libmujoco this extension is linked against, as reported at runtime. Compare with " + "mujoco.mj_versionString() -- they MUST be equal, and they are only equal because there is exactly one " + "libmujoco loaded in the process."); + + // ── Frames ──────────────────────────────────────────────────────────── + // Exposed rather than reimplemented, so frames.hpp stays the one definition. + + m.def( + "mj_from_xr_pos", [](std::array p_xr) { return mujoco_xr::mj_from_xr_pos(p_xr); }, "p_xr"_a, + "XR reference-space point (metres, Y-up) -> MuJoCo world point (Z-up). Applies both the handedness " + "rotation and the workspace translation."); + + m.def( + "mj_from_xr_quat", [](std::array q_xyzw) { return mujoco_xr::mj_from_xr_quat(q_xyzw); }, "q_xyzw"_a, + "XR orientation as xyzw (the order OpenXR and Teleop's GRIP_ORIENTATION use) -> MuJoCo world " + "orientation as wxyz. The ONLY quaternion crossing in the app."); + + // SCREAMING_CASE attributes, not getters: a snake_case getter would put + // `quat_mj_from_xr` beside `mj_from_xr_quat`, with only word order telling a + // constant from a transform. + m.attr("QUAT_MJ_FROM_XR") = py::tuple(py::cast(mujoco_xr::kQuatMjFromXr)); + m.attr("TRANS_MJ_FROM_XR") = py::tuple(py::cast(mujoco_xr::kTransMjFromXr)); + + // ── Projection ──────────────────────────────────────────────────────── + + m.def( + "frustum_from_fov", + [](std::array fov_lrud, float near_z, float far_z) + { + const mujoco_xr::Frustum f = mujoco_xr::frustum_from_fov(fov_lrud, near_z, far_z); + return std::vector{ f.center, f.half_width, f.bottom, f.top, f.near_z, f.far_z }; + }, + "fov_lrud"_a, "near_z"_a, "far_z"_a, + "The mjvGLCamera frustum fields for one asymmetric fov (angle_left, angle_right, angle_up, angle_down) " + "in radians, as (center, half_width, bottom, top, near, far). Same code path the renderer uses; exposed " + "so the convention is testable without a GPU. Raises ValueError on a degenerate fov or a bad near/far."); + + m.def( + "submitted_depth", + [](float distance, float near_z, float far_z) { return mujoco_xr::submitted_depth(distance, near_z, far_z); }, + "distance"_a, "near_z"_a, "far_z"_a, + "What a view-space distance ahead of the eye becomes in the depth buffer handed to " + "ProjectionLayer.submit(): standard Z, near -> 0, far -> 1. MuJoCo's renderer writes the reverse; " + "shaders/readback inverts it."); + + // ── Renderer ────────────────────────────────────────────────────────── + + py::class_(m, "CudaImageView", + R"doc( +Non-owning CUDA view of one of the renderer's pixel-pack buffers. + +Exposes ``__cuda_array_interface__``, which is all +``isaacteleop.viz.ProjectionLayer.submit()`` needs. Do NOT hold one past the +frame it came from, and never past ``Renderer.close()``: the memory belongs to +the renderer and is unmapped on the next ``render()``. +)doc") + .def_property_readonly("__cuda_array_interface__", &mujoco_xr::CudaImageView::cuda_array_interface); + + py::class_(m, "Renderer", + R"doc( +MuJoCo's OpenGL renderer, read back into CUDA-visible colour + depth buffers. + +An OpenGL context must be current on this thread BEFORE construction, on the +same GPU viz chose (``mujoco.GLContext``; set ``MUJOCO_EGL_DEVICE_ID`` if the +machine has more than one card). The constructor checks this and raises rather +than render into another card's memory. + +Per frame, in this order and on ONE thread:: + + info = session.begin_frame() + if info.should_render: + mujoco.mj_step(model, data) # Python owns the simulation + renderer.update_scene(m_addr, d_addr) + renderer.render(poses, fovs) # poses/fovs from info.views + layer.submit(renderer.color(0), renderer.depth(0), ...) + session.end_frame() +)doc") + .def(py::init(), "width"_a, "height"_a, "view_count"_a, + "near_z"_a, "far_z"_a, "model_address"_a, + "`model_address` is mujoco.MjModel._address. No Vulkan handles: this renderer reaches viz through " + "CUDA alone, and finds viz's GPU as the process's current CUDA device.") + .def( + "update_scene", + [](mujoco_xr::PyRenderer& self, uintptr_t model_address, uintptr_t data_address) + { + return self.get().update_scene( + reinterpret_cast(model_address), reinterpret_cast(data_address)); + }, + "model_address"_a, "data_address"_a, + "One mjv_updateScene for the frame. Call AFTER mj_step, on the same thread. mjData is treated as " + "const. Returns the geom count.") + .def( + "render", + [](mujoco_xr::PyRenderer& self, std::vector poses_xyz_qwxyz, std::vector fovs_lrud) + { + // No gil_scoped_release: this all runs on the GL context bound + // to THIS thread, and releasing the GIL would let another + // thread issue GL on a context it does not hold. + self.get().render(poses_xyz_qwxyz, fovs_lrud); + }, + "poses_xyz_qwxyz"_a, "fovs_lrud"_a, + "Render every view. `poses_xyz_qwxyz` is view_count*7 floats (x, y, z, qw, qx, qy, qz) and " + "`fovs_lrud` is view_count*4 (angle_left, angle_right, angle_up, angle_down) -- flatten them from " + "FrameInfo.views.") + .def( + "frustum", [](mujoco_xr::PyRenderer& self, int view) { return self.get().frustum(view); }, "view"_a, + "The mjvGLCamera frustum used for `view` on the last render(), as (center, half_width, bottom, top, " + "near, far), so the caller can assert the convention per frame.") + .def( + "color", + [](mujoco_xr::PyRenderer& self, int view) + { return mujoco_xr::image_view(self.get(), view, /*is_depth=*/false); }, + // keep_alive<0, 1>: the view is a bare device pointer into the + // Renderer's buffers, so a caller who keeps `buf = renderer.color(0)` + // and drops `renderer` would use-after-free at submit time. + py::keep_alive<0, 1>(), "view"_a, + "RGBA8 colour for `view` as a CudaImageView. Valid until the next render().") + .def( + "depth", + [](mujoco_xr::PyRenderer& self, int view) + { return mujoco_xr::image_view(self.get(), view, /*is_depth=*/true); }, + py::keep_alive<0, 1>(), "view"_a, // see color() above + "float32 depth for `view` as a CudaImageView, standard Z: near -> 0.0, far -> 1.0. Valid until the " + "next render().") + .def_property_readonly("view_count", [](mujoco_xr::PyRenderer& self) { return self.get().view_count(); }) + .def_property_readonly("ngeom", [](mujoco_xr::PyRenderer& self) { return self.get().ngeom(); }) + .def_property_readonly("maxgeom", [](mujoco_xr::PyRenderer& self) { return self.get().maxgeom(); }) + .def("close", &mujoco_xr::PyRenderer::close, + "Release the OpenGL and CUDA resources. Must happen while the GL context is still current, so " + "BEFORE mujoco.GLContext.free()."); +} diff --git a/examples/mujoco_xr/cpp/scene_renderer.cpp b/examples/mujoco_xr/cpp/scene_renderer.cpp new file mode 100644 index 000000000..4dc1f7446 --- /dev/null +++ b/examples/mujoco_xr/cpp/scene_renderer.cpp @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "scene_renderer.hpp" + +#include "frames.hpp" +#include "glcamera.hpp" + +#include +#include +#include +#include +#include + +namespace mujoco_xr +{ + +namespace +{ + +// Not a knob: overflowing it is a hard error, and 20k is ~30x a tabletop scene. +constexpr int kMaxGeom = 20000; + +// OpenXR view space: -Z forward, +Y up. +constexpr std::array kXrForward = { 0.0, 0.0, -1.0 }; +constexpr std::array kXrUp = { 0.0, 1.0, 0.0 }; + +void check_cuda(cudaError_t err, const char* what) +{ + if (err != cudaSuccess) + { + throw std::runtime_error(std::string("mujoco_xr: ") + what + " failed: " + cudaGetErrorString(err)); + } +} + +// viz's VkContext::init() already cudaSetDevice'd the card matching its Vulkan +// device, so "viz's GPU" is just the current CUDA device. The GL context is +// made independently, and nothing makes it land on the same card. +void require_gl_on_viz_device() +{ + int cuda_device = -1; + check_cuda(cudaGetDevice(&cuda_device), "cudaGetDevice"); + + unsigned int count = 0; + int gl_devices[8] = { 0 }; + const cudaError_t err = cudaGLGetDevices(&count, gl_devices, 8, cudaGLDeviceListAll); + if (err != cudaSuccess || count == 0) + { + throw std::runtime_error( + std::string("mujoco_xr: the current OpenGL context is on no CUDA-capable device (") + cudaGetErrorString(err) + + "). Create it with mujoco.GLContext BEFORE the renderer, on this thread, with MUJOCO_GL=egl."); + } + for (unsigned int i = 0; i < count; ++i) + { + if (gl_devices[i] == cuda_device) + { + return; + } + } + throw std::runtime_error( + "mujoco_xr: the OpenGL context is on CUDA device " + std::to_string(gl_devices[0]) + " but viz is on device " + + std::to_string(cuda_device) + + ". Set MUJOCO_EGL_DEVICE_ID to viz's GPU; a cross-device pixel-pack buffer cannot be imported."); +} + +// A direction from XR into MuJoCo world: rotation only, no workspace offset. +std::array mj_from_xr_dir(const std::array& q_mj, const std::array& v_xr) +{ + std::array out{}; + mju_rotVecQuat(out.data(), v_xr.data(), q_mj.data()); + return out; +} + +} // namespace + +SceneRenderer::SceneRenderer(const Config& config, const mjModel* model) : config_(config) +{ + if (model == nullptr) + { + throw std::invalid_argument("mujoco_xr: null mjModel*"); + } + if (config_.width == 0 || config_.height == 0 || config_.view_count == 0) + { + throw std::invalid_argument("mujoco_xr: renderer needs a non-empty size and at least one view"); + } + // Validates near/far up front, on a fov that is certainly non-degenerate. + (void)frustum_from_fov({ -0.5f, 0.5f, 0.5f, -0.5f }, config_.near_z, config_.far_z); + + gl::load(); + require_gl_on_viz_device(); + + mjv_defaultOption(&scene_option_); + mjv_defaultFreeCamera(model, &camera_); + mjv_makeScene(model, &scene_, kMaxGeom); + scene_made_ = true; + // Each eye is drawn on its own, into the whole offscreen buffer. + scene_.stereo = mjSTEREO_NONE; + + mjr_defaultContext(&context_); + mjr_makeContext(model, &context_, mjFONTSCALE_100); + context_made_ = true; + // Overrides whatever the scene declared. + mjr_resizeOffscreen(static_cast(config_.width), static_cast(config_.height), &context_); + if (context_.offWidth != static_cast(config_.width) || context_.offHeight != static_cast(config_.height)) + { + throw std::runtime_error("mujoco_xr: mjr_resizeOffscreen did not take; the GL context is too small or lost"); + } + if (context_.offSamples != 0) + { + throw std::runtime_error( + "mujoco_xr: model.vis.quality.offsamples must be 0. Multisample renderbuffers cannot be blitted with a " + "y flip in one step, and MuJoCo resolves them only inside mjr_readPixels, which this path does not call."); + } + mjr_setBuffer(mjFB_OFFSCREEN, &context_); + if (context_.currentBuffer != mjFB_OFFSCREEN) + { + throw std::runtime_error("mujoco_xr: the offscreen framebuffer is unavailable in this OpenGL context"); + } + + readback_.create(config_.width, config_.height, config_.view_count, context_.offFBO); + cameras_.resize(config_.view_count); +} + +SceneRenderer::~SceneRenderer() +{ + destroy(); +} + +void SceneRenderer::destroy() +{ + readback_.destroy(); + if (context_made_) + { + mjr_freeContext(&context_); + context_made_ = false; + } + if (scene_made_) + { + mjv_freeScene(&scene_); + scene_made_ = false; + } +} + +int SceneRenderer::update_scene(const mjModel* model, mjData* data) +{ + if (model == nullptr || data == nullptr) + { + throw std::invalid_argument("mujoco_xr: update_scene got a null mjModel* / mjData*"); + } + mjv_updateScene(model, data, &scene_option_, nullptr, &camera_, mjCAT_ALL, &scene_); + return scene_.ngeom; +} + +std::vector SceneRenderer::frustum(int view) const +{ + if (view < 0 || static_cast(view) >= config_.view_count) + { + throw std::out_of_range("mujoco_xr: view index out of range"); + } + const mjvGLCamera& c = cameras_[static_cast(view)]; + return { c.frustum_center, c.frustum_width, c.frustum_bottom, c.frustum_top, c.frustum_near, c.frustum_far }; +} + +void SceneRenderer::render(const std::vector& poses_xyz_qwxyz, const std::vector& fovs_lrud) +{ + const size_t n = config_.view_count; + if (poses_xyz_qwxyz.size() != n * 7 || fovs_lrud.size() != n * 4) + { + throw std::invalid_argument( + "mujoco_xr: render() expects view_count*7 pose floats and view_count*4 fov floats; the renderer's " + "view_count must match len(FrameInfo.views)"); + } + + const mjrRect viewport{ 0, 0, static_cast(config_.width), static_cast(config_.height) }; + + for (size_t v = 0; v < n; ++v) + { + const float* pose = poses_xyz_qwxyz.data() + v * 7; + const float* fov = fovs_lrud.data() + v * 4; + + // The eye crosses into MuJoCo world, not the geometry the other way: + // mjr_render draws MuJoCo world and takes a camera, not a view matrix. + const std::array q_xyzw = { pose[4], pose[5], pose[6], pose[3] }; + const std::array q_mj = mj_from_xr_quat(q_xyzw); + const std::array p_mj = mj_from_xr_pos({ pose[0], pose[1], pose[2] }); + const std::array forward = mj_from_xr_dir(q_mj, kXrForward); + const std::array up = mj_from_xr_dir(q_mj, kXrUp); + + const Frustum f = frustum_from_fov({ fov[0], fov[1], fov[2], fov[3] }, config_.near_z, config_.far_z); + + mjvGLCamera& cam = cameras_[v]; + cam = mjvGLCamera{}; + for (int i = 0; i < 3; ++i) + { + cam.pos[i] = static_cast(p_mj[i]); + cam.forward[i] = static_cast(forward[i]); + cam.up[i] = static_cast(up[i]); + } + cam.frustum_center = f.center; + cam.frustum_width = f.half_width; + cam.frustum_bottom = f.bottom; + cam.frustum_top = f.top; + cam.frustum_near = f.near_z; + cam.frustum_far = f.far_z; + cam.orthographic = 0; + + // mjv_updateScene wrote both cameras from mjvCamera; overwrite them + // after it, and both, because mjSTEREO_NONE renders their average. + // Lights are NOT overwritten with them: mjv_updateScene already baked + // the headlight from camera_, so it stays a world-fixed directional + // light rather than following the eye. + scene_.camera[0] = cam; + scene_.camera[1] = cam; + + mjr_render(viewport, &scene_, &context_); + readback_.capture(static_cast(v), context_.offFBO); + } + + readback_.map(); +} + +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/cpp/scene_renderer.hpp b/examples/mujoco_xr/cpp/scene_renderer.hpp new file mode 100644 index 000000000..6f2cf5945 --- /dev/null +++ b/examples/mujoco_xr/cpp/scene_renderer.hpp @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// MuJoCo's own renderer, driven once per eye into an offscreen framebuffer that +// Readback turns into CUDA pointers. Shading, materials, lights and shadows all +// stay MuJoCo's -- none of it lives here. +// +// The OpenGL context is NOT created here: `mujoco.GLContext` makes it, and it +// must be current on this thread and on viz's GPU before the constructor runs. +// The constructor checks the GPU rather than render into another card's memory. +// +// C++ owns mjvScene/mjvOption/mjvCamera; Python owns mjModel/mjData/mj_step. +// render() runs on the mj_step thread, after it, and treats mjData as const. + +#include "gl_readback.hpp" + +#include + +#include +#include + +namespace mujoco_xr +{ + +class SceneRenderer +{ +public: + struct Config + { + uint32_t width = 0; + uint32_t height = 0; + // A field because the render loop reads it, not because mono works. + uint32_t view_count = 2; + // No default, and no near/far literal anywhere in cpp/: this pair must + // equal VizSessionConfig.xr_near_z / xr_far_z, or the runtime reprojects + // the submitted depth against the wrong range. + float near_z = 0.0f; + float far_z = 0.0f; + }; + + SceneRenderer(const Config& config, const mjModel* model); + ~SceneRenderer(); + + SceneRenderer(const SceneRenderer&) = delete; + SceneRenderer& operator=(const SceneRenderer&) = delete; + + // mjv_updateScene, exactly once per frame. Returns the geom count. + int update_scene(const mjModel* model, mjData* data); + + // Draws every view and leaves the CUDA pointers mapped, ready for + // ProjectionLayer.submit() the moment this returns. `poses_xyz_qwxyz` is + // view_count*7 (position then w,x,y,z, viz.Pose3D's spelling) and + // `fovs_lrud` view_count*4 radians in viz.Fov's field order. + void render(const std::vector& poses_xyz_qwxyz, const std::vector& fovs_lrud); + + // Last render()'s frustum for `view`, as (center, half_width, bottom, top, + // near, far), so the app can assert the convention per frame. + std::vector frustum(int view) const; + + const Readback& readback() const + { + return readback_; + } + uint32_t view_count() const + { + return config_.view_count; + } + int ngeom() const + { + return scene_.ngeom; + } + int maxgeom() const + { + return scene_.maxgeom; + } + +private: + void destroy(); + + Config config_; + Readback readback_; + + mjvScene scene_{}; + mjvOption scene_option_{}; + mjvCamera camera_{}; + mjrContext context_{}; + bool scene_made_ = false; + bool context_made_ = false; + + std::vector cameras_; +}; + +} // namespace mujoco_xr diff --git a/examples/mujoco_xr/pyproject.toml b/examples/mujoco_xr/pyproject.toml new file mode 100644 index 000000000..57526f94d --- /dev/null +++ b/examples/mujoco_xr/pyproject.toml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# This example is its own wheel, and the wheel is the only way to run it: +# +# uv pip install ./examples/mujoco_xr +# python -m isaacteleop_examples.mujoco_xr # needs a headset + CloudXR +# +# Its own wheel because _mujoco_xr links libmujoco: folded into `isaacteleop`, +# that wheel's contents would depend on whether the build host had mujoco. +# `install_python_example()` cannot ship a compiled, ABI-tagged extension. + +[build-system] +requires = [ + "scikit-build-core>=1.0", + # Not pinned to the root build's pybind11: no pybind11-registered type + # crosses this boundary, so the two share no ABI. + "pybind11>=2.12", + # Not optional: the extension compiles against this wheel's headers, and + # without it an isolated build emits a wheel with no extension. Always equal + # to `dependencies` below -- and never restate the number in prose here, + # since CMakeLists.txt matches every `mujoco==` in this file. + "mujoco==3.11.0", +] +build-backend = "scikit_build_core.build" + +[project] +# The dist name mirrors the import path rather than the directory name, so an +# installed example does not claim a bare top-level `mujoco_xr` in +# site-packages, right next to the real `mujoco`. +name = "isaacteleop-examples-mujoco-xr" +version = "0.0.0" # Internal example - not versioned +description = "MuJoCo scene rendered into an Isaac Teleop Televiz XR session" + +# A range, not a pin: scikit-build-core tags the wheel with the installing +# interpreter's ABI. Bounds match ISAAC_TELEOP_PYTHON_VERSION_MIN / +# _MAX_EXCLUSIVE in the root CMakeLists.txt. +requires-python = ">=3.11,<3.14" + +dependencies = [ + # Run-time mujoco, equal to the build-time pin above. Exactly one libmujoco + # may be loaded: mjModel* / mjData* addresses cross the pybind boundary. + "mujoco==3.11.0", + # Unversioned, and a live hazard: a published isaacteleop exists on PyPI, so + # this resolves happily against a release that is not this checkout's viz. + # Install the locally built wheel first, into the same environment: + # uv pip install "isaacteleop[cloudxr]" --find-links=./install/wheels/ + "isaacteleop", + # app.py imports numpy directly. mujoco would pull it in anyway, but a + # transitive dependency imported by name breaks on an upstream change. + "numpy", +] + +[tool.scikit-build] +# Floor matches CMakeLists.txt's cmake_minimum_required; the <4 cap mirrors the +# root pyproject.toml. +cmake.version = ">=3.20,<4" +cmake.build-type = "Release" + +# Persistent, so `uv pip install --reinstall-package ...` stays incremental +# instead of reconfiguring from scratch. `{cache_tag}` keeps per-interpreter +# caches apart. `**/build/` is gitignored. +build-dir = "build/wheel-{cache_tag}" + +# `wheel.packages` owns every authored file under the package; CMake's +# install(TARGETS) owns the one build-produced file. Their intersection must be +# empty, and without this line it is not, because the in-tree build drops its +# own .so there for ctest. Measured: omit it and a cp313 wheel ships the root +# build's stale cp312 .so as well. +# +# `sdist.exclude`, not `wheel.exclude`: the latter is applied twice, so `*.so` +# there would delete the freshly compiled extension too. +[tool.scikit-build.sdist] +exclude = ["python/isaacteleop_examples/mujoco_xr/*.so"] + +# Key is the path inside the wheel, value the source directory. +# +# `isaacteleop_examples` is deliberately not listed: it is a PEP 420 namespace +# with no __init__.py and no owner, and scikit-build-core creates the +# intermediate directory from this key. Adding an __init__.py there (or listing +# the directory as a package) makes it a regular package owned by this wheel, +# and a second example distribution then collides or is shadowed. +[tool.scikit-build.wheel] +packages = { "isaacteleop_examples/mujoco_xr" = "python/isaacteleop_examples/mujoco_xr" } + +# The absent [tool.scikit-build.editable] block is deliberate. `pip install -e` +# is not supported: an editable install redirects the package back to the source +# tree, which is where the in-tree CMake build drops its own _mujoco_xr*.so, so +# you would silently import that one instead. `mode = "redirect"` is already the +# default, so adding the block would not help. Use +# `uv pip install --reinstall-package isaacteleop-examples-mujoco-xr .` instead. diff --git a/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/__init__.py b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/__init__.py new file mode 100644 index 000000000..8ce4f1cde --- /dev/null +++ b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/__init__.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""MuJoCo scene rendered into an Isaac Teleop Televiz XR session.""" + +import os as _os + +# Must precede `import mujoco`, which reads MUJOCO_GL at import time. EGL rather +# than the GLFW default because this renders offscreen, usually with no display, +# and only the EGL path honours MUJOCO_EGL_DEVICE_ID. setdefault, so an explicit +# MUJOCO_GL still wins. +_os.environ.setdefault("MUJOCO_GL", "egl") + +# Load order is load-bearing -- do not let an import sorter move this. `import +# mujoco` pulls the wheel's libmujoco in first, and `_mujoco_xr` has a NEEDED +# entry for that same SONAME with no RPATH, so it binds to the already-loaded +# copy. That is what guarantees one libmujoco, and so one mjModel* layout. +import mujoco as _mujoco + +from . import _mujoco_xr + +if _mujoco.mj_versionString() != _mujoco_xr.mujoco_version(): + raise ImportError( + "mujoco_xr: two different libmujoco libraries are loaded -- " + f"the `mujoco` wheel reports {_mujoco.mj_versionString()} but the compiled " + f"extension reports {_mujoco_xr.mujoco_version()}. The extension is what has to be " + "rebuilt. Both `mujoco==` pins in examples/mujoco_xr/pyproject.toml (build-system.requires " + "and project.dependencies) must name one version, and reinstalling recompiles against it: " + "uv pip install --reinstall ./examples/mujoco_xr. (If you hit this from the in-tree ctest " + "path instead, the extension came from the root build: install that same version into " + "build//teleop_build_venv/bin/python and re-run cmake --preset.) " + "mjModel* / mjData* pointers cannot cross this boundary otherwise." + ) + +__all__ = ["_mujoco_xr"] diff --git a/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/__main__.py b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/__main__.py new file mode 100644 index 000000000..0b4cb8ca9 --- /dev/null +++ b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/__main__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Entry point: ``python -m isaacteleop_examples.mujoco_xr``.""" + +import sys + +from .app import main + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/app.py b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/app.py new file mode 100644 index 000000000..284d30551 --- /dev/null +++ b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/app.py @@ -0,0 +1,588 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A MuJoCo scene drawn into a Televiz XR session. + +One OpenXR session shared between VizSession (rendering) and TeleopSession +(input); the scene is drawn by MuJoCo's own renderer and reaches +ProjectionLayer.submit() by CUDA pointer, never through host memory. + + VizSession(kXr) ──get_oxr_handles()──▶ TeleopSession + │ │ + │ recommended resolution │ controller grip poses + ▼ ▼ │ + _mujoco_xr.Renderer ──__cuda_array_interface__──▶ ProjectionLayer │ + ▲ │ + └──────────────── mjData.mocap_pos/_quat ◀─────────────────────┘ + +The renderer needs an OpenGL context current on this thread, made below and torn +down after it; viz and the renderer meet through CUDA alone, on VizSession's GPU. + +C++ owns mjvScene/mjvOption/mjvCamera/mjrContext; Python owns +mjModel/mjData/mj_step, so everything reading a controller and writing mjData is +testable without a GPU. Frame order is load-bearing: input is sampled before the +physics it feeds, on every frame that will step or draw. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import logging +import math +import sys +from pathlib import Path +from typing import NamedTuple + +import mujoco +import numpy as np + +from isaacteleop import viz +from isaacteleop.cloudxr import CloudXRLauncher +from isaacteleop.oxr import OpenXRSessionHandles +from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource +from isaacteleop.retargeting_engine.interface import OutputCombiner +from isaacteleop.retargeting_engine.tensor_types import ControllerInputIndex +from isaacteleop.retargeters.SO101.gripper_retargeter import ( + GRIPPER_COMMAND_KEY, + SO101GripperRetargeter, +) +from isaacteleop.teleop_session_manager import ( + TeleopSession, + TeleopSessionConfig, + get_required_oxr_extensions_from_pipeline, +) + +from . import _mujoco_xr + +LOG = logging.getLogger("mujoco_xr") + +# The app's only clip planes. VizSessionConfig, the projection and the submitted +# depth must all agree, or world-locked geometry swims under head motion -- and +# only a headset shows it. There is no near/far literal in cpp/, by construction. +NEAR_Z = 0.05 +FAR_Z = 50.0 + +# Wall-clock ceiling for one simulation advance. See _clamp_dt. +MAX_DT_S = 0.1 + +# The only mode: this needs a headset and a CloudXR runtime. A headless fallback +# should arrive with the CI job that runs it (NVIDIA/IsaacTeleop#880). +_DISPLAY_MODE = viz.DisplayMode.kXr + +# layer.submit() in _loop is spelled out per eye and cannot read this name, so +# changing it means editing that call too. +_VIEW_COUNT = 2 + +_CLOCK_SOURCE = ( + "FrameInfo.predicted_display_time; frames with no prediction are skipped, " + "not sampled as 0" +) + +# Package data, so it resolves the same from the wheel and the source tree. Keep +# it ABSOLUTE: on mujoco 3.11.0 a relative model path mis-composes the mesh paths +# of an d fragment and fails naming a file that is right there on disk. +DEFAULT_SCENE = Path(__file__).parent / "assets" / "scene.xml" + +# Checked by name before MuJoCo sees the scene: its failure for a missing +# target is a bare "Error opening file .stl", naming a file +# nobody asked for. +FETCH_SCRIPT = "examples/mujoco_xr/scripts/fetch-so-arm.sh" +_LEADER_ASSETS = Path(__file__).parent / "assets" / "leader" +_LEADER_MESHES = ( + "Wrist_Roll_SO101.stl", + "Trigger_SO101.stl", + "Handle_SO101.stl", + "STS3215_03a.stl", +) + + +def _missing_leader_assets() -> list[str]: + """Names of the fetched meshes that are not on disk. Empty when fetched.""" + return [n for n in _LEADER_MESHES if not (_LEADER_ASSETS / n).is_file()] + + +# One hand and no flag: the ghost is a right-handed gripper. +GHOST_HAND = ControllersSource.RIGHT + +# The two mocap bodies leader_gripper.xml declares. +GHOST_BODY = "leader_ghost" +GHOST_JAW_BODY = "leader_ghost_jaw" + +# ── Where the ghost sits on the hand ─────────────────────────────────────── +# Measured on a headset, not derived: this is a claim about a hand holding a +# CONTROLLER, so do not re-derive it from the mesh. Euler degrees, intrinsic +# XYZ, i.e. MuJoCo's `euler=`. Re-tuning procedure and the mesh trap: +# README.md#where-the-ghost-sits-on-the-hand-apppy. +_EULER_GRIP_FROM_GHOST_DEG = (60, 180, 270) +_POS_GRIP_FROM_GHOST = np.array((0, 0.02, -0.025)) + +# ── The trigger hinge ────────────────────────────────────────────────────── +# The follower's `gripper` revolute joint, from SO-ARM100's +# so101_new_calib.urdf: origin xyz="0.0202 0.0188 -0.0234" rpy="1.5708 0 0", +# axis "0 0 1" -- the leader's trigger sits in the moving-jaw slot and shares +# the hinge. The axis below is that "0 0 1" carried through the joint frame's +# 90-degree roll. Do not re-derive either from the meshes: both look right at +# the joint's zero and are wrong by the far end of its travel. +_TRIGGER_HINGE_POS = np.array((0.0202, 0.0188, -0.0234)) # metres, ghost frame +_TRIGGER_HINGE_AXIS = np.array((0.0, -1.0, 0.0)) # unit, ghost frame + +# The travel is the URDF joint's own: `upper="1.74533"` is 100.0 degrees, and +# squeezed is its authored zero. Do not extend to the joint's lower limit +# (-10 deg): that end swings the lever 0.4 mm into the servo. +_TRIGGER_RELEASED_RAD = math.radians(100.0) # closedness 0, jaw wide open +_TRIGGER_SQUEEZED_RAD = 0.0 # closedness 1, tucked to the authored pose + + +def _quat_from_euler_deg(angles_deg) -> np.ndarray: + """Intrinsic X-then-Y-then-Z degrees -> a wxyz quaternion, MuJoCo's `euler=`. + + Right-multiplication is what makes it intrinsic. Spelled out rather than + calling mju_euler2Quat so the sequence is visible where it is used. + """ + quat = np.array((1.0, 0.0, 0.0, 0.0)) + for axis, angle in zip(np.eye(3), angles_deg): + step = np.empty(4) + mujoco.mju_axisAngle2Quat(step, axis, math.radians(angle)) + composed = np.empty(4) + mujoco.mju_mulQuat(composed, quat, step) + quat = composed + return quat + + +# ── Derived below; nothing from here on is authored ──────────────────────── +_QUAT_GRIP_FROM_GHOST = _quat_from_euler_deg(_EULER_GRIP_FROM_GHOST_DEG) + + +def _clamp_dt(dt: float) -> float: + """NaN-safe clamp into [0, MAX_DT_S]. + + Comparisons, not min/max: max(nan, 0) is nan, so the obvious form passes + NaN through both limits and into mj_step. + """ + if dt > 0: + return MAX_DT_S if dt > MAX_DT_S else dt + return 0.0 + + +def _build_pipeline() -> OutputCombiner: + """Controllers, plus the shipped SO-101 jaw retargeter as a graph edge. + + A BaseRetargeter node in the pipeline, not a library call beside it. With no + robot in the scene the jaw it drives is the operator's own trigger. + """ + controllers = ControllersSource(name="controllers") + jaw = SO101GripperRetargeter(name="ghost_jaw", input_device=GHOST_HAND).connect( + {GHOST_HAND: controllers.output(GHOST_HAND)} + ) + return OutputCombiner( + { + ControllersSource.LEFT: controllers.output(ControllersSource.LEFT), + ControllersSource.RIGHT: controllers.output(ControllersSource.RIGHT), + GRIPPER_COMMAND_KEY: jaw.output(GRIPPER_COMMAND_KEY), + } + ) + + +def _flatten_xr_views(info) -> tuple[list[float], list[float]]: + """FrameInfo.views -> the flat float arrays the renderer takes. + + Field by field, never sliced: viz.Pose3D.orientation is (w,x,y,z) while a + controller's GRIP_ORIENTATION is (x,y,z,w). + """ + poses: list[float] = [] + fovs: list[float] = [] + for view in info.views: + px, py, pz = view.pose.position + qw, qx, qy, qz = view.pose.orientation + poses.extend((px, py, pz, qw, qx, qy, qz)) + fovs.extend( + ( + view.fov.angle_left, + view.fov.angle_right, + view.fov.angle_up, + view.fov.angle_down, + ) + ) + return poses, fovs + + +def _assert_frustum(f: list[float], fov, near: float, far: float) -> None: + """The frustum handed to mjvGLCamera, checked against the fov it came from. + + `f` is (center, half_width, bottom, top, near, far). The projection's shape + is MuJoCo's business; which numbers reach it is this app's. + """ + center, half_width, bottom, top, f_near, f_far = f + + # At zero half_width mjr_render derives the horizontal extent from the + # viewport aspect, rendering something plausible from a fov carrying nothing. + assert half_width > 0.0 and top > bottom, ( + f"degenerate frustum {f}: a zeroed Fov reached the camera" + ) + # float32 tolerances throughout: the frustum crosses as C floats, so an + # exact comparison against a Python float fails on rounding alone. + for name, got, want in ( + ("left", center - half_width, near * math.tan(fov.angle_left)), + ("right", center + half_width, near * math.tan(fov.angle_right)), + ("bottom", bottom, near * math.tan(fov.angle_down)), + ("top", top, near * math.tan(fov.angle_up)), + ): + assert abs(got - want) <= 1e-6 * max(1.0, abs(want)), ( + f"frustum {name}={got}, expected {want}" + ) + + # viz's XrCompositionLayerDepthInfoKHR pair must be the encoding pair, or + # the runtime reprojects against the wrong range. + assert abs(f_near - near) <= 1e-6 * near and abs(f_far - far) <= 1e-6 * far, ( + f"clip planes drifted: camera has ({f_near}, {f_far}), viz was told ({near}, {far})" + ) + + +def _log_startup(resolution, gl_backend: str) -> None: + """One block naming every assumption that is invisible at runtime.""" + try: + version = importlib.metadata.version("isaacteleop") + except importlib.metadata.PackageNotFoundError: + version = "" + trans = _mujoco_xr.TRANS_MJ_FROM_XR + + LOG.info("scene: %s", DEFAULT_SCENE) + # Several examples ship their own .venv, and picking up the wrong + # isaacteleop is invisible without this line. + LOG.info( + "isaacteleop: %s (version %s)", Path(viz.__file__).resolve().parent, version + ) + LOG.info( + "mujoco: %s (extension links %s)", + mujoco.mj_versionString(), + _mujoco_xr.mujoco_version(), + ) + LOG.info( + "views: %d (stereo) view resolution: %sx%s", + _VIEW_COUNT, + resolution.width, + resolution.height, + ) + LOG.info( + "renderer: MuJoCo's own (mjr_render), OpenGL backend %s, offsamples=0; " + "blitted, y-flipped, depth-inverted, read back through a PBO CUDA imports", + gl_backend, + ) + LOG.info( + "clip: near=%.4f far=%.2f (one pair -> VizSessionConfig, projection, submitted depth)", + NEAR_Z, + FAR_Z, + ) + LOG.info( + "frames: mj_from_xr translation = (%.3f, %.3f, %.3f) m -- x is operator standoff, " + "z is a FLOOR datum this session's reference space does not establish (cpp/frames.hpp)", + trans[0], + trans[1], + trans[2], + ) + LOG.info("clock: %s", _CLOCK_SOURCE) + LOG.info( + "depth: D32F requested. Whether the runtime ACCEPTED it is not queryable, so " + "the absence of errors is not confirmation." + ) + + +class _GhostChannels(NamedTuple): + """The ghost's two mocap rows, resolved once at startup. + + Mocap indices, not body ids: mocap_pos/mocap_quat index by body_mocapid, + and a body id there writes into another body's row. + """ + + body: int + jaw: int + + +def _resolve_ghost(model) -> _GhostChannels: + """Both ghost mocap rows. The shipped scene always declares them.""" + body = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, GHOST_BODY) + jaw = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, GHOST_JAW_BODY) + if body < 0 or jaw < 0: + raise RuntimeError( + f"mujoco_xr: {DEFAULT_SCENE} declares no `{GHOST_BODY}` / " + f"`{GHOST_JAW_BODY}` pair; it must assets/leader/leader_gripper.xml." + ) + return _GhostChannels(int(model.body_mocapid[body]), int(model.body_mocapid[jaw])) + + +def _update_ghost(data, ghost: _GhostChannels, result) -> None: + """Lock the leader gripper to the GHOST_HAND grip pose; swing its trigger. + + Keep the validity gate: an untracked controller reports (0, 0, 0), which is + the scene origin and a place a legitimate pose could put it, so freezing is + the honest rendering of "tracking lost" and there is no else branch. + _QUAT_GRIP_FROM_GHOST right-multiplies because it is fixed in the gripper's + frame; left-multiplying swings the ghost around the room as the operator turns. + """ + controller = result[GHOST_HAND] + if controller.is_none: + return + if not bool(controller[ControllerInputIndex.GRIP_IS_VALID]): + return + position = controller[ControllerInputIndex.GRIP_POSITION] + orientation = controller[ControllerInputIndex.GRIP_ORIENTATION] + p_xr = [float(position[0]), float(position[1]), float(position[2])] + q_xyzw = [ + float(orientation[0]), + float(orientation[1]), + float(orientation[2]), + float(orientation[3]), + ] + + q_grip = np.array(_mujoco_xr.mj_from_xr_quat(q_xyzw), dtype=float) + p_grip = np.array(_mujoco_xr.mj_from_xr_pos(p_xr), dtype=float) + + q_body = np.empty(4) + mujoco.mju_mulQuat(q_body, q_grip, _QUAT_GRIP_FROM_GHOST) + p_offset = np.empty(3) + mujoco.mju_rotVecQuat(p_offset, _POS_GRIP_FROM_GHOST, q_grip) + p_body = p_grip + p_offset + + data.mocap_pos[ghost.body] = p_body + data.mocap_quat[ghost.body] = q_body + + # The deadzone and clamp are the retargeter's contract, not this app's. + # Rotated ABOUT the hinge, not placed at it: the jaw's XML rest pose equals + # the ghost's, so the pivot lives in exactly one place. + closedness = float(result[GRIPPER_COMMAND_KEY][0]) + angle = _TRIGGER_RELEASED_RAD + closedness * ( + _TRIGGER_SQUEEZED_RAD - _TRIGGER_RELEASED_RAD + ) + q_hinge = np.empty(4) + mujoco.mju_axisAngle2Quat(q_hinge, _TRIGGER_HINGE_AXIS, angle) + q_jaw = np.empty(4) + mujoco.mju_mulQuat(q_jaw, q_body, q_hinge) + + # Rotating the ghost frame about the hinge maps 0 to (pivot - R_hinge.pivot). + swung = np.empty(3) + mujoco.mju_rotVecQuat(swung, _TRIGGER_HINGE_POS, q_hinge) + offset = np.empty(3) + mujoco.mju_rotVecQuat(offset, _TRIGGER_HINGE_POS - swung, q_body) + + data.mocap_pos[ghost.jaw] = p_body + offset + data.mocap_quat[ghost.jaw] = q_jaw + + +def _frame_clock(info) -> float | None: + """The simulation clock, or None if this frame carries no time. + + viz zeroes predicted_display_time with should_render on every frame before + kRunning; sampling it makes the next real frame compute dt from 0 and step + 50 times in one display frame. The caller must skip it entirely. + """ + if info.predicted_display_time == 0: + return None + return info.predicted_display_time / 1e9 + + +def run() -> int: + model = mujoco.MjModel.from_xml_path(str(DEFAULT_SCENE)) + data = mujoco.MjData(model) + + # Order is load-bearing: VizSession calls xrCreateInstance, so an extension + # discovered after it cannot be added -- and a controller tracker missing + # XR_NVX1_action_context is silently dead rather than an error. + pipeline = _build_pipeline() + required_extensions = get_required_oxr_extensions_from_pipeline(pipeline) + + config = viz.VizSessionConfig() + config.mode = _DISPLAY_MODE + config.app_name = "MuJoCoXR" + config.xr_near_z = NEAR_Z + config.xr_far_z = FAR_Z + config.required_extensions = required_extensions + # Alpha 0 = "show passthrough here", honoured at the runtime's discretion: + # viz sets the source-alpha blend bit only for a non-opaque environment, so + # a VR headset composites black instead, which is legible rather than broken. + config.clear_color = (0.0, 0.0, 0.0, 0.0) + + viz_session = viz.VizSession.create(config) + renderer = None + gl_context = None + try: + resolution = viz_session.get_recommended_resolution() + + layer_config = viz.ProjectionLayerConfig() + layer_config.name = "mujoco_scene" + layer_config.view_resolution = resolution + layer_config.color_format = viz.PixelFormat.kRGBA8 + layer_config.depth_format = viz.PixelFormat.kD32F + layer_config.stereo = _VIEW_COUNT == 2 + layer = viz_session.add_projection_layer(layer_config) + + # After VizSession.create, which cudaSetDevice's the GPU behind its + # Vulkan device; the renderer checks this context landed on that one. + gl_context = mujoco.GLContext(resolution.width, resolution.height) + gl_context.make_current() + + # MuJoCo resolves multisample renderbuffers only inside mjr_readPixels, + # which this path never calls, and a multisample source cannot be + # blitted with a y flip in one step. + model.vis.quality.offsamples = 0 + + renderer = _mujoco_xr.Renderer( + width=resolution.width, + height=resolution.height, + view_count=_VIEW_COUNT, + near_z=NEAR_Z, + far_z=FAR_Z, + model_address=model._address, + ) + + _log_startup(resolution, type(gl_context).__module__) + + # After the startup block, so its line reads as part of the same report. + ghost = _resolve_ghost(model) + LOG.info( + "leader ghost: bound to mocap %d (body) / %d (trigger); trigger driven by " + "SO101GripperRetargeter, %.0f deg released to %.0f deg squeezed", + ghost.body, + ghost.jaw, + math.degrees(_TRIGGER_RELEASED_RAD), + math.degrees(_TRIGGER_SQUEEZED_RAD), + ) + + oxr = viz_session.get_oxr_handles() + if oxr is None: + raise RuntimeError( + "VizSession is in kXr mode but produced no OpenXR handles; the backend did not initialize." + ) + teleop_config = TeleopSessionConfig( + app_name="MuJoCoXR", + pipeline=pipeline, + # Never pass trackers=: TeleopSession discovers them from the graph, + # and passing them again duplicates the set. + oxr_handles=OpenXRSessionHandles(*oxr), + ) + with TeleopSession(teleop_config) as teleop_session: + _loop(viz_session, layer, renderer, model, data, teleop_session, ghost) + finally: + # Innermost first: the renderer's GL objects need a current context. + if renderer is not None: + renderer.close() + if gl_context is not None: + gl_context.free() + viz_session.destroy() + return 0 + + +def _loop(viz_session, layer, renderer, model, data, teleop_session, ghost) -> None: + view_count = renderer.view_count + previous_clock: float | None = None + # NOT reset or drained on a non-render frame: the simulation owes that time + # whether or not anything was displayed. + accumulator = 0.0 + checked_frustum = False + + while not viz_session.should_close(): + info = viz_session.begin_frame() + try: + # None means no usable timestamp -- skip the sample, never record 0. + now = _frame_clock(info) + if now is not None: + if previous_clock is not None: + accumulator += _clamp_dt(now - previous_clock) + previous_clock = now + + # Above both the should_render gate and the step loop, so it + # precedes the physics it feeds. Gated rather than every frame: an + # ungated step() calls xrSyncActions on the unthrottled + # pre-kRunning burst, hundreds of frames in milliseconds. + result = None + will_step = accumulator >= model.opt.timestep + if will_step or info.should_render: + result = teleop_session.step() + _update_ghost(data, ghost, result) + + steps = 0 + while accumulator >= model.opt.timestep and steps < 64: + mujoco.mj_step(model, data) + accumulator -= model.opt.timestep + steps += 1 + + if not info.should_render: + # Deliberately does NOT touch the accumulator. + continue + + renderer.update_scene(model._address, data._address) + # mjv_updateScene truncates on overflow and returns normally, with + # only a stderr warning nobody reads in a frame loop. + if renderer.ngeom >= renderer.maxgeom: + raise RuntimeError( + f"mjvScene is full: ngeom={renderer.ngeom} maxgeom={renderer.maxgeom}. " + "Geometry is being dropped -- raise kMaxGeom in " + "cpp/scene_renderer.cpp." + ) + + # No view-count check here: render() sees the flattened lengths and + # rejects a mismatch in those terms. + poses, fovs = _flatten_xr_views(info) + renderer.render(poses, fovs) + + # First rendered frame only: the fov changes per frame, the + # convention does not. + if not checked_frustum: + for view in range(view_count): + _assert_frustum( + renderer.frustum(view), info.views[view].fov, NEAR_Z, FAR_Z + ) + LOG.info( + "frustum verified on the first rendered frame (matches FrameInfo fov, clip planes agree " + "with VizSessionConfig)" + ) + checked_frustum = True + + layer.submit( + renderer.color(0), + renderer.depth(0), + renderer.color(1), + renderer.depth(1), + ) + finally: + # Follows EVERY begin_frame(), including the should_render == False + # path and any exception above. Skipping it wedges the frame loop. + viz_session.end_frame() + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--verbose", action="store_true", help="Debug-level logging.") + CloudXRLauncher.add_launcher_arguments(parser) + args = parser.parse_args(argv[1:]) + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="[mujoco_xr] %(message)s", + ) + + # Before launch_context starts the runtime, so an unfetched checkout says so + # plainly instead of buried in the runtime's own startup logging. + missing = _missing_leader_assets() + if missing: + raise SystemExit( + f"mujoco_xr: the leader gripper meshes are not fetched ({', '.join(missing)}).\n" + f" Run {FETCH_SCRIPT} from the repository root, then reinstall:\n" + " uv pip install --reinstall-package isaacteleop-examples-mujoco-xr " + "./examples/mujoco_xr" + ) + + with CloudXRLauncher.launch_context(args) as launcher: + if launcher is not None: + LOG.info("CloudXR runtime started (WSS log: %s)", launcher.wss_log_path) + try: + return run() + except KeyboardInterrupt: + LOG.info("interrupted") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml new file mode 100644 index 000000000..0f0de1cb2 --- /dev/null +++ b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/scene.xml b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/scene.xml new file mode 100644 index 000000000..136f285f9 --- /dev/null +++ b/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/scene.xml @@ -0,0 +1,30 @@ + + + + diff --git a/examples/mujoco_xr/scripts/fetch-so-arm.sh b/examples/mujoco_xr/scripts/fetch-so-arm.sh new file mode 100755 index 000000000..7a24f1405 --- /dev/null +++ b/examples/mujoco_xr/scripts/fetch-so-arm.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Fetches the SO-101 leader-gripper assets, rather than vendoring 2.3 MB of +# binary STL that Git LFS made every clone pay for. +# +# Nothing calls this at build time: an isolated PEP-517 wheel build must not +# reach the network, so it is an explicit step and the app names it at startup. +# The files are package data, so REINSTALL afterwards -- skip that and the ghost +# works from the source tree and fails from the wheel. +set -euo pipefail + +# The pin. Bump it and the checksums together or the download is refused. +COMMIT="fda892cba81032c46c40976a48c9ceadbf40a9ca" +REPO="TheRobotStudio/SO-ARM100" + +DEST="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/python/isaacteleop_examples/mujoco_xr/assets/leader" + +# upstream path local name sha256 +# +# The URDF is where app.py's trigger hinge comes from, and having it on disk is +# what lets test_ghost.py check those constants against their source. +ASSETS=( + "STL/SO101/Individual/Wrist_Roll_SO101.stl Wrist_Roll_SO101.stl de3a65044dd4ae8bcb9659d8ca2b49598e3f5571edf89f45ad975e9776a7ffee" + "STL/SO101/Individual/Trigger_SO101.stl Trigger_SO101.stl 48ecec3a3710cffdc0ae96d28547e49ddf4cbc93ccd915be7549f78e00ad2850" + "STL/SO101/Individual/Handle_SO101.stl Handle_SO101.stl fb8757bdff009c04c207481dd664813ccdac2ad989acea6057df780b52327281" + "Simulation/SO101/assets/sts3215_03a_v1.stl STS3215_03a.stl a37c871fb502483ab96c256baf457d36f2e97afc9205313d9c5ab275ef941cd0" + "Simulation/SO101/so101_new_calib.urdf so101_new_calib.urdf 3a65d2d35e68a8d2f0c2cc176d19b884506543c93ba72980145b80abe276022c" + "LICENSE LICENSE c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" +) + +mkdir -p "$DEST" +echo "Fetching SO-ARM100 assets at ${COMMIT:0:12} into ${DEST}" + +for entry in "${ASSETS[@]}"; do + read -r remote local sha <<<"$entry" + target="${DEST}/${local}" + if [[ -f "$target" ]] && echo "${sha} ${target}" | sha256sum --check --status; then + echo " ok ${local}" + continue + fi + url="https://raw.githubusercontent.com/${REPO}/${COMMIT}/${remote}" + echo " fetching ${local}" + curl -fsSL "$url" -o "${target}.part" + # A raw.githubusercontent path is not immutable in practice, and a silently + # substituted mesh renders as a broken gripper rather than an error. + if ! echo "${sha} ${target}.part" | sha256sum --check --status; then + rm -f "${target}.part" + echo "ERROR: checksum mismatch for ${remote}." >&2 + echo " Upstream changed, or COMMIT and the hashes above disagree." >&2 + exit 1 + fi + mv "${target}.part" "$target" +done + +echo +echo "Done. These are package data, so install before running:" +echo " uv pip install --reinstall-package isaacteleop-examples-mujoco-xr ./examples/mujoco_xr" diff --git a/examples/mujoco_xr/tests/CMakeLists.txt b/examples/mujoco_xr/tests/CMakeLists.txt new file mode 100644 index 000000000..a14366f85 --- /dev/null +++ b/examples/mujoco_xr/tests/CMakeLists.txt @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# mujoco_xr tests. Each test_*.py registers as its own ctest entry under the +# `mujoco_xr` label -- same shape as examples/camera_viz/tests/CMakeLists.txt. +# +# Keep tests here unit-level -- no GPU, no headset, no CloudXR runtime, no +# window system -- because a test gated on hardware one developer has reports +# green by skipping, and examples have no CI to run it in +# (NVIDIA/IsaacTeleop#880). test_readback.py is the one exception and adding a +# second needs the same justification: what it covers is otherwise invisible +# until someone is wearing a headset, and it needs no headset itself. + +file(GLOB TEST_FILES + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py" +) + +foreach(test_file ${TEST_FILES}) + get_filename_component(test_name "${test_file}" NAME_WE) + add_test( + NAME "mujoco_xr_${test_name}" + COMMAND uv run --python ${ISAAC_TELEOP_PYTHON_VERSION} --extra dev + pytest -v --tb=short "${test_file}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + # Resolves isaacteleop only; the example's own package is reached from the + # source tree by conftest.py. + # + # If a second root is ever added: CMake's ENVIRONMENT property is a + # semicolon-separated list, so two "PYTHONPATH=..." elements silently + # produce two entries and only the last survives. Join additional roots with + # ':' inside one quoted string. + set_tests_properties("mujoco_xr_${test_name}" PROPERTIES + ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/python_package/$" + LABELS "mujoco_xr" + ) +endforeach() diff --git a/examples/mujoco_xr/tests/conftest.py b/examples/mujoco_xr/tests/conftest.py new file mode 100644 index 000000000..da957eec0 --- /dev/null +++ b/examples/mujoco_xr/tests/conftest.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Resolve `isaacteleop_examples.mujoco_xr` against the in-tree source, and with +# it the _mujoco_xr*.so built in place beside __init__.py. Here rather than in +# the ctest ENVIRONMENT so a bare `pytest` works too. +# +# python/, not python/isaacteleop_examples/: that is a PEP 420 namespace. Do not +# add an __init__.py to make an import work -- it breaks the installed wheel's +# ability to share the namespace. + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "python")) diff --git a/examples/mujoco_xr/tests/pyproject.toml b/examples/mujoco_xr/tests/pyproject.toml new file mode 100644 index 000000000..25875cbf7 --- /dev/null +++ b/examples/mujoco_xr/tests/pyproject.toml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Pyproject for the mujoco_xr tests. It exists so that `uv run` resolves HERE +# rather than walking up to examples/ and then to the repository root project +# (which has no mujoco and entirely different deps). Precedent: +# examples/camera_viz/tests/pyproject.toml. + +[project] +# Cosmetic -- nothing reads it, and ../CMakeLists.txt's pin check reads this +# file rather than this field. +name = "isaacteleop-examples-mujoco-xr-tests" +version = "0.0.0" +# Pinned because these tests import the ABI-specific _mujoco_xr*.so built by the +# preset's interpreter. +requires-python = "==3.12.*" + +[project.optional-dependencies] +dev = [ + "pytest", + "numpy", + # Keep in sync with the pin in ../pyproject.toml; ../CMakeLists.txt reads + # both files and fails the configure on a disagreement. It matches every + # `mujoco==` here, so do not restate the number in prose -- say "the pin + # below" or a comment edit becomes a configure failure. + "mujoco==3.11.0", +] + +[tool.pytest.ini_options] +pythonpath = ["."] diff --git a/examples/mujoco_xr/tests/test_app_helpers.py b/examples/mujoco_xr/tests/test_app_helpers.py new file mode 100644 index 000000000..23bbbdb47 --- /dev/null +++ b/examples/mujoco_xr/tests/test_app_helpers.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pure helpers from the app that guard against silent-corruption bugs.""" + +import pytest + +app = pytest.importorskip( + "isaacteleop_examples.mujoco_xr.app", reason="isaacteleop is not on PYTHONPATH" +) + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + (0.011, 0.011), + (0.0, 0.0), + (-1.0, 0.0), # clock went backwards + (5.0, app.MAX_DT_S), # a long stall + (float("inf"), app.MAX_DT_S), + ], +) +def test_clamp_dt(raw, expected): + assert app._clamp_dt(raw) == expected + + +def test_clamp_dt_sends_nan_to_zero(): + """The whole reason the clamp uses comparisons. + + ``min(max(nan, 0), 0.1)`` is nan, so NaN passes both limits into mj_step and + poisons every qpos. ``nan > 0`` is False, so the comparison form sends it to 0. + """ + assert app._clamp_dt(float("nan")) == 0.0 + + +def test_frame_clock_refuses_the_zeroed_timestamp(): + """Regression: the 50-step physics lurch at every session start. + + ``viz_session.cpp:255-256`` zeroes ``predicted_display_time`` with + ``should_render`` on every pre-kRunning frame. Sampling that zero makes the + next real frame compute ``dt = t_now - 0``, clamp to MAX_DT_S, and step 0.1 s + inside one display frame. + """ + + class _Info: + predicted_display_time = 0 + + assert app._frame_clock(_Info()) is None + + _Info.predicted_display_time = 2_000_000_000 # ns + assert app._frame_clock(_Info()) == 2.0 + + +class _Fov: + angle_left = -0.7 + angle_right = 0.7 + angle_up = 0.7 + angle_down = -0.7 + + +def _good_frustum(): + from isaacteleop_examples.mujoco_xr import _mujoco_xr + + return list( + _mujoco_xr.frustum_from_fov( + [_Fov.angle_left, _Fov.angle_right, _Fov.angle_up, _Fov.angle_down], + app.NEAR_Z, + app.FAR_Z, + ) + ) + + +def test_assert_frustum_accepts_what_the_renderer_builds(): + """Its rejections mean nothing until it passes on the real thing: float32 + round-tripping alone could make it fire on every frame.""" + app._assert_frustum(_good_frustum(), _Fov(), app.NEAR_Z, app.FAR_Z) + + +@pytest.mark.parametrize( + ("index", "broken", "message"), + [ + # Zero half-width is the one wrong value mjr_render does not complain + # about: it turns the viewport-aspect fallback on. + (1, lambda v: 0.0, "degenerate frustum"), + (5, lambda v: v * 2.0, "clip planes drifted"), + # The optical axis slid, with nothing else touched. + (0, lambda v: v + 0.01, "frustum left"), + ], +) +def test_assert_frustum_rejects(index, broken, message): + f = _good_frustum() + f[index] = broken(f[index]) + with pytest.raises(AssertionError, match=message): + app._assert_frustum(f, _Fov(), app.NEAR_Z, app.FAR_Z) diff --git a/examples/mujoco_xr/tests/test_frames.py b/examples/mujoco_xr/tests/test_frames.py new file mode 100644 index 000000000..5ceee2bce --- /dev/null +++ b/examples/mujoco_xr/tests/test_frames.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The XR -> MuJoCo crossing: the handedness map and the quaternion order. + +Both are cheap to get wrong and expensive to debug on hardware. +""" + +import math + +import numpy as np +import pytest + +from isaacteleop_examples.mujoco_xr import _mujoco_xr + + +def test_extension_and_wheel_share_one_libmujoco(): + import mujoco + + assert _mujoco_xr.mujoco_version() == mujoco.mj_versionString() + + +def test_axis_map_is_rep103(): + """XR -Z -> MJ +x, +Y -> +z, +X -> -y, on the rotation alone. + + The workspace translation is subtracted, so re-measuring it cannot fail this. + """ + t = np.asarray(_mujoco_xr.TRANS_MJ_FROM_XR) + + forward = np.asarray(_mujoco_xr.mj_from_xr_pos([0.0, 0.0, -1.0])) - t + up = np.asarray(_mujoco_xr.mj_from_xr_pos([0.0, 1.0, 0.0])) - t + right = np.asarray(_mujoco_xr.mj_from_xr_pos([1.0, 0.0, 0.0])) - t + + np.testing.assert_allclose(forward, [1.0, 0.0, 0.0], atol=1e-12) + np.testing.assert_allclose(up, [0.0, 0.0, 1.0], atol=1e-12) + np.testing.assert_allclose(right, [0.0, -1.0, 0.0], atol=1e-12) + + +@pytest.mark.parametrize("eye_height", [0.0, 1.2, 1.6]) +def test_point_one_metre_in_front_at_eye_height(eye_height): + """frames.hpp's definition, executable: a point 1 m in front of the operator + at eye height h lands at MuJoCo (+1, 0, h), before the workspace translation. + """ + t = np.asarray(_mujoco_xr.TRANS_MJ_FROM_XR) + p_mj = np.asarray(_mujoco_xr.mj_from_xr_pos([0.0, eye_height, -1.0])) - t + np.testing.assert_allclose(p_mj, [1.0, 0.0, eye_height], atol=1e-12) + + +def test_translation_has_both_terms(): + """Neither term may be silently zeroed: x is operator standoff and z the + floor datum, and they are independent.""" + t = _mujoco_xr.TRANS_MJ_FROM_XR + assert t[0] != 0.0, "operator standoff was zeroed" + assert t[2] != 0.0, "floor datum was zeroed" + assert t[1] == 0.0 + + +def test_identity_orientation_maps_to_the_convention_quaternion(): + q_xyzw_identity = [0.0, 0.0, 0.0, 1.0] + q_wxyz = _mujoco_xr.mj_from_xr_quat(q_xyzw_identity) + np.testing.assert_allclose(q_wxyz, _mujoco_xr.QUAT_MJ_FROM_XR, atol=1e-12) + + +def test_quaternion_input_order_is_xyzw_not_wxyz(): + """A 90-degree roll about XR +Z, spelled xyzw. + + NINETY degrees, not 180: a 180-degree roll is (0, 0, 1, 0), which read as + wxyz is a roll about XR +Y, and BOTH send local +x to MuJoCo +y -- a probe + that passes whichever way the binding reads its input. The second half pins + that this probe does discriminate. + """ + import mujoco + + s = math.sin(math.radians(45.0)) + q_xyzw = [0.0, 0.0, s, s] # (x, y, z, w) = 90 deg about z_xr + q_wxyz = np.asarray(_mujoco_xr.mj_from_xr_quat(q_xyzw)) + + local_x = np.zeros(3) + mujoco.mju_rotVecQuat(local_x, np.array([1.0, 0.0, 0.0]), q_wxyz) + np.testing.assert_allclose(local_x, [0.0, 0.0, 1.0], atol=1e-12) + + # The same four numbers misread as wxyz are a 180-degree rotation about + # (0, s, s), which lands on MuJoCo +y instead. So the assertion above is + # genuinely sensitive to the component order. + q_misread = np.asarray( + _mujoco_xr.mj_from_xr_quat([q_xyzw[1], q_xyzw[2], q_xyzw[3], q_xyzw[0]]) + ) + misread_x = np.zeros(3) + mujoco.mju_rotVecQuat(misread_x, np.array([1.0, 0.0, 0.0]), q_misread) + np.testing.assert_allclose(misread_x, [0.0, 1.0, 0.0], atol=1e-12) + + assert math.isclose(float(np.linalg.norm(q_wxyz)), 1.0, rel_tol=1e-9) diff --git a/examples/mujoco_xr/tests/test_ghost.py b/examples/mujoco_xr/tests/test_ghost.py new file mode 100644 index 000000000..c1d46105a --- /dev/null +++ b/examples/mujoco_xr/tests/test_ghost.py @@ -0,0 +1,453 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The leader-gripper ghost: the overlay, its geometry, and when it is written. + +Everything here is headless, so the one thing it cannot check is how the ghost +looks through a headset. +""" + +import math +from xml.etree import ElementTree + +import numpy as np +import pytest + +app = pytest.importorskip( + "isaacteleop_examples.mujoco_xr.app", + reason="isaacteleop is not on PYTHONPATH", +) +_mujoco_xr = pytest.importorskip("isaacteleop_examples.mujoco_xr._mujoco_xr") +mujoco = pytest.importorskip("mujoco") + +from isaacteleop.retargeting_engine.tensor_types import ( # noqa: E402 + ControllerInputIndex, +) + +GHOST_GEOMS = ( + "leader_ghost_wrist_roll", + "leader_ghost_motor", + "leader_ghost_trigger", + "leader_ghost_handle", +) + + +def _default_scene(): + """The shipped scene, skipping on an unfetched checkout. + + Saying which meshes are missing beats MuJoCo's "Error opening file". + """ + missing = app._missing_leader_assets() + if missing: + pytest.skip( + f"leader meshes not fetched ({', '.join(missing)}); run {app.FETCH_SCRIPT}" + ) + return mujoco.MjModel.from_xml_path(str(app.DEFAULT_SCENE)) + + +def _scene(model, data): + mujoco.mj_forward(model, data) + option = mujoco.MjvOption() + mujoco.mjv_defaultOption(option) + camera = mujoco.MjvCamera() + mujoco.mjv_defaultFreeCamera(model, camera) + scene = mujoco.MjvScene(model, 20000) + mujoco.mjv_updateScene( + model, data, option, None, camera, mujoco.mjtCatBit.mjCAT_ALL, scene + ) + return scene + + +def _geom_verts_world(model, data, name): + gid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_GEOM, name) + mesh = model.geom_dataid[gid] + adr, num = model.mesh_vertadr[mesh], model.mesh_vertnum[mesh] + verts = np.array(model.mesh_vert[adr : adr + num], dtype=float) + rot = data.geom_xmat[gid].reshape(3, 3) + return verts @ rot.T + data.geom_xpos[gid] + + +def _nearest_gap(a, b, stride=7, block=200): + a = a[::stride] + b = b[::stride] + best = math.inf + for i in range(0, len(a), block): + d = np.linalg.norm(a[i : i + block, None, :] - b[None, :, :], axis=2) + best = min(best, float(d.min())) + return best + + +# --------------------------------------------------------------------------- +# A stubbed pipeline result. ``app._update_ghost`` reads three fields through +# the mapping protocol, and stubbing them is what keeps this file headless. +# --------------------------------------------------------------------------- + + +class _Controller: + is_none = False + + def __init__(self, valid, pos=(0.0, 0.0, 0.0), quat_xyzw=(0.0, 0.0, 0.0, 1.0)): + self._fields = { + ControllerInputIndex.GRIP_IS_VALID: valid, + ControllerInputIndex.GRIP_POSITION: pos, + ControllerInputIndex.GRIP_ORIENTATION: quat_xyzw, + } + + def __getitem__(self, index): + return self._fields[index] + + +class _NoController: + """What the pipeline yields for a hand it has no sample for.""" + + is_none = True + + def __getitem__(self, index): # pragma: no cover -- reaching this IS the bug + raise AssertionError("an is_none controller must never be read") + + +def _result(controller, closedness=0.0): + """Both hands plus the jaw channel, shaped like the real combiner output.""" + other = ( + app.ControllersSource.LEFT + if app.GHOST_HAND == app.ControllersSource.RIGHT + else app.ControllersSource.RIGHT + ) + return { + app.GHOST_HAND: controller, + other: _NoController(), + app.GRIPPER_COMMAND_KEY: [closedness], + } + + +def test_the_ghost_is_opaque_and_collides_with_nothing(): + """Opaque, so draw order and the blending risks stop mattering. + + Read off the SCENE geom: model.geom_rgba still holds MuJoCo's default, so + asserting that would pass on a translucent ghost too. + """ + model = _default_scene() + data = mujoco.MjData(model) + scene = _scene(model, data) + by_objid = { + int(scene.geoms[i].objid): i + for i in range(scene.ngeom) + if scene.geoms[i].objtype == mujoco.mjtObj.mjOBJ_GEOM + } + for name in GHOST_GEOMS: + gid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_GEOM, name) + assert scene.geoms[by_objid[gid]].rgba[3] == pytest.approx(1.0) + # Contact would let the hand shove scene content around. + assert model.geom_contype[gid] == 0 + assert model.geom_conaffinity[gid] == 0 + # mjModel aggregates geom mass, so `mass="0"` is checked where it lands. A + # mocap body is kinematic either way, but a non-zero mass here would change + # the model's total and any inertia-derived diagnostic built on it. + for body_name in (app.GHOST_BODY, app.GHOST_JAW_BODY): + body = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, body_name) + assert model.body_mass[body] == 0.0 + + +def test_both_ghost_bodies_are_mocap_and_kinematic(): + """Two mocap bodies, no joints, parented to world. + + The trigger is a second mocap body rather than a jointed child because + mj_step integrates gravity into a joint (measured: 0.06 rad over 50 steps). + """ + model = _default_scene() + bodies = [ + mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, n) + for n in (app.GHOST_BODY, app.GHOST_JAW_BODY) + ] + assert all(b >= 0 for b in bodies) + for body in bodies: + assert model.body_mocapid[body] >= 0 + assert model.body_parentid[body] == 0, "a mocap body must be a child of world" + assert model.body_jntnum[body] == 0 + + +# --------------------------------------------------------------------------- +# The geometry. All three transforms are DERIVED; this is the derivation +# checking itself. +# --------------------------------------------------------------------------- + + +def test_the_three_leader_parts_form_one_assembly(): + """Sub-mm where the parts bolt, mm of clearance where one pivots. + + An STL refresh that broke the shared CAD datum opens these gaps rather than + quietly rendering three pieces near each other. + """ + model = _default_scene() + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + verts = {n: _geom_verts_world(model, data, n) for n in GHOST_GEOMS} + + bolted = _nearest_gap( + verts["leader_ghost_wrist_roll"], verts["leader_ghost_handle"] + ) + assert bolted < 1e-3, f"shank-to-handle gap {bolted * 1000:.2f} mm" + for other in ("leader_ghost_trigger",): + for part in ("leader_ghost_wrist_roll", "leader_ghost_handle"): + gap = _nearest_gap(verts[part], verts[other]) + assert gap < 5e-3, f"{part} to {other} gap {gap * 1000:.2f} mm" + + +def test_the_servo_fills_the_notch_in_the_wrist_bracket(): + """`wrist_roll` is a C-shaped bracket; the servo is what sits in it. + + Contact plus the size of a real STS3215, which catches the units trap: this + mesh is Menagerie's, in metres, while its neighbours are mm print STLs. + """ + model = _default_scene() + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + servo = _geom_verts_world(model, data, "leader_ghost_motor") + bracket = _geom_verts_world(model, data, "leader_ghost_wrist_roll") + assert _nearest_gap(servo, bracket) < 1e-3, "the servo is not seated in the bracket" + extent = np.ptp(servo, axis=0) + assert np.allclose(np.sort(extent), (0.0248, 0.0396, 0.0454), atol=2e-3), ( + f"servo spans {np.round(extent * 1000, 1)} mm -- an STS3215 is 45x25x40" + ) + + +def test_the_leader_meshes_are_scaled_from_millimetres(): + """`scale="0.001"`, and getting it wrong does not read as "a big mesh" -- + the camera ends up inside a 65 m solid. The servo is absent from this list + deliberately: it is authored in metres and carries no scale.""" + model = _default_scene() + for name in ("leader_wrist_roll", "leader_trigger", "leader_handle"): + mesh = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_MESH, name) + assert mesh >= 0 + adr, num = model.mesh_vertadr[mesh], model.mesh_vertnum[mesh] + verts = np.array(model.mesh_vert[adr : adr + num], dtype=float) + extent = float(np.ptp(verts, axis=0).max()) + assert 0.02 < extent < 0.30, f"{name} spans {extent:.3f} m" + + +# --------------------------------------------------------------------------- +# When the ghost is written. +# --------------------------------------------------------------------------- + + +def test_the_ghost_is_rigidly_attached_to_the_grip_frame(): + """The contract the calibration must satisfy, whatever its value. + + Left-multiplying instead swings the ghost around the room as the operator + turns, while still looking right at one orientation -- which is what makes + it survive a spot check. Asserted as invariance rather than a posture, so + re-tuning on a headset cannot turn it red. + """ + model = _default_scene() + data = mujoco.MjData(model) + ghost = app._resolve_ghost(model) + + seen = [] + for grip_pos, grip_quat_xyzw in ( + ((0.0, 1.2, -0.5), (0.0, 0.0, 0.0, 1.0)), + ((0.31, 1.24, -0.42), (0.0, 0.3826834, 0.0, 0.9238795)), + ((-0.2, 0.9, -0.8), (0.5, 0.5, 0.5, 0.5)), + ): + app._update_ghost( + data, ghost, _result(_Controller(True, grip_pos, grip_quat_xyzw)) + ) + q_world_from_grip = np.array(_mujoco_xr.mj_from_xr_quat(list(grip_quat_xyzw))) + inverse, relative = np.empty(4), np.empty(4) + mujoco.mju_negQuat(inverse, q_world_from_grip) + mujoco.mju_mulQuat(relative, inverse, np.array(data.mocap_quat[ghost.body])) + + rot = np.empty(9) + mujoco.mju_quat2Mat(rot, q_world_from_grip) + offset = ( + np.array(data.mocap_pos[ghost.body]) + - np.array(_mujoco_xr.mj_from_xr_pos(list(grip_pos))) + ) @ rot.reshape(3, 3) + seen.append((relative, offset)) + + for relative, offset in seen[1:]: + assert np.allclose(relative, seen[0][0], atol=1e-6), ( + "the ghost's orientation in the grip frame changes with the " + "controller's orientation -- the correction is composed on the wrong side" + ) + assert np.allclose(offset, seen[0][1], atol=1e-6), ( + "the ghost's offset in the grip frame changes with the controller's " + "orientation -- the translation is not being rotated with the grip" + ) + # And it is the configured correction, not some other rigid attachment. + assert np.allclose(seen[0][0], app._QUAT_GRIP_FROM_GHOST, atol=1e-6) + assert np.allclose(seen[0][1], app._POS_GRIP_FROM_GHOST, atol=1e-6) + + +def test_squeezing_drives_the_jaw_from_released_to_squeezed(): + """Closedness 0..1 must drive the hinge from released to squeezed. + + On the recovered ANGLE, not on where a point ends up: over a large sweep a + point on the lever traces an arc, rising along any fixed axis before falling. + """ + model = _default_scene() + data = mujoco.MjData(model) + ghost = app._resolve_ghost(model) + controller = _Controller(True, (0.0, 1.2, -0.5)) + + def hinge_angle_at(closedness): + app._update_ghost(data, ghost, _result(controller, closedness)) + mujoco.mj_forward(model, data) + inverse, hinge = np.empty(4), np.empty(4) + mujoco.mju_negQuat(inverse, np.array(data.mocap_quat[ghost.body])) + mujoco.mju_mulQuat(hinge, inverse, np.array(data.mocap_quat[ghost.jaw])) + # Signed against the hinge axis, so a wrong-way rotation reads negative + # rather than folding onto the same magnitude. + turn = 2.0 * math.atan2(float(np.linalg.norm(hinge[1:])), float(hinge[0])) + if float(np.dot(hinge[1:], app._TRIGGER_HINGE_AXIS)) < 0: + turn = -turn + return turn + + angles = [hinge_angle_at(c) for c in (0.0, 0.25, 0.5, 0.75, 1.0)] + assert angles[0] == pytest.approx(app._TRIGGER_RELEASED_RAD, abs=1e-6) + assert angles[-1] == pytest.approx(app._TRIGGER_SQUEEZED_RAD, abs=1e-6) + assert all(b < a for a, b in zip(angles, angles[1:])), ( + f"squeezing did not close the jaw monotonically: {np.round(angles, 4)}" + ) + + # And it is big enough to see: the far end of the lever sweeps ~90 mm. + def trigger_at(closedness): + app._update_ghost(data, ghost, _result(controller, closedness)) + mujoco.mj_forward(model, data) + return _geom_verts_world(model, data, "leader_ghost_trigger") + + travel = float(np.linalg.norm(trigger_at(1.0) - trigger_at(0.0), axis=1).max()) + # 84.5 mm at the tip across the joint's 0..100 degrees. Below ~50 mm + # "released" stops reading as OPEN, which is why the range is the joint's. + assert travel > 0.05, f"the trigger moves {travel * 1000:.1f} mm -- not visible" + + +def test_the_released_end_is_the_urdf_joints_upper_limit(): + """The travel is the URDF's, not a tuned number. + + Read out of the fetched so101_new_calib.urdf, so the constant is checked + against its source instead of against itself. + """ + urdf = app._LEADER_ASSETS / "so101_new_calib.urdf" + if not urdf.is_file(): + pytest.skip(f"{urdf.name} not fetched; run {app.FETCH_SCRIPT}") + tree = ElementTree.parse(urdf) + joint = next(j for j in tree.iter("joint") if j.get("name") == "gripper") + upper = float(joint.find("limit").get("upper")) + assert app._TRIGGER_RELEASED_RAD == pytest.approx(upper, abs=1e-4) + # The other end is the joint's authored zero, NOT its lower limit, which + # swings the lever into the servo. + assert app._TRIGGER_SQUEEZED_RAD == 0.0 + assert float(joint.find("limit").get("lower")) == pytest.approx( + math.radians(-10.0), abs=1e-4 + ) + + +def test_the_trigger_clears_the_whole_gripper_across_its_driven_range(): + """The lever must not pass through the gripper at any closedness. + + Against all three other parts, not the bracket alone: a range that swung + the loop into the SERVO once passed a bracket-only check. The 0.8 mm bound + is thin on purpose -- the tightest legitimate pass is 2.10 mm, while a lever + driven to the joint's -10 degree limit closes to 0.4 mm, and nearest-vertex + distance cannot go negative so interpenetration reads as a small positive. + """ + model = _default_scene() + data = mujoco.MjData(model) + ghost = app._resolve_ghost(model) + others = ( + "leader_ghost_wrist_roll", + "leader_ghost_motor", + "leader_ghost_handle", + ) + + worst = (0.0, "", 1e9) + for step in range(9): + closedness = step / 8 + app._update_ghost( + data, ghost, _result(_Controller(True, (0.0, 1.2, -0.5)), closedness) + ) + mujoco.mj_forward(model, data) + trigger = _geom_verts_world(model, data, "leader_ghost_trigger") + for part in others: + gap = _nearest_gap(trigger, _geom_verts_world(model, data, part)) + if gap < worst[2]: + worst = (closedness, part, gap) + assert worst[2] > 0.8e-3, ( + f"the trigger is {worst[2] * 1000:.2f} mm into {worst[1]} at closedness " + f"{worst[0]:.3f} -- the driven range pushes it through the body" + ) + + +def test_the_shipped_retargeter_drives_the_jaw_channel(): + """The graph edge itself: trigger -> SO101GripperRetargeter -> combiner key. + + The real pipeline on synthetic DeviceIO snapshots, so the key, the indexing + and the deadzone are the shipped retargeter's, not this file's idea of them. + """ + from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource + from isaacteleop.retargeting_engine.interface.tensor_group import TensorGroup + from isaacteleop.schema import ( + ControllerInputState, + ControllerPose, + ControllerSnapshot, + ControllerSnapshotTrackedT, + Point, + Pose, + Quaternion, + ) + + def snapshot(trigger): + pose = ControllerPose( + Pose(Point(0.1, 1.2, -0.4), Quaternion(0.0, 0.0, 0.0, 1.0)), True + ) + state = ControllerInputState( + primary_click=False, + secondary_click=False, + thumbstick_click=False, + menu_click=False, + thumbstick_x=0.0, + thumbstick_y=0.0, + squeeze_value=0.0, + trigger_value=trigger, + ) + return ControllerSnapshotTrackedT(ControllerSnapshot(pose, pose, state)) + + pipeline = app._build_pipeline() + spec = ControllersSource(name="controllers").input_spec() + + def closedness(trigger): + inputs = {} + for name in spec: + group = TensorGroup(spec[name]) + group[0] = snapshot(trigger) + inputs[name] = group + out = pipeline.execute_pipeline({"controllers": inputs}) + assert app.GRIPPER_COMMAND_KEY in out + return float(out[app.GRIPPER_COMMAND_KEY][0]) + + assert closedness(0.0) == pytest.approx(0.0) + assert closedness(1.0) == pytest.approx(1.0) + # The retargeter's own released-end deadzone, not this app's: (0.5 - 0.05) / 0.95. + assert closedness(0.5) == pytest.approx(0.4737, abs=1e-4) + + +def test_an_untracked_controller_freezes_the_whole_gripper(): + """(0, 0, 0) is the scene origin, a pose a tracked controller could hold. + + Freezing is the honest rendering of "tracking lost", and the jaw freezes + with the body rather than articulating on a stale pose. + """ + model = _default_scene() + data = mujoco.MjData(model) + ghost = app._resolve_ghost(model) + app._update_ghost( + data, ghost, _result(_Controller(True, (0.2, 1.3, -0.5)), closedness=0.0) + ) + seen_body = data.mocap_pos[ghost.body].copy() + seen_jaw = data.mocap_quat[ghost.jaw].copy() + + for controller in (_Controller(False, (9.0, 9.0, 9.0)), _NoController()): + for _ in range(3): + app._update_ghost(data, ghost, _result(controller, closedness=1.0)) + assert np.array_equal(data.mocap_pos[ghost.body], seen_body) + assert np.array_equal(data.mocap_quat[ghost.jaw], seen_jaw) diff --git a/examples/mujoco_xr/tests/test_projection.py b/examples/mujoco_xr/tests/test_projection.py new file mode 100644 index 000000000..7d25f48b6 --- /dev/null +++ b/examples/mujoco_xr/tests/test_projection.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""What the app hands mjvGLCamera, and what comes back in the depth buffer. + +mjr_render builds its own projection from the frustum fields, so the app's share +of the clip convention is those six numbers plus the depth inversion on readback. +Neither needs a GPU, a headset or a VizSession. +""" + +import math + +import pytest + +from isaacteleop_examples.mujoco_xr import _mujoco_xr + +NEAR = 0.05 +FAR = 50.0 + +# A plausible asymmetric headset fov, in radians. +FOV = [math.radians(-45.0), math.radians(42.0), math.radians(48.0), math.radians(-46.0)] + +CENTER, HALF_WIDTH, BOTTOM, TOP, F_NEAR, F_FAR = range(6) + + +def test_the_frustum_is_the_fov_projected_onto_the_near_plane(): + """Also pins that there is no y flip here: angle_up lands on TOP, and the + flip happens once, on readback.""" + f = _mujoco_xr.frustum_from_fov(FOV, NEAR, FAR) + assert f[CENTER] - f[HALF_WIDTH] == pytest.approx(NEAR * math.tan(FOV[0])) + assert f[CENTER] + f[HALF_WIDTH] == pytest.approx(NEAR * math.tan(FOV[1])) + assert f[TOP] == pytest.approx(NEAR * math.tan(FOV[2])) + assert f[BOTTOM] == pytest.approx(NEAR * math.tan(FOV[3])) + assert (f[F_NEAR], f[F_FAR]) == pytest.approx((NEAR, FAR), rel=1e-6) + + +def test_half_width_is_set_and_not_left_to_the_aspect_fallback(): + """The load-bearing assertion. + + At zero, render_gl3.c's setView derives the horizontal extent from the + viewport aspect instead, and the drift shows on a headset as world-locked + geometry sliding sideways under head motion. + """ + f = _mujoco_xr.frustum_from_fov(FOV, NEAR, FAR) + assert f[HALF_WIDTH] > 0.0 + + aspect_derived = 0.5 * (f[TOP] - f[BOTTOM]) + assert f[HALF_WIDTH] != pytest.approx(aspect_derived), ( + "this fov happens to be square, so the test cannot tell the fallback apart" + ) + + +def test_a_default_constructed_fov_is_rejected_loudly(): + """A default-constructed viz::Fov is four ZEROS, and must never render. + + Zero half_width turns the aspect fallback on, so the frame comes back + looking plausible from a fov carrying nothing. The runtime fills + ``FrameInfo.views``, so the app can only refuse, not prevent. + """ + with pytest.raises(ValueError): + _mujoco_xr.frustum_from_fov([0.0, 0.0, 0.0, 0.0], NEAR, FAR) + + +def test_near_far_are_validated(): + with pytest.raises(ValueError): + _mujoco_xr.frustum_from_fov(FOV, 0.0, FAR) + with pytest.raises(ValueError): + _mujoco_xr.frustum_from_fov(FOV, FAR, NEAR) + + +def test_submitted_depth_is_standard_z_not_the_reverse_z_mujoco_writes(): + """near -> 0, far -> 1, monotonic between. + + mjr_render writes the opposite and gl_readback.cpp's shader subtracts it + from 1; this is the specification that subtraction implements. + """ + assert _mujoco_xr.submitted_depth(NEAR, NEAR, FAR) == pytest.approx(0.0, abs=1e-6) + assert _mujoco_xr.submitted_depth(FAR, NEAR, FAR) == pytest.approx(1.0, abs=1e-6) + + depths = [_mujoco_xr.submitted_depth(d, NEAR, FAR) for d in (NEAR, 0.5, 5.0, FAR)] + assert depths == sorted(depths) diff --git a/examples/mujoco_xr/tests/test_readback.py b/examples/mujoco_xr/tests/test_readback.py new file mode 100644 index 000000000..cd0b99675 --- /dev/null +++ b/examples/mujoco_xr/tests/test_readback.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The OpenGL -> CUDA readback, on a real GPU and with no headset. + +The only test here that touches hardware, and it skips loudly without it. What +it buys is the two conversions that are otherwise invisible until someone is +wearing a headset: the y flip and the depth inversion. It stops at +ProjectionLayer.submit and covers nothing downstream. +""" + +import ctypes +import math + +import numpy as np +import pytest + +mujoco = pytest.importorskip("mujoco") +_mujoco_xr = pytest.importorskip("isaacteleop_examples.mujoco_xr._mujoco_xr") + +from isaacteleop_examples.mujoco_xr.app import DEFAULT_SCENE, FAR_Z, NEAR_Z # noqa: E402 + +W = H = 256 +HALF_FOV = math.radians(45.0) +GHOST_DISTANCE = 0.6 # metres straight ahead of the eye +GHOST_OFFSET = 0.15 # metres off-axis, comfortably outside the gripper's own size + + +@pytest.fixture(scope="module") +def rendered(): + """A live Renderer plus a device-to-host copier, or a skip saying why.""" + model = mujoco.MjModel.from_xml_path(str(DEFAULT_SCENE)) + data = mujoco.MjData(model) + + try: + gl = mujoco.GLContext(W, H) + gl.make_current() + except Exception as exc: # noqa: BLE001 -- any GL backend failure means "no GPU here" + pytest.skip(f"no usable OpenGL context: {exc}") + + model.vis.quality.offsamples = 0 + try: + renderer = _mujoco_xr.Renderer( + width=W, + height=H, + view_count=2, + near_z=NEAR_Z, + far_z=FAR_Z, + model_address=model._address, + ) + except RuntimeError as exc: + gl.free() + # Includes the multi-GPU case, where the EGL device and the process's + # CUDA device differ and the fix is MUJOCO_EGL_DEVICE_ID. + pytest.skip(f"renderer unavailable: {exc}") + + cuda = ctypes.CDLL("libcuda.so.1") + + def read(view, is_depth): + img = renderer.depth(view) if is_depth else renderer.color(view) + shape = (H, W) if is_depth else (H, W, 4) + out = np.empty(shape, dtype=np.float32 if is_depth else np.uint8) + rc = cuda.cuMemcpyDtoH_v2( + out.ctypes.data_as(ctypes.c_void_p), + ctypes.c_uint64(img.__cuda_array_interface__["data"][0]), + ctypes.c_size_t(out.nbytes), + ) + assert rc == 0, f"cuMemcpyDtoH_v2 -> {rc}" + return out + + def render(xr_offset, eye_separation=0.0): + """Park the ghost at `xr_offset` from the eye and draw both views.""" + data.mocap_pos[:] = _mujoco_xr.mj_from_xr_pos(xr_offset) + mujoco.mj_forward(model, data) + renderer.update_scene(model._address, data._address) + poses, fovs = [], [] + for sign in (-1.0, 1.0): + poses += [sign * eye_separation, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0] + fovs += [-HALF_FOV, HALF_FOV, HALF_FOV, -HALF_FOV] + renderer.render(poses, fovs) + + yield render, read + renderer.close() + gl.free() + + +def _drawn_centre(color): + """(row, col) centre of the drawn pixels; alpha 0 is 'show passthrough'.""" + drawn = color[..., 3] > 0 + assert drawn.any(), "nothing was drawn" + return np.flatnonzero(drawn.any(axis=1)).mean(), np.flatnonzero( + drawn.any(axis=0) + ).mean() + + +def test_something_is_drawn_at_all(rendered): + render, read = rendered + render([0.0, 0.0, -GHOST_DISTANCE]) + color = read(0, is_depth=False) + assert (color[..., 3] > 0).any(), ( + "the whole frame is transparent -- mjr_render drew into another framebuffer, " + "or the blit missed" + ) + assert set(np.unique(color[..., 3])) <= {0, 255}, ( + "alpha must be 0 (passthrough) or 255 (opaque); a partial alpha means blending " + "leaked into the readback pass" + ) + + +def test_row_zero_is_the_top_of_the_operators_view(rendered): + """The y flip: OpenGL renders bottom-up, XR swapchains are top-down, and + nothing short of a headset would show the whole scene upside down.""" + render, read = rendered + render([0.0, GHOST_OFFSET, -GHOST_DISTANCE]) + above, _ = _drawn_centre(read(0, is_depth=False)) + render([0.0, -GHOST_OFFSET, -GHOST_DISTANCE]) + below, _ = _drawn_centre(read(0, is_depth=False)) + + assert above < H / 2 < below, ( + f"XR +Y landed at row {above:.0f} and -Y at row {below:.0f}: the image is upside down" + ) + + +def test_the_image_is_not_mirrored(rendered): + """Horizontal, checked alongside the flip: mirroring one axis and not the + other is what a mistaken second flip looks like.""" + render, read = rendered + render([GHOST_OFFSET, 0.0, -GHOST_DISTANCE]) + _, right = _drawn_centre(read(0, is_depth=False)) + render([-GHOST_OFFSET, 0.0, -GHOST_DISTANCE]) + _, left = _drawn_centre(read(0, is_depth=False)) + + assert left < W / 2 < right, ( + f"XR +X landed at column {right:.0f} and -X at {left:.0f}: the image is mirrored" + ) + + +def test_depth_is_the_standard_z_projection_layer_is_promised(rendered): + """near -> 0, far -> 1, and the background is far. + + Getting it backwards leaves colour perfect and reprojection inverted, so the + values are checked against the geometry, not merely for being in range. + """ + render, read = rendered + render([0.0, 0.0, -GHOST_DISTANCE]) + depth = read(0, is_depth=True) + color = read(0, is_depth=False) + drawn = color[..., 3] > 0 + + background = np.unique(depth[~drawn]) + assert background == pytest.approx([1.0]), ( + f"background depth {background}, expected exactly 1.0 (far). MuJoCo clears its " + "reverse-Z buffer to 0, so anything else means the inversion is missing." + ) + + # The gripper is a few centimetres deep, so its depths must bracket the + # value for its centre. + expected = _mujoco_xr.submitted_depth(GHOST_DISTANCE, NEAR_Z, FAR_Z) + assert depth[drawn].min() < expected < depth[drawn].max(), ( + f"drawn depth spans [{depth[drawn].min():.4f}, {depth[drawn].max():.4f}], which does " + f"not bracket {expected:.4f} for a gripper at {GHOST_DISTANCE} m" + ) + + +def test_the_eyes_see_the_object_at_different_offsets(rendered): + """Stereo parallax and its sign: an object ahead sits further left in the + RIGHT eye, and swapping the views reads as eye strain rather than as a bug.""" + render, read = rendered + render([0.0, 0.0, -GHOST_DISTANCE], eye_separation=0.032) + _, left_eye = _drawn_centre(read(0, is_depth=False)) + _, right_eye = _drawn_centre(read(1, is_depth=False)) + + assert right_eye < left_eye, ( + f"left eye sees the object at column {left_eye:.0f} and the right eye at " + f"{right_eye:.0f}: the views are swapped" + ) diff --git a/rigs/mujoco_xr.yaml b/rigs/mujoco_xr.yaml new file mode 100644 index 000000000..22c806e00 --- /dev/null +++ b/rigs/mujoco_xr.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Run with: python -m isaacteleop.rig rigs/mujoco_xr.yaml +# +# Two panes, one declared: with no `runtime:` key the runtime pane comes from +# DEFAULT_RUNTIME_COMMAND. No producers either -- the app opens the OpenXR +# session itself and reads controllers straight from the runtime, so there is no +# rendezvous and deliberately no `params:` / `collection_id:`. +# +# `{python}` expands to the launching interpreter, so both wheels must be +# installed there rather than in a private .venv: +# +# uv pip install "isaacteleop[cloudxr]" --find-links=./install/wheels/ --reinstall +# uv pip install ./examples/mujoco_xr # same environment +# +# See rigs/se3_tracker.yaml for the fully annotated exemplar of every key. +name: mujoco_xr +description: CloudXR runtime + MuJoCo scene in XR with the SO-101 leader gripper +cwd: .. # -> Teleop repo root +consumers: + - name: mujoco xr app (requires headset) + # Keep --no-launch-cloudxr-runtime. The runtime is a host singleton on WSS + # port 48322, so a second one kills this rig's own pane: the headset drops + # mid-session and reads as a runtime crash rather than a config edit. + # find_runtime_footguns() warns but never gates. + command: "{python} -m isaacteleop_examples.mujoco_xr --no-launch-cloudxr-runtime"