Jiwenc nv/robot viz

.github/workflows/build-ubuntu.yml

@@ -18,7 +18,7 @@ concurrency:
 env:
   # Shared apt build dependencies. Each job installs these plus its own extras
   # (ccache, clang-format, the OAK camera autotools chain).
-  CORE_APT_DEPS: build-essential cmake glslang-tools libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols
+  CORE_APT_DEPS: build-essential cmake glslang-tools libegl-dev libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols
   # Pin vcpkg to an immutable commit for reproducible, supply-chain-stable CI
   # instead of tracking its moving default branch. Port versions are fixed by the
   # consuming manifest's builtin-baseline (e.g. DepthAI's vcpkg.json), so this

.gitignore

@@ -73,11 +73,7 @@ deps/v2d/wheels/
 # SO-101 leader-gripper and follower-arm assets, fetched by
 # scripts/fetch-so-arm.sh into the package's assets directory, so only the
 # authored wrapper XML is tracked.
-#
-# Keep this rule HERE, not in examples/mujoco_xr/.gitignore: scikit-build-core
-# resolves .gitignore against the project root, so a rule there would strip the
-# meshes out of the wheel too.
-/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/*
-!/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml
-/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/*
-!/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/follower_arm.xml
+/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/leader/*
+!/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/leader/leader_gripper.xml
+/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/follower/*
+!/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/follower/follower_arm.xml

CMakeLists.txt

@@ -172,7 +172,6 @@ if(BUILD_EXAMPLES)
     add_subdirectory(examples/haptic_feedback)
     if(BUILD_VIZ)
         add_subdirectory(examples/camera_viz/tests)
-        add_subdirectory(examples/mujoco_xr)
     endif()
 elseif(BUILD_EXAMPLE_TELEOP_ROS2)
     add_subdirectory(examples/teleop_ros2)

cmake/CheckBuildDeps.cmake

@@ -65,6 +65,17 @@ function(isaac_teleop_check_build_deps)
         endif()
     endif()
 
+    # EGL's headers, for the robot twin's headless OpenGL context. This function has
+    # already returned on anything but Linux, so BUILD_VIZ is the whole gate. Only the
+    # headers: gl_context.cpp dlopens libEGL, so the wheel carries no NEEDED entry.
+    if(BUILD_VIZ)
+        find_path(EGL_INCLUDE_DIR "EGL/eglext.h")
+        if(NOT EGL_INCLUDE_DIR)
+            list(APPEND _missing_tools "EGL/eglext.h     (BUILD_VIZ=ON -- the robot twin's headless OpenGL context)")
+            list(APPEND _missing_pkgs "libegl-dev")
+        endif()
+    endif()
+
     # patchelf strips the spurious libssl.so.3 NEEDED entry from libcloudxr.so
     # (src/core/cloudxr/python/CMakeLists.txt). Checked unconditionally: the SDK
     # tarball that triggers it is downloaded *during* configure, so whether it

deps/third_party/CMakeLists.txt

@@ -202,3 +202,38 @@ if(BUILD_VIZ)
     FetchContent_MakeAvailable(glfw)
     message(STATUS "GLFW 3.4 fetched")
 endif()
+
+# ==============================================================================
+# MuJoCo (robot twin scene + renderer)
+# ==============================================================================
+# Built from upstream sources unmodified and shipped under a private name, so the robot
+# twin's MuJoCo is an implementation detail nothing else in the process can see.
+# Mujoco.cmake beside this file is that contract and the function consumers reach it
+# through; src/viz/robot_twin is the only consumer.
+#
+# Upstream builds one library -- engine, renderer, MJCF parser -- with no renderer-only
+# target, so this is the whole of it even though the twin calls no dynamics.
+if(BUILD_VIZ)
+    message(STATUS "Fetching MuJoCo from GitHub...")
+    FetchContent_Declare(
+        mujoco
+        GIT_REPOSITORY https://github.com/google-deepmind/mujoco.git
+        GIT_TAG        3.11.0
+        GIT_SHALLOW    TRUE
+    )
+
+    # Before MakeAvailable, which is what runs MuJoCo's own CMakeLists -- so these cannot
+    # move into Mujoco.cmake below. Plain variables, not the `CACHE ... FORCE` the entries
+    # above use: MuJoCo sets CMAKE_POLICY_DEFAULT_CMP0077 NEW before its own project(), so
+    # a normal variable wins over its option() and leaves no knob in the cache. Turning the
+    # tests off is what keeps abseil, googletest and benchmark out of _deps.
+    set(MUJOCO_BUILD_EXAMPLES OFF)
+    set(MUJOCO_BUILD_SIMULATE OFF)
+    set(MUJOCO_BUILD_TESTS OFF)
+    set(MUJOCO_TEST_PYTHON_UTIL OFF)
+
+    FetchContent_MakeAvailable(mujoco)
+    message(STATUS "MuJoCo 3.11.0 fetched")
+
+    include("${CMAKE_CURRENT_LIST_DIR}/Mujoco.cmake")
+endif()

deps/third_party/Mujoco.cmake

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Makes the MuJoCo fetched beside this file private, so a user may `pip install mujoco` at
+# any version, or none, and never collide with ours. Three things hold that up, none
+# optional:
+#
+#   OUTPUT_NAME   a private SONAME. The mujoco wheel's extensions carry a DT_NEEDED on
+#                 libmujoco.so.3.x, and the loader satisfies it from whatever is already
+#                 loaded under that SONAME -- so an unrenamed copy of ours, loaded first,
+#                 answers the user's own `import mujoco`.
+#   -Bsymbolic    libmujoco's cross-references bind to itself rather than to whatever copy
+#                 is in the global scope. Not -Bsymbolic-functions: mju_user_error and
+#                 mju_user_warning are data, and libmujoco reads them.
+#   dlopen/dlsym  src/viz/robot_twin/cpp/mj_api.cpp resolves MuJoCo at import instead of
+#                 linking it, so the extension has no undefined mj* for a foreign
+#                 libmujoco to answer.
+#
+# Do not replace the dlopen with a plain link: an undefined mj* resolves through the
+# global scope, which is searched first, and the wrong libmujoco answering is silent --
+# no error, no version warning, just mjModel laid out one way and read another.
+
+# Upstream's own install rules come with the subdirectory and stay; suppressing them would
+# mean patching. They cannot reach the wheel -- pyproject.toml's install.components names
+# only isaacteleop_wheel and isaacteleop_binaries.
+
+set_target_properties(mujoco PROPERTIES OUTPUT_NAME isaacteleop_mujoco)
+# Upstream's VERSION would make libisaacteleop_mujoco.so a symlink to ...so.3.11.0, and
+# wheels do not carry symlinks. Unset with no value, which REMOVES the property; "" leaves
+# it set and names the library `libisaacteleop_mujoco.so.`.
+set_property(TARGET mujoco PROPERTY VERSION)
+set_property(TARGET mujoco PROPERTY SOVERSION)
+set_property(TARGET mujoco APPEND PROPERTY LINK_OPTIONS "-Wl,-Bsymbolic")
+
+# Set up an extension that reaches MuJoCo through mj_api.cpp: headers to compile against,
+# libisaacteleop_mujoco.so staged beside the module for the dlopen to find, and a dynamic
+# symbol table holding nothing but the entry point. `module_name` is the importable name,
+# whose PyInit_ symbol is the one export.
+function(isaacteleop_link_mujoco target module_name)
+    # Headers only. Nothing links MuJoCo, so add_dependencies supplies the build order a
+    # link line would otherwise have implied.
+    target_include_directories(${target} PRIVATE
+        $<TARGET_PROPERTY:mujoco,INTERFACE_INCLUDE_DIRECTORIES>)
+    add_dependencies(${target} mujoco)
+
+    # mj_api.cpp opens it by this module's own directory, so the copy has to be there --
+    # in the build tree for ctest, and in the staged python_package that install(DIRECTORY)
+    # turns into the wheel.
+    add_custom_command(TARGET ${target} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                "$<TARGET_FILE:mujoco>" "$<TARGET_FILE_DIR:${target}>"
+        COMMENT "Staging $<TARGET_FILE_NAME:mujoco> beside $<TARGET_FILE_NAME:${target}>")
+
+    # A whitelist, so it covers what pybind11's -fvisibility=hidden misses: symbols from
+    # static archives (cudart_static) and the typeinfo pybind11 emits for the mjt* enums.
+    # Measured on robot_twin_py: 1 export, against 11 for -Wl,--exclude-libs,ALL alone.
+    set(_version_script "${CMAKE_CURRENT_BINARY_DIR}/${target}_exports.map")
+    file(GENERATE OUTPUT "${_version_script}"
+         CONTENT "{ global: PyInit_${module_name}; local: *; };\n")
+    target_link_options(${target} PRIVATE "-Wl,--version-script,${_version_script}")
+    set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS "${_version_script}")
+endfunction()

docs/source/getting_started/build_from_source/index.rst

@@ -58,7 +58,7 @@ the list of dependencies. On **Ubuntu**, install build tools and clang-format:
 .. code-block:: bash
 
    sudo apt-get update
-   sudo apt-get install -y build-essential cmake libx11-dev libwayland-dev clang-format-14 ccache patchelf pkg-config glslang-tools
+   sudo apt-get install -y build-essential cmake libegl-dev libx11-dev libwayland-dev clang-format-14 ccache patchelf pkg-config glslang-tools
 
 Runtime-only dependencies (needed to actually run teleop, not to build):
 

docs/source/references/cloudxr.rst

@@ -200,7 +200,7 @@ accept ``--no-launch-cloudxr-runtime``. That returns a :class:`~isaacteleop.clou
 the process does not start or attach to CloudXR and leaves ``XR_RUNTIME_JSON`` and
 related environment variables unchanged. Use this when another runtime is already
 configured (for example Monado) or when a host singleton must not be duplicated
-(see ``examples/mujoco_xr/README.md`` and ``--no-launch-cloudxr-runtime`` there).
+(see ``examples/robot_viz/README.md`` and ``--no-launch-cloudxr-runtime`` there).
 
 .. code-block:: bash
 

docs/source/references/retargeting/so101.rst

@@ -152,7 +152,7 @@ fires against *that* frame's controller pose, not the one it was first squeezed
 ``ValueInput``-leaf rule as ``MEASURED_BASE_T_EE_INPUT`` applies: a producer that wires it must
 send the key on every step.
 
-The motivating consumer is ``examples/mujoco_xr``, whose owner shows the operator an SO-101
+The motivating consumer is ``examples/robot_viz``, whose owner shows the operator an SO-101
 follower before engaging and withholds permission until the operator's wrist is turned the way
 that arm is -- see that example's README.