Robot twin: tests

.github/workflows/build-ubuntu.yml

@@ -18,7 +18,7 @@ concurrency:
 env:
   # Shared apt build dependencies. Each job installs these plus its own extras
   # (ccache, clang-format, the OAK camera autotools chain).
-  CORE_APT_DEPS: build-essential cmake glslang-tools libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols
+  CORE_APT_DEPS: build-essential cmake glslang-tools libegl-dev libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols
   # Pin vcpkg to an immutable commit for reproducible, supply-chain-stable CI
   # instead of tracking its moving default branch. Port versions are fixed by the
   # consuming manifest's builtin-baseline (e.g. DepthAI's vcpkg.json), so this

.gitignore

@@ -73,11 +73,7 @@ deps/v2d/wheels/
 # SO-101 leader-gripper and follower-arm assets, fetched by
 # scripts/fetch-so-arm.sh into the package's assets directory, so only the
 # authored wrapper XML is tracked.
-#
-# Keep this rule HERE, not in examples/mujoco_xr/.gitignore: scikit-build-core
-# resolves .gitignore against the project root, so a rule there would strip the
-# meshes out of the wheel too.
-/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/*
-!/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml
-/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/*
-!/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/follower_arm.xml
+/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/leader/*
+!/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/leader/leader_gripper.xml
+/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/follower/*
+!/examples/robot_viz/python/isaacteleop_examples/robot_viz/assets/follower/follower_arm.xml

CMakeLists.txt

@@ -172,7 +172,14 @@ if(BUILD_EXAMPLES)
     add_subdirectory(examples/haptic_feedback)
     if(BUILD_VIZ)
         add_subdirectory(examples/camera_viz/tests)
-        add_subdirectory(examples/mujoco_xr)
+    endif()
+    # Pure Python, so this registers tests and nothing else -- but all three of them
+    # import SceneTwin, which needs the compiled twin. Keyed on the target rather than on
+    # BUILD_VIZ: src/viz owns whether the twin is built, and restating that here would be
+    # a second copy to keep in sync. src/viz is processed above, so the target exists by
+    # now if it is going to.
+    if(TARGET robot_twin_py)
+        add_subdirectory(examples/robot_viz)
     endif()
 elseif(BUILD_EXAMPLE_TELEOP_ROS2)
     add_subdirectory(examples/teleop_ros2)

cmake/CheckBuildDeps.cmake

@@ -65,6 +65,17 @@ function(isaac_teleop_check_build_deps)
         endif()
     endif()
 
+    # EGL's headers, for the robot twin's headless OpenGL context. This function has
+    # already returned on anything but Linux, so BUILD_VIZ is the whole gate. Only the
+    # headers: gl_context.cpp dlopens libEGL, so the wheel carries no NEEDED entry.
+    if(BUILD_VIZ)
+        find_path(EGL_INCLUDE_DIR "EGL/eglext.h")
+        if(NOT EGL_INCLUDE_DIR)
+            list(APPEND _missing_tools "EGL/eglext.h     (BUILD_VIZ=ON -- the robot twin's headless OpenGL context)")
+            list(APPEND _missing_pkgs "libegl-dev")
+        endif()
+    endif()
+
     # patchelf strips the spurious libssl.so.3 NEEDED entry from libcloudxr.so
     # (src/core/cloudxr/python/CMakeLists.txt). Checked unconditionally: the SDK
     # tarball that triggers it is downloaded *during* configure, so whether it

deps/third_party/CMakeLists.txt

@@ -202,3 +202,49 @@ if(BUILD_VIZ)
     FetchContent_MakeAvailable(glfw)
     message(STATUS "GLFW 3.4 fetched")
 endif()
+
+# ==============================================================================
+# MuJoCo (robot twin physics + renderer)
+# ==============================================================================
+# Built from upstream sources unmodified and shipped under a private name, so the robot
+# twin's MuJoCo is an implementation detail nothing else in the process can see.
+# Mujoco.cmake beside this file is that contract and the function consumers reach it
+# through; src/viz/robot_twin is the only consumer.
+if(BUILD_VIZ)
+    # Cached because src/viz/robot_twin_tests reads it to tell our copy from the wheel's,
+    # and that is a sibling of this directory -- plain variables only reach downward. The
+    # version the twin is built against, unrelated to whatever `mujoco` wheel the user
+    # has, which is the whole point; bumping it is an isaacteleop release decision.
+    set(ISAACTELEOP_MUJOCO_VERSION 3.11.0
+        CACHE INTERNAL "MuJoCo version the robot twin is built against")
+
+    message(STATUS "Fetching MuJoCo ${ISAACTELEOP_MUJOCO_VERSION} from GitHub...")
+    FetchContent_Declare(
+        mujoco
+        GIT_REPOSITORY https://github.com/google-deepmind/mujoco.git
+        GIT_TAG        ${ISAACTELEOP_MUJOCO_VERSION}
+        # The ref is a tag, so it is fetchable without history. Measured here: 140 MB of
+        # .git without this, ~90 MB with. FALSE would be needed only for a raw commit
+        # SHA, which is why the entries above set it that way.
+        #
+        # Not the order-of-magnitude win it looks like, because CMake spells GIT_SHALLOW
+        # as `--depth 1 --no-single-branch` -- depth 1 of EVERY branch. A release tarball
+        # (URL + URL_HASH) would carry no history at all; it is the bigger change and has
+        # not been measured on a fast link.
+        GIT_SHALLOW    TRUE
+    )
+
+    # Plain variables, not the `CACHE ... FORCE` the entries above use: MuJoCo sets
+    # CMAKE_POLICY_DEFAULT_CMP0077 NEW before its own project(), so a normal variable
+    # wins over its option() and leaves no knob in the cache. Turning the tests off is
+    # what keeps abseil, googletest and benchmark out of _deps.
+    set(MUJOCO_BUILD_EXAMPLES OFF)
+    set(MUJOCO_BUILD_SIMULATE OFF)
+    set(MUJOCO_BUILD_TESTS OFF)
+    set(MUJOCO_TEST_PYTHON_UTIL OFF)
+
+    FetchContent_MakeAvailable(mujoco)
+    message(STATUS "MuJoCo ${ISAACTELEOP_MUJOCO_VERSION} fetched")
+
+    include("${CMAKE_CURRENT_LIST_DIR}/Mujoco.cmake")
+endif()

deps/third_party/Mujoco.cmake

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Makes the MuJoCo fetched by deps/third_party/CMakeLists.txt private, so a user may
+# `pip install mujoco` at any version, or none, and never collide with ours. That wheel is
+# not a dependency of this path. Three things hold it up, none of them optional:
+#
+#   OUTPUT_NAME   a private SONAME. The mujoco wheel's extension modules carry a
+#                 DT_NEEDED on libmujoco.so.3.x, and the loader satisfies that from
+#                 whatever is already loaded under that SONAME -- so an unrenamed copy
+#                 of ours, loaded first, answers the user's own `import mujoco`.
+#   -Bsymbolic    libmujoco's cross-references bind to itself rather than to whatever
+#                 copy is in the global scope. Not -Bsymbolic-functions: mju_user_error
+#                 and mju_user_warning are data, and libmujoco reads them.
+#   dlopen/dlsym  src/viz/robot_twin/cpp/mj_api.cpp resolves MuJoCo at import instead of
+#                 linking it, so the extension has no undefined mj* symbol for a foreign
+#                 libmujoco to answer.
+#
+# src/viz/robot_twin_tests/test_symbol_isolation.py asserts all three. Do not replace the
+# dlopen with a plain link: an undefined mj* resolves through the global scope, which is
+# searched first, and the wrong libmujoco answering is silent -- no error, no version
+# warning, just mjModel laid out one way and read another.
+
+# Upstream's own install rules come with the subdirectory and stay; suppressing them would
+# mean patching. They cannot reach the wheel -- pyproject.toml's install.components names
+# only isaacteleop_wheel and isaacteleop_binaries.
+
+set_target_properties(mujoco PROPERTIES OUTPUT_NAME isaacteleop_mujoco)
+# Upstream's VERSION would make libisaacteleop_mujoco.so a symlink to ...so.3.11.0, and
+# wheels do not carry symlinks. Unset with no value, which REMOVES the property; "" leaves
+# it set and names the library `libisaacteleop_mujoco.so.`.
+set_property(TARGET mujoco PROPERTY VERSION)
+set_property(TARGET mujoco PROPERTY SOVERSION)
+set_property(TARGET mujoco APPEND PROPERTY LINK_OPTIONS "-Wl,-Bsymbolic")
+
+# Set up an extension that reaches MuJoCo through mj_api.cpp: headers to compile against,
+# libisaacteleop_mujoco.so staged beside the module for the dlopen to find, and a dynamic
+# symbol table holding nothing but the entry point. `module_name` is the importable name,
+# whose PyInit_ symbol is the one export.
+function(isaacteleop_link_mujoco target module_name)
+    # Headers only. Nothing links MuJoCo, so add_dependencies supplies the build order a
+    # link line would otherwise have implied.
+    target_include_directories(${target} PRIVATE
+        $<TARGET_PROPERTY:mujoco,INTERFACE_INCLUDE_DIRECTORIES>)
+    add_dependencies(${target} mujoco)
+
+    # mj_api.cpp opens it by this module's own directory, so the copy has to be there --
+    # in the build tree for ctest, and in the staged python_package that install(DIRECTORY)
+    # turns into the wheel.
+    add_custom_command(TARGET ${target} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                "$<TARGET_FILE:mujoco>" "$<TARGET_FILE_DIR:${target}>"
+        COMMENT "Staging $<TARGET_FILE_NAME:mujoco> beside $<TARGET_FILE_NAME:${target}>")
+
+    # A whitelist, so it covers what pybind11's -fvisibility=hidden misses: symbols from
+    # static archives (cudart_static) and the typeinfo pybind11 emits for the mjt* enums.
+    # Measured on robot_twin_py: 1 export, against 11 for -Wl,--exclude-libs,ALL alone.
+    set(_version_script "${CMAKE_CURRENT_BINARY_DIR}/${target}_exports.map")
+    file(GENERATE OUTPUT "${_version_script}"
+         CONTENT "{ global: PyInit_${module_name}; local: *; };\n")
+    target_link_options(${target} PRIVATE "-Wl,--version-script,${_version_script}")
+    set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS "${_version_script}")
+endfunction()

docs/source/getting_started/build_from_source/index.rst

@@ -47,6 +47,14 @@ Prerequisites
    ``pip install isaacteleop`` already ships the compiled ``isaacteleop.viz`` module. See
    `Other Build options`_ for the full option table.
 
+   On **Linux**, ``BUILD_VIZ=ON`` also builds the robot twin's scene backend
+   (:class:`isaacteleop.viz.robot.SceneTwin`). There is no separate flag. It compiles
+   MuJoCo from source and ships it under a private name, so it is unrelated to whatever
+   ``mujoco`` the environment has -- any version, or none. It needs **EGL's
+   headers** (``libegl-dev``) for its headless OpenGL context, fetches MuJoCo and six
+   vendored dependencies from GitHub on the first configure, and costs about 40 s of
+   build time on 12 cores. A Windows Televiz build omits it: the headless context is EGL.
+
 .. _one-time-setup:
 
 One time setup
@@ -58,7 +66,7 @@ the list of dependencies. On **Ubuntu**, install build tools and clang-format:
 .. code-block:: bash
 
    sudo apt-get update
-   sudo apt-get install -y build-essential cmake libx11-dev libwayland-dev clang-format-14 ccache patchelf pkg-config glslang-tools
+   sudo apt-get install -y build-essential cmake libegl-dev libx11-dev libwayland-dev clang-format-14 ccache patchelf pkg-config glslang-tools
 
 Runtime-only dependencies (needed to actually run teleop, not to build):
 
@@ -236,7 +244,7 @@ The CMake options (defined in root :code-file:`CMakeLists.txt` and :code-file:`c
      - ``ON`` on Linux
    * - **Televiz visualization**
      - ``BUILD_VIZ``
-     - Auto: ``ON`` when Vulkan and the CUDA Toolkit are detected, else ``OFF``. Force with ``-DBUILD_VIZ=ON`` / ``-DBUILD_VIZ=OFF``. (Most users don't need this — ``pip install isaacteleop`` already ships the compiled ``isaacteleop.viz`` module.)
+     - Auto: ``ON`` when Vulkan and the CUDA Toolkit are detected, else ``OFF``. Force with ``-DBUILD_VIZ=ON`` / ``-DBUILD_VIZ=OFF``. On Linux it also builds the robot twin's scene backend, which needs ``libegl-dev`` and fetches MuJoCo on the first configure. (Most users don't need this — ``pip install isaacteleop`` already ships the compiled ``isaacteleop.viz`` module.)
 
 .. list-table:: Plugin Specific Options
    :widths: 26 34 40

docs/source/references/cloudxr.rst

@@ -200,7 +200,7 @@ accept ``--no-launch-cloudxr-runtime``. That returns a :class:`~isaacteleop.clou
 the process does not start or attach to CloudXR and leaves ``XR_RUNTIME_JSON`` and
 related environment variables unchanged. Use this when another runtime is already
 configured (for example Monado) or when a host singleton must not be duplicated
-(see ``examples/mujoco_xr/README.md`` and ``--no-launch-cloudxr-runtime`` there).
+(see ``examples/robot_viz/README.md`` and ``--no-launch-cloudxr-runtime`` there).
 
 .. code-block:: bash
 

docs/source/references/retargeting/so101.rst

@@ -152,7 +152,15 @@ fires against *that* frame's controller pose, not the one it was first squeezed
 ``ValueInput``-leaf rule as ``MEASURED_BASE_T_EE_INPUT`` applies: a producer that wires it must
 send the key on every step.
 
-The motivating consumer is ``examples/mujoco_xr``, whose owner shows the operator an SO-101
+:class:`~isaacteleop.retargeters.EngageAlignmentGate` is the shipped producer: a graph node that
+compares the controller's orientation against a reference pose the owner supplies, with hysteresis
+and a dwell, and emits straight into this input. Because the graph is a pull-based DAG it cannot
+read the clutch it feeds, so the owner hands back one bool -- ``is_engaged``, as of the previous
+frame -- on :data:`~isaacteleop.retargeters.EngageAlignmentGate.ENGAGED_INPUT`. That disjunction is
+load-bearing: during a tracking dropout the clutch disarms and owes a latch on recovery, and a gate
+answering "not aligned" there would stop the arm mid-engagement.
+
+The motivating consumer is ``examples/robot_viz``, whose owner shows the operator an SO-101
 follower before engaging and withholds permission until the operator's wrist is turned the way
 that arm is -- see that example's README.