From 7aff35415ee3a75df5dc49e059b4da8b04dae286 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Tue, 18 Aug 2026 13:45:38 +0000 Subject: [PATCH] Keep topology discovery CUDA-context agnostic Topology discovery must be safe to use independently of CUDA runtime and context state. It should not initialize CUDA, require an active context, create a context, or otherwise change CUDA process state. This change restores that invariant that was previously modified in https://github.com/NVIDIA/cuCascade/pull/167 and documents it as part of the topology-discovery API contract. It also adds GPU coverage to guard the behavior going forward. --- docs/topology-and-configuration.md | 8 +++ .../cucascade/memory/topology_discovery.hpp | 4 ++ src/memory/topology_discovery.cpp | 29 ++++------ test/CMakeLists.txt | 6 ++- test/memory/test_topology_discovery.cpp | 53 +++++++++++++++++++ 5 files changed, 80 insertions(+), 20 deletions(-) diff --git a/docs/topology-and-configuration.md b/docs/topology-and-configuration.md index 7ddc7d8..5350b76 100644 --- a/docs/topology-and-configuration.md +++ b/docs/topology-and-configuration.md @@ -8,6 +8,7 @@ Hardware topology discovery, NUMA-aware memory placement, and system configurati - [Hardware Topology Discovery](#hardware-topology-discovery) - [What Gets Detected](#what-gets-detected) - [How Detection Works](#how-detection-works) + - [CUDA Context Independence](#cuda-context-independence) - [GPU-to-NUMA Affinity](#gpu-to-numa-affinity) - [PCIe Path Types](#pcie-path-types) - [Configuring cuCascade](#configuring-cucascade) @@ -105,6 +106,13 @@ throws rather than sizing a host space from device memory. 5. **Network devices** -- scans `/sys/class/infiniband/` for NICs with NUMA info, with configurable verification (see [Network Device Verification](#network-device-verification)) 6. **Storage devices** -- scans `/sys/block/` for NVMe and SATA devices with NUMA info +### CUDA Context Independence + +Topology discovery is independent of CUDA runtime and context state. `discover()` may be called +before or after CUDA initialization, but it must never initialize CUDA, create or require a CUDA +context, or otherwise alter CUDA process state. This is a public API invariant, not merely an +implementation detail. + ### Network Device Verification The `discover()` method accepts a `NetworkDeviceVerification` parameter that controls how strictly network devices are validated before inclusion. This prevents assigning non-functional devices to `UCX_NET_DEVICES` (which would cause connection failures). diff --git a/include/cucascade/memory/topology_discovery.hpp b/include/cucascade/memory/topology_discovery.hpp index a6057db..e21ad1d 100644 --- a/include/cucascade/memory/topology_discovery.hpp +++ b/include/cucascade/memory/topology_discovery.hpp @@ -192,6 +192,10 @@ class topology_discovery { * This method performs the actual discovery of GPUs, NUMA nodes, CPU affinity, * and network devices. It must be called before `get_topology()`. * + * @note CUDA-context invariant: this method is CUDA runtime and context agnostic. + * It must not initialize CUDA, create a CUDA context, require an active CUDA context, + * or otherwise alter CUDA process state. + * * @param net_verification Controls how strictly network devices are validated. * @return true if discovery was successful, false otherwise. */ diff --git a/src/memory/topology_discovery.cpp b/src/memory/topology_discovery.cpp index c044570..c22aab3 100644 --- a/src/memory/topology_discovery.cpp +++ b/src/memory/topology_discovery.cpp @@ -5,9 +5,6 @@ #include -#include -#include - #include #include #include @@ -63,23 +60,18 @@ void report_nvml_error(nvmlReturn_t result, std::string const& context) /** * @brief Query whether a CUDA device supports hardware-accelerated decompression. * - * Delegates to `rmm::detail::hwdecompress::is_supported()`, which checks the CUDA - * driver version. RMM's capability queries are scoped to the current device, so the - * call is wrapped in an `rmm::cuda_set_device_raii`. Best-effort: any failure while - * setting the device or probing yields false. + * Hardware decompression requires CUDA driver 12.8 or newer. Use NVML rather than + * the CUDA runtime so topology discovery neither initializes CUDA nor observes the + * process's CUDA_VISIBLE_DEVICES setting. * - * @param cuda_ordinal CUDA device ordinal (matches the runtime device index used - * elsewhere in discovery under the same CUDA_VISIBLE_DEVICES ordering). * @return true iff the hardware decompression engine is available. */ -bool query_hw_decompression(unsigned int cuda_ordinal) +bool query_hw_decompression() { - try { - rmm::cuda_set_device_raii set_device{rmm::cuda_device_id{static_cast(cuda_ordinal)}}; - return rmm::detail::hwdecompress::is_supported(); - } catch (...) { - return false; - } + constexpr int min_hw_decompression_cuda_version = 12080; + int driver_version = 0; + return nvmlSystemGetCudaDriverVersion(&driver_version) == NVML_SUCCESS && + driver_version >= min_hw_decompression_cuda_version; } /** @@ -991,13 +983,14 @@ bool topology_discovery::discover(NetworkDeviceVerification net_verification) auto visible_indices = resolve_visible_gpu_indices(nvml_gpus, nvml_index_by_pci, nvml_index_by_uuid); - topology.num_gpus = static_cast(visible_indices.size()); + auto const hw_decompression_available = query_hw_decompression(); + topology.num_gpus = static_cast(visible_indices.size()); for (size_t visible_idx = 0; visible_idx < visible_indices.size(); ++visible_idx) { size_t nvml_idx = visible_indices[visible_idx]; if (nvml_idx >= nvml_gpus.size()) { continue; } auto gpu = nvml_gpus[nvml_idx]; gpu.id = static_cast(visible_idx); - gpu.hw_decompression_available = query_hw_decompression(gpu.id); + gpu.hw_decompression_available = hw_decompression_available; topology.gpus.push_back(std::move(gpu)); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 961696b..a7ed81f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -132,8 +132,10 @@ target_include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/utils) # Link dependencies -target_link_libraries(cucascade_topology_discovery_tests - PRIVATE cucascade_topology_discovery Catch2::Catch2) +target_link_libraries( + cucascade_topology_discovery_tests + PRIVATE cucascade_topology_discovery Catch2::Catch2 CUDA::cuda_driver + CUDA::cudart CUDA::nvml_static) # Register tests with CTest add_test(NAME cucascade_topology_discovery_tests diff --git a/test/memory/test_topology_discovery.cpp b/test/memory/test_topology_discovery.cpp index b68a496..f1f0add 100644 --- a/test/memory/test_topology_discovery.cpp +++ b/test/memory/test_topology_discovery.cpp @@ -31,7 +31,13 @@ #include +#include +#include + #include +#include +#include +#include #include #include @@ -290,3 +296,50 @@ TEST_CASE("Topology Discovery rejects overflow CUDA_VISIBLE_DEVICES", "[hw_topol REQUIRE_THROWS_WITH(discovery.discover(), "Invalid numeric CUDA_VISIBLE_DEVICES entry: 999999999999999999999999999999"); } + +TEST_CASE("Topology Discovery does not initialize CUDA while visibility is widened", + "[hw_topology][cuda-context]") +{ + auto const child = fork(); + REQUIRE(child >= 0); + + if (child == 0) { + // NVML is used only to make this test meaningful on a multi-GPU host. It + // does not create a CUDA context or initialize the CUDA runtime. + if (nvmlInit_v2() != NVML_SUCCESS) { _exit(77); } + unsigned int device_count = 0; + auto const count_status = nvmlDeviceGetCount_v2(&device_count); + nvmlShutdown(); + if (count_status != NVML_SUCCESS || device_count < 2) { _exit(77); } + + // This mirrors callers that temporarily widen visibility to collect + // physical topology, then restore the process's assigned GPU. If + // discover() initializes the CUDA runtime while CVD is unset, the runtime + // permanently sees all GPUs and the final cudaGetDeviceCount() is wrong. + setenv("CUDA_VISIBLE_DEVICES", "1", 1); + unsetenv("CUDA_VISIBLE_DEVICES"); + topology_discovery discovery; + bool const discovered = discovery.discover(); + setenv("CUDA_VISIBLE_DEVICES", "1", 1); + if (!discovered) { _exit(1); } + + CUcontext context = nullptr; + if (cuInit(0) != CUDA_SUCCESS || cuCtxGetCurrent(&context) != CUDA_SUCCESS || + context != nullptr) { + _exit(1); + } + + int visible_count = 0; + if (cudaGetDeviceCount(&visible_count) != cudaSuccess || visible_count != 1) { _exit(1); } + _exit(0); + } + + int status = 0; + REQUIRE(waitpid(child, &status, 0) == child); + if (WIFEXITED(status) && WEXITSTATUS(status) == 77) { + SUCCEED("Skipped: requires NVML and at least two visible GPUs"); + return; + } + REQUIRE(WIFEXITED(status)); + REQUIRE(WEXITSTATUS(status) == 0); +}