From 522e3c6bdea8b55842155e67a15c64e4e3eda1b3 Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Wed, 29 Jul 2026 20:15:23 +0000 Subject: [PATCH] Add nvmlDeviceGetGpuFabricInfoV. --- nvml-wrapper/src/device.rs | 32 ++++++ nvml-wrapper/src/enums/device.rs | 124 +++++++++++++++++++++ nvml-wrapper/src/struct_wrappers/device.rs | 104 ++++++++++++++++- nvml-wrapper/unwrapped_functions.txt | 1 - 4 files changed, 259 insertions(+), 2 deletions(-) diff --git a/nvml-wrapper/src/device.rs b/nvml-wrapper/src/device.rs index 6db3abb..6844539 100644 --- a/nvml-wrapper/src/device.rs +++ b/nvml-wrapper/src/device.rs @@ -4508,6 +4508,38 @@ impl<'nvml> Device<'nvml> { P2pStatus::try_from(status_c) } + /** + Gets the NVLink fabric registration info for this [`Device`]. + + On multi-node NVLink systems, this reports the NVLink fabric UUID, the clique + id, fabric health, and the state of the device's registration with the fabric. + Devices that are not part of an NVLink fabric report + [`GpuFabricState::NotSupported`](crate::enums::device::GpuFabricState::NotSupported). + + # Errors + + * `Uninitialized`, if the library has not been successfully initialized + * `InvalidArg`, if this `Device` is invalid + * `NotSupported`, if this `Device` does not support this driver call + * `Unknown`, on any unexpected error + */ + #[doc(alias = "nvmlDeviceGetGpuFabricInfoV")] + pub fn gpu_fabric_info(&self) -> Result { + let sym = nvml_sym(self.nvml.lib.nvmlDeviceGetGpuFabricInfoV.as_ref())?; + + let info = unsafe { + let mut info: nvmlGpuFabricInfoV_t = mem::zeroed(); + // Implements NVML_STRUCT_VERSION(GpuFabricInfo, 3), as detailed in nvml.h + info.version = + (mem::size_of::() | (3_usize << 24_usize)) as u32; + nvml_try(sym(self.device, &mut info))?; + + info + }; + + Ok(GpuFabricInfo::from(info)) + } + /** Gets the power source of this [`Device`]. diff --git a/nvml-wrapper/src/enums/device.rs b/nvml-wrapper/src/enums/device.rs index a60f6b4..784804e 100644 --- a/nvml-wrapper/src/enums/device.rs +++ b/nvml-wrapper/src/enums/device.rs @@ -427,3 +427,127 @@ impl TryFrom for FanControlPolicy { } } } + +/// The state of a GPU's registration with an NVLink fabric. +/// +/// This mirrors `nvmlGpuFabricState_t` in a Rust enum. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub enum GpuFabricState { + /// The GPU is not part of an NVLink fabric. + NotSupported, + /// Fabric registration has not started. + NotStarted, + /// Fabric registration is in progress. + InProgress, + /// Fabric registration has completed. [GpuFabricState::Completed] does not imply NVLink health. + Completed, + /// An unrecognized state value reported by the driver. + Unknown(u8), +} + +impl From for GpuFabricState { + fn from(value: nvmlGpuFabricState_t) -> Self { + match u32::from(value) { + NVML_GPU_FABRIC_STATE_NOT_SUPPORTED => GpuFabricState::NotSupported, + NVML_GPU_FABRIC_STATE_NOT_STARTED => GpuFabricState::NotStarted, + NVML_GPU_FABRIC_STATE_IN_PROGRESS => GpuFabricState::InProgress, + NVML_GPU_FABRIC_STATE_COMPLETED => GpuFabricState::Completed, + _ => GpuFabricState::Unknown(value), + } + } +} + +/// Overall NVLink fabric health. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub enum GpuFabricHealthSummary { + /// Health reporting is not supported. + NotSupported, + /// The fabric is healthy. + Healthy, + /// The fabric is unhealthy. + Unhealthy, + /// The fabric is healthy but running at limited capacity. + LimitedCapacity, + /// An unrecognized value reported by the driver. + Unknown(u8), +} + +impl From for GpuFabricHealthSummary { + fn from(value: u8) -> Self { + match u32::from(value) { + NVML_GPU_FABRIC_HEALTH_SUMMARY_NOT_SUPPORTED => Self::NotSupported, + NVML_GPU_FABRIC_HEALTH_SUMMARY_HEALTHY => Self::Healthy, + NVML_GPU_FABRIC_HEALTH_SUMMARY_UNHEALTHY => Self::Unhealthy, + NVML_GPU_FABRIC_HEALTH_SUMMARY_LIMITED_CAPACITY => Self::LimitedCapacity, + _ => Self::Unknown(value), + } + } +} + +/// Represents the value of an NVLink fabric health flag. +/// See `nvml.h` for more details. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub enum GpuFabricHealthFlag { + /// The flag is not supported. + NotSupported, + /// The condition is present. + True, + /// The condition is absent. + False, + /// An unrecognized value reported by the driver. + Unknown(u32), +} + +impl From for GpuFabricHealthFlag { + fn from(value: u32) -> Self { + match value { + 0 => Self::NotSupported, + 1 => Self::True, + 2 => Self::False, + _ => Self::Unknown(value), + } + } +} + +/// NVLink fabric misconfiguration reported in the health mask. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub enum GpuFabricIncorrectConfig { + /// The flag is not supported. + NotSupported, + /// No misconfiguration. + None, + /// Incorrect system GUID. + IncorrectSystemGuid, + /// Incorrect chassis serial number. + IncorrectChassisSerial, + /// No partition assigned. + NoPartition, + /// Insufficient NVLink resources. + InsufficientNvlinks, + /// An unrecognized value reported by the driver. + Unknown(u32), +} + +impl From for GpuFabricIncorrectConfig { + fn from(value: u32) -> Self { + match value { + NVML_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NOT_SUPPORTED => Self::NotSupported, + NVML_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NONE => Self::None, + NVML_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INCORRECT_SYSGUID => { + Self::IncorrectSystemGuid + } + NVML_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INCORRECT_CHASSIS_SN => { + Self::IncorrectChassisSerial + } + NVML_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NO_PARTITION => Self::NoPartition, + NVML_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_INSUFFICIENT_NVLINKS => { + Self::InsufficientNvlinks + } + _ => Self::Unknown(value), + } + } +} diff --git a/nvml-wrapper/src/struct_wrappers/device.rs b/nvml-wrapper/src/struct_wrappers/device.rs index 9a4bd11..15bb59b 100644 --- a/nvml-wrapper/src/struct_wrappers/device.rs +++ b/nvml-wrapper/src/struct_wrappers/device.rs @@ -2,7 +2,10 @@ use crate::bitmasks::device::FbcFlags; use crate::enum_wrappers::device::{ BridgeChip, Clock, EncoderType, FbcSessionType, PerformanceState, SampleValueType, }; -use crate::enums::device::{FirmwareVersion, SampleValue, UsedGpuMemory}; +use crate::enums::device::{ + FirmwareVersion, GpuFabricHealthFlag, GpuFabricHealthSummary, GpuFabricIncorrectConfig, + GpuFabricState, SampleValue, UsedGpuMemory, +}; use crate::error::{nvml_try, Bits, NvmlError}; use crate::ffi::bindings::*; use crate::structs::device::FieldId; @@ -1033,6 +1036,105 @@ impl VgpuSchedulerSetState { } } +/// Information about a GPU's registration with an NVLink fabric. +/// +/// Returned from `Device.gpu_fabric_info()`. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct GpuFabricInfo { + /// UUID of the NVLink fabric the GPU is registered with. + /// All zeroes when the GPU is not part of a multi-node NVLink fabric. + pub cluster_uuid: [u8; 16], + /// Status of the fabric registration process as a `nvmlReturn_t`. + /// This is only meaningful once `state` is [`GpuFabricState::Completed`]. + /// Use [`GpuFabricInfo::registration_result`] to interpret it as an + /// [`NvmlError`]. + pub status: u32, + /// Clique ID of the GPU within the cluster. + /// Only valid on NVLink multi-node systems. + pub clique_id: u32, + /// State of the GPU's fabric registration. + pub state: GpuFabricState, + /// NVLink fabric health. + pub health: GpuFabricHealth, +} + +impl GpuFabricInfo { + /// The fabric registration [`status`](Self::status) interpreted as an + /// [`NvmlError`]: `Ok(())` on success, otherwise the corresponding error. + /// + /// Only meaningful once [`state`](Self::state) is + /// [`GpuFabricState::Completed`]. + pub fn registration_result(&self) -> Result<(), NvmlError> { + nvml_try(self.status) + } +} + +impl From for GpuFabricInfo { + fn from(value: nvmlGpuFabricInfoV_t) -> Self { + GpuFabricInfo { + cluster_uuid: value.clusterUuid, + status: value.status, + clique_id: value.cliqueId, + state: GpuFabricState::from(value.state), + health: GpuFabricHealth::from_raw(value.healthMask, value.healthSummary), + } + } +} + +/// NVLink fabric health, decoded from the driver's health mask and summary. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct GpuFabricHealth { + /// Overall health summary. + pub summary: GpuFabricHealthSummary, + /// Whether NVLink bandwidth is degraded. + pub degraded_bandwidth: GpuFabricHealthFlag, + /// Whether NVLink route recovery is in progress. + pub route_recovery: GpuFabricHealthFlag, + /// Whether NVLink route recovery has failed or been aborted. + pub route_unhealthy: GpuFabricHealthFlag, + /// Whether NVLink access timeout recovery is in progress. + pub access_timeout_recovery: GpuFabricHealthFlag, + /// Reported misconfiguration, if any. + pub incorrect_config: GpuFabricIncorrectConfig, +} + +impl GpuFabricHealth { + fn from_raw(mask: u32, summary: u8) -> Self { + // Each field is a sub-range of `healthMask`. See `NVML_GPU_FABRIC_HEALTH_MASK` in nvml.h. + let field = |shift: u32, width: u32| (mask >> shift) & ((1u32 << width) - 1); + GpuFabricHealth { + summary: summary.into(), + degraded_bandwidth: field( + NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_DEGRADED_BW, + NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_DEGRADED_BW, + ) + .into(), + route_recovery: field( + NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY, + NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY, + ) + .into(), + route_unhealthy: field( + NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY, + NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY, + ) + .into(), + access_timeout_recovery: field( + NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ACCESS_TIMEOUT_RECOVERY, + NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ACCESS_TIMEOUT_RECOVERY, + ) + .into(), + incorrect_config: field( + NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_INCORRECT_CONFIGURATION, + NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_INCORRECT_CONFIGURATION, + ) + .into(), + } + } +} + #[cfg(test)] #[allow(unused_variables, unused_imports)] mod tests { diff --git a/nvml-wrapper/unwrapped_functions.txt b/nvml-wrapper/unwrapped_functions.txt index ff127c8..540a9fb 100644 --- a/nvml-wrapper/unwrapped_functions.txt +++ b/nvml-wrapper/unwrapped_functions.txt @@ -23,7 +23,6 @@ nvmlDeviceGetDramEncryptionMode nvmlDeviceGetDynamicPstatesInfo nvmlDeviceGetGpcClkMinMaxVfOffset nvmlDeviceGetGpuFabricInfo -nvmlDeviceGetGpuFabricInfoV nvmlDeviceGetGpuInstanceById nvmlDeviceGetGpuInstanceId nvmlDeviceGetGpuInstancePossiblePlacements