Add nested virtualization support for WSLC containers

doc/docs/api-reference/c/enumerations/wslcsessionfeatureflags.md

@@ -4,11 +4,13 @@
 typedef enum WslcSessionFeatureFlags
 {
     WSLC_SESSION_FEATURE_FLAG_NONE = 0x00000000,
-    WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU = 0x00000004
+    WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU = 0x00000004,
+    WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION = 0x00000040
 } WslcSessionFeatureFlags;
 ```
 
 | Enumerator | Value |
 |---|---|
 | `WSLC_SESSION_FEATURE_FLAG_NONE` | `0x00000000` |
 | `WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU` | `0x00000004` |
+| `WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION` | `0x00000040` |

doc/docs/api-reference/c/session-apis/wslcsetsessionsettingsfeatureflags.md

@@ -16,5 +16,6 @@ Example:
 ```c
 HRESULT hr = WslcSetSessionSettingsFeatureFlags(
     &sessionSettings,
-    WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU);
+    WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU |
+        WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION);
 ```

doc/docs/api-reference/cpp/settings-classes/sessionsettings.md

@@ -29,6 +29,7 @@ Do not put credentials or other sensitive information in the session's name.
 - `VhdRequirements()` / setter
   - setter rejects `nullptr`
 - `EnableGpu()` / setter
+- `EnableNestedVirtualization()` / setter
 
 ```cpp
 SessionSettings settings{ L"demo", L"C:\\WSLC\\demo" };
@@ -39,11 +40,13 @@ settings.MemorySizeInMB(winrt::box_value<uint32_t>(4096).as<winrt::Windows::Foun
 settings.Timeout(winrt::box_value(winrt::Windows::Foundation::TimeSpan{ std::chrono::minutes(5) })
     .as<winrt::Windows::Foundation::IReference<winrt::Windows::Foundation::TimeSpan>>());
 settings.EnableGpu(true);
+settings.EnableNestedVirtualization(true);
 
 auto name = settings.Name();
 auto path = settings.StoragePath();
 auto cpu = settings.CpuCount();
 auto memory = settings.MemorySizeInMB();
 auto timeout = settings.Timeout();
 auto enableGpu = settings.EnableGpu();
+auto enableNestedVirtualization = settings.EnableNestedVirtualization();
 ```

doc/docs/api-reference/csharp/settings-classes/sessionsettings.md

@@ -14,6 +14,7 @@ public sealed class SessionSettings
     public TimeSpan? Timeout { get; set; }
     public VhdOptions VhdRequirements { get; set; }
     public bool EnableGpu { get; set; }
+    public bool EnableNestedVirtualization { get; set; }
 }
 ```
 
@@ -43,6 +44,7 @@ var sessionSettings = new SessionSettings("demo-session", @"C:\WslcData")
     CpuCount = 4,
     MemorySizeInMB = 4096,
     Timeout = TimeSpan.FromMinutes(5),
-    EnableGpu = true
+    EnableGpu = true,
+    EnableNestedVirtualization = true
 };
 ```

intune/WSL.admx

@@ -198,6 +198,17 @@
       </disabledValue>
     </policy>
 
+    <policy name="AllowWSLContainerNestedVirtualization" class="Machine" displayName="$(string.AllowWSLContainerNestedVirtualization)" explainText="$(string.AllowWSLContainerNestedVirtualizationExplain)" key="Software\Policies\WSL" valueName="AllowWSLContainerNestedVirtualization">
+      <parentCategory ref="WSLContainer" />
+      <supportedOn ref="windows:SUPPORTED_Windows10" />
+      <enabledValue>
+        <decimal value="1" />
+      </enabledValue>
+      <disabledValue>
+        <decimal value="0" />
+      </disabledValue>
+    </policy>
+
     <policy name="WSLContainerRegistryAllowlist" class="Machine" displayName="$(string.WSLContainerRegistryAllowlist)" explainText="$(string.WSLContainerRegistryAllowlistExplain)" presentation="$(presentation.WSLContainerRegistryAllowlist)" key="Software\Policies\WSL">
       <parentCategory ref="WSLContainer" />
       <supportedOn ref="windows:SUPPORTED_Windows10" />

intune/en-US/WSL.adml

@@ -58,6 +58,9 @@
         <string id="AllowWSLContainerPrivileged"><!-- _locComment_text='{Locked="WSL"}' -->Allow privileged WSL containers</string>
         <string id="AllowWSLContainerPrivilegedExplain"><!-- _locComment_text='{Locked="WSL"}{Locked="Disabled"}' -->This policy controls whether WSL containers can run in privileged mode on this machine. When enabled or not configured, users and Windows applications can start privileged containers. When set to 'Disabled', requests to start a privileged container are refused for all users and Windows apps. Privileged containers run with elevated Linux capabilities and reduced isolation from the container host.</string>
 
+        <string id="AllowWSLContainerNestedVirtualization"><!-- _locComment_text='{Locked="WSL"}' -->Allow nested virtualization for WSL containers</string>
+        <string id="AllowWSLContainerNestedVirtualizationExplain"><!-- _locComment_text='{Locked="WSL"}' -->This policy controls whether WSL container sessions can enable nested virtualization. When enabled or not configured, applications can request nested virtualization for WSL container sessions. When set to disabled, requests to create WSL container sessions with nested virtualization enabled are rejected.</string>
+
         <string id="WSLContainerRegistryAllowlist"><!-- _locComment_text='{Locked="WSL"}' -->Allowlist for WSL container registries</string>
         <string id="WSLContainerRegistryAllowlistExplain"><!-- _locComment_text='{Locked="WSL"}' -->When enabled, WSL container will only be allowed to pull images from the registries listed here. This affects both the WSL container CLI and all applications using the WSL container API.</string>
     </stringTable>

localization/strings/en-US/Resources.resw

@@ -1005,6 +1005,9 @@ Falling back to NAT networking.</value>
   <data name="MessageWSLContainerDisabled" xml:space="preserve">
     <value>WSL container is disabled by the computer policy.</value>
   </data>
+  <data name="MessageWSLContainerNestedVirtualizationDisabled" xml:space="preserve">
+    <value>Nested virtualization for WSL containers is disabled by the computer policy.</value>
+  </data>
   <data name="MessageRegistryBlockedByPolicy" xml:space="preserve">
     <value>The container image registry '{}' is blocked by the computer policy.</value>
     <comment>{FixedPlaceholder="{}"}Command line arguments, file names and string inserts should not be translated</comment>

src/linux/init/WSLCInit.cpp

@@ -30,6 +30,12 @@ Module Name:
 #include <sys/signalfd.h>
 #include <arpa/inet.h>
 
+#ifdef __x86_64__
+#include <cpuid.h>
+#endif
+
+#include <array>
+#include <cstring>
 #include <pty.h>
 #include <mutex>
 #include "mountutilcpp.h"
@@ -70,6 +76,7 @@ struct WSLCState
 static WSLCState g_state;
 
 constexpr auto c_kernelModulesVhdMountPoint = "/kernel_modules_vhd";
+constexpr auto c_kvmDevicePath = "/dev/kvm";
 
 void WriteWslcCdiSpec()
 try
@@ -671,6 +678,54 @@ void HandleMessageImpl(
     Transaction.Send(Response);
 }
 
+void LoadKvmModule()
+{
+#ifdef __x86_64__
+
+    // On x86_64 the vendor specific KVM module is loadable and isn't loaded by default. Containers
+    // can't load it themselves, so it needs to be present before the container is launched.
+    unsigned int eax{};
+    unsigned int ebx{};
+    unsigned int ecx{};
+    unsigned int edx{};
+    THROW_ERRNO_IF(ENOTSUP, __get_cpuid(0, &eax, &ebx, &ecx, &edx) == 0);
+
+    std::array<char, 13> vendor{};
+    memcpy(vendor.data(), &ebx, sizeof(ebx));
+    memcpy(vendor.data() + sizeof(ebx), &edx, sizeof(edx));
+    memcpy(vendor.data() + sizeof(ebx) + sizeof(edx), &ecx, sizeof(ecx));
+
+    const char* module = nullptr;
+    if (strcmp(vendor.data(), "GenuineIntel") == 0)
+    {
+        module = "kvm_intel";
+    }
+    else if (strcmp(vendor.data(), "AuthenticAMD") == 0)
+    {
+        module = "kvm_amd";
+    }
+
+    if (module == nullptr)
+    {
+        LOG_ERROR("Unsupported processor vendor for KVM: '{}'", vendor.data());
+        THROW_ERRNO(ENOTSUP);
+    }
+
+    const char* argv[] = {"/sbin/modprobe", module, nullptr};
+    THROW_ERRNO_IF(EIO, UtilCreateProcessAndWait("/sbin/modprobe", argv) < 0);
+
+#endif
+
+    // On aarch64 KVM is built into the kernel rather than shipped as a loadable module, so there is
+    // nothing to load. Validate the device on all architectures so an unsupported guest fails here
+    // instead of surfacing as a missing device once the container starts.
+    if (!std::filesystem::exists(c_kvmDevicePath))
+    {
+        LOG_ERROR("{} is not present, nested virtualization is not available in this guest", c_kvmDevicePath);
+        THROW_ERRNO(ENOTSUP);
+    }
+}
+
 template <typename TMessage>
 void HandleMountMessage(
     wsl::shared::SocketChannel& Channel, wsl::shared::Transaction& Transaction, const TMessage& Message, const gsl::span<gsl::byte>& Buffer)
@@ -856,6 +911,11 @@ void HandleMessageImpl(
         THROW_LAST_ERROR_IF(
             UtilMount(modulesSource.c_str(), g_state.ModulesMountPoint->c_str(), nullptr, (MS_BIND | MS_REC), nullptr, c_defaultRetryTimeout) < 0);
 
+        if (Message.LoadKvm)
+        {
+            LoadKvmModule();
+        }
+
         response.Result = 0;
     }
     catch (...)

src/shared/inc/lxinitshared.h

@@ -1714,9 +1714,10 @@ struct WSLC_MOUNT_MODULES
 
     MESSAGE_HEADER Header{};
     unsigned int SourceIndex{};
+    bool LoadKvm{};
     char Buffer[];
 
-    PRETTY_PRINT(FIELD(Header), STRING_FIELD(SourceIndex));
+    PRETTY_PRINT(FIELD(Header), STRING_FIELD(SourceIndex), FIELD(LoadKvm));
 };
 
 struct WSLC_EXEC

src/windows/WslcSDK/winrt/SessionSettings.cpp

@@ -183,6 +183,21 @@ void SessionSettings::EnableGpu(bool value)
     WI_UpdateFlag(m_featureFlags, WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU, value);
 }
 
+bool SessionSettings::EnableNestedVirtualization()
+{
+    return WI_IsFlagSet(m_featureFlags, WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION);
+}
+
+void SessionSettings::EnableNestedVirtualization(bool value)
+{
+    if (m_sessionSettings)
+    {
+        throw hresult_illegal_state_change(L"Cannot change nested virtualization setting after session has been initialized");
+    }
+
+    WI_UpdateFlag(m_featureFlags, WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION, value);
+}
+
 WslcSessionSettings* SessionSettings::ToStructPointer()
 {
     if (m_sessionSettings)

src/windows/WslcSDK/winrt/SessionSettings.h

@@ -37,6 +37,8 @@ struct SessionSettings : SessionSettingsT<SessionSettings>
     void VhdRequirements(winrt::Microsoft::WSL::Containers::VhdOptions const& value);
     bool EnableGpu();
     void EnableGpu(bool value);
+    bool EnableNestedVirtualization();
+    void EnableNestedVirtualization(bool value);
 
     WslcSessionSettings* ToStructPointer();
 

src/windows/WslcSDK/winrt/wslcsdk.idl

@@ -59,6 +59,7 @@ namespace Microsoft.WSL.Containers
         Windows.Foundation.IReference<Windows.Foundation.TimeSpan> Timeout;
         VhdOptions VhdRequirements;
         Boolean EnableGpu;
+        Boolean EnableNestedVirtualization;
     };
 
     runtimeclass Session : Windows.Foundation.IClosable

src/windows/WslcSDK/wslcsdk.cpp

@@ -43,8 +43,10 @@ template <>
 struct FlagsTraits<WslcSessionFeatureFlags>
 {
     using WslcType = WSLCFeatureFlags;
-    constexpr static WslcSessionFeatureFlags Mask = WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU;
+    constexpr static WslcSessionFeatureFlags Mask =
+        static_cast<WslcSessionFeatureFlags>(WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU | WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION);
     WSLC_FLAG_VALUE_ASSERT(WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU, WslcFeatureFlagsGPU);
+    WSLC_FLAG_VALUE_ASSERT(WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION, WslcFeatureFlagsNestedVirtualization);
 };
 
 template <>

src/windows/WslcSDK/wslcsdk.h

@@ -115,7 +115,8 @@ typedef struct WslcVhdRequirements
 typedef enum WslcSessionFeatureFlags
 {
     WSLC_SESSION_FEATURE_FLAG_NONE = 0x00000000,
-    WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU = 0x00000004
+    WSLC_SESSION_FEATURE_FLAG_ENABLE_GPU = 0x00000004,
+    WSLC_SESSION_FEATURE_FLAG_NESTED_VIRTUALIZATION = 0x00000040
 } WslcSessionFeatureFlags;
 
 DEFINE_ENUM_FLAG_OPERATORS(WslcSessionFeatureFlags);

src/windows/common/hcs.cpp

@@ -16,6 +16,7 @@ Module Name:
 #include "precomp.h"
 #include "hcs.hpp"
 #include <ComputeCore.h>
+#include "helpers.hpp"
 
 #pragma hdrstop
 
@@ -134,6 +135,17 @@ const std::vector<std::string>& wsl::windows::common::hcs::GetProcessorFeatures(
     return g_processorFeatures;
 }
 
+bool wsl::windows::common::hcs::IsNestedVirtualizationSupported()
+{
+    if (!wsl::windows::common::helpers::IsWindows11OrAbove())
+    {
+        return false;
+    }
+
+    const auto& processorFeatures = GetProcessorFeatures();
+    return std::find(processorFeatures.begin(), processorFeatures.end(), "NestedVirt") != processorFeatures.end();
+}
+
 wsl::shared::hns::HNSEndpoint wsl::windows::common::hcs::GetEndpointProperties(HCN_ENDPOINT Endpoint)
 {
     WSL_LOG_DEBUG("HcsGetEndpointProperties");

src/windows/common/hcs.hpp

@@ -59,6 +59,8 @@ wsl::shared::hns::HNSEndpoint GetEndpointProperties(HCN_ENDPOINT endpoint);
 
 const std::vector<std::string>& GetProcessorFeatures();
 
+bool IsNestedVirtualizationSupported();
+
 GUID GetRuntimeId(_In_ HCS_SYSTEM ComputeSystem);
 
 std::pair<uint32_t, uint32_t> GetSchemaVersion();

src/windows/inc/wslpolicies.h

@@ -35,6 +35,7 @@ inline constexpr auto c_allowCustomFirewallUserSetting = L"AllowFirewallUserSett
 inline constexpr auto c_defaultNetworkingMode = L"DefaultNetworkingMode";
 inline constexpr auto c_allowWSLContainer = L"AllowWSLContainer";
 inline constexpr auto c_allowWSLContainerPrivileged = L"AllowWSLContainerPrivileged";
+inline constexpr auto c_allowWSLContainerNestedVirtualization = L"AllowWSLContainerNestedVirtualization";
 inline constexpr auto c_wslContainerRegistryAllowlist = L"WSLContainerRegistryAllowlist";
 
 inline std::optional<DWORD> GetPolicyValue(HKEY key, LPCWSTR name)
@@ -98,6 +99,15 @@ inline wil::unique_hkey OpenPoliciesKey()
     return key;
 }
 
+inline void EnsureWslContainerNestedVirtualizationAllowed()
+{
+    const auto policiesKey = OpenPoliciesKey();
+    THROW_HR_WITH_USER_ERROR_IF(
+        HRESULT_FROM_WIN32(ERROR_ACCESS_DISABLED_BY_POLICY),
+        wsl::shared::Localization::MessageWSLContainerNestedVirtualizationDisabled(),
+        !IsFeatureAllowed(policiesKey.get(), c_allowWSLContainerNestedVirtualization));
+}
+
 // Opens the WSLContainerRegistryAllowlist sub-key under the supplied policies key for
 // read-only enumeration. Returns an empty handle when the policy is not configured (sub-key
 // absent) or the parent key is null.

src/windows/service/exe/HcsVirtualMachine.cpp

@@ -24,7 +24,10 @@ Module Name:
 #include "wslutil.h"
 #include "lxinitshared.h"
 #include "DnsResolver.h"
+#include "ExecutionContext.h"
+#include "Localization.h"
 #include "string.hpp"
+#include "wslpolicies.h"
 
 using namespace wsl::windows::common;
 using helpers::WindowsBuildNumbers;
@@ -147,6 +150,18 @@ HcsVirtualMachine::HcsVirtualMachine(_In_ const WSLCSessionSettings* Settings)
         vmSettings.ComputeTopology.Memory.HostingProcessNameSuffix = SanitizeHostingProcessNameSuffix(Settings->DisplayName);
     }
 
+    if (FeatureEnabled(WslcFeatureFlagsNestedVirtualization))
+    {
+        wsl::windows::policies::EnsureWslContainerNestedVirtualizationAllowed();
+
+        THROW_HR_WITH_USER_ERROR_IF(
+            HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED),
+            wsl::shared::Localization::MessageNestedVirtualizationNotSupported(),
+            !hcs::IsNestedVirtualizationSupported());
+
+        vmSettings.ComputeTopology.Processor.ExposeVirtualizationExtensions = true;
+    }
+
 #ifdef _AMD64_
 
     HV_X64_HYPERVISOR_HARDWARE_FEATURES hardwareFeatures{};

src/windows/service/exe/WSLCSessionManager.cpp

@@ -38,7 +38,9 @@ Module Name:
 #include "wslutil.h"
 #include "filesystem.hpp"
 #include "APICompat.h"
+#include "hcs.hpp"
 #include "Localization.h"
+#include "wslpolicies.h"
 
 extern wsl::windows::service::PluginManager g_pluginManager;
 
@@ -207,6 +209,15 @@ void WSLCSessionManagerImpl::CreateSession(
         THROW_HR_IF_MSG(E_INVALIDARG, WI_IsAnyFlagSet(Flags, ~WSLCSessionFlagsValid), "Invalid session flags: 0x%x", Flags);
         THROW_HR_IF_MSG(
             E_INVALIDARG, WI_IsAnyFlagSet(Settings->FeatureFlags, ~WSLCFeatureFlagsValid), "Invalid feature flags: 0x%x", Settings->FeatureFlags);
+        if (WI_IsFlagSet(Settings->FeatureFlags, WslcFeatureFlagsNestedVirtualization))
+        {
+            wsl::windows::policies::EnsureWslContainerNestedVirtualizationAllowed();
+            THROW_HR_WITH_USER_ERROR_IF(
+                HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED),
+                wsl::shared::Localization::MessageNestedVirtualizationNotSupported(),
+                !wsl::windows::common::hcs::IsNestedVirtualizationSupported());
+        }
+
         THROW_HR_IF_MSG(
             E_INVALIDARG,
             WI_IsAnyFlagSet(Settings->StorageFlags, ~WSLCSessionStorageFlagsValid),

src/windows/service/exe/WslCoreVm.cpp

@@ -1543,17 +1543,7 @@ std::wstring WslCoreVm::GenerateConfigJson()
     {
         try
         {
-            if (wsl::windows::common::helpers::IsWindows11OrAbove())
-            {
-                const auto& processorFeatures = wsl::windows::common::hcs::GetProcessorFeatures();
-                auto feature = std::find(processorFeatures.begin(), processorFeatures.end(), "NestedVirt");
-                m_vmConfig.EnableNestedVirtualization = (feature != processorFeatures.end());
-            }
-            else
-            {
-                m_vmConfig.EnableNestedVirtualization = false;
-            }
-
+            m_vmConfig.EnableNestedVirtualization = wsl::windows::common::hcs::IsNestedVirtualizationSupported();
             vmSettings.ComputeTopology.Processor.ExposeVirtualizationExtensions = m_vmConfig.EnableNestedVirtualization;
             if (!m_vmConfig.EnableNestedVirtualization)
             {