Expose kernel headers and perf to distributions

doc/docs/technical-documentation/boot-process.md

@@ -83,6 +83,7 @@ When started, the virtual machine will boot into the provided kernel, and then e
 - An entropy buffer, to seed the virtual machine's entropy
 - Information about the GPU drivers shares to mount, if any
 - Whether [wslg](https://github.com/microsoft/wslg) is enabled
+- Whether to mount the bundled Linux kernel headers and perf tooling (shipped in the kernel artifacts VHD; headers are mounted at `/usr/src/linux-headers-$(uname -r)` with `/lib/modules/$(uname -r)/build` linked to them or bind mounted when it is an existing directory, and perf is mounted at `/usr/lib/linux-tools/$(uname -r)`, added to the default `$PATH` and, when the distribution ships its own `/usr/bin/perf`, bind mounted over it)
 
 After applying all the configuration requested by [wslservice.exe](wslservice.exe.md), the virtual machine is ready to start Linux distributions.
 

src/linux/init/WslDistributionConfig.h

@@ -85,6 +85,7 @@ struct WslDistributionConfig
 
     bool GuiAppsEnabled = false;
     std::optional<int> FeatureFlags;
+    std::optional<std::string> KernelPerfPath;
     std::optional<LX_MINI_INIT_NETWORKING_MODE> NetworkingMode;
     std::optional<std::string> VmId;
 

src/linux/init/config.cpp

@@ -55,6 +55,8 @@ Module Name:
 #define LOCALE_FILE_PATH ETC_DEFAULT_FOLDER "locale"
 #define LOCALE_CONF_FILE_PATH ETC_FOLDER "locale.conf"
 #define PATH_ENV "PATH"
+#define PERF_BINARY_PATH "/usr/bin/perf"
+#define PERF_EXEC_PATH_ENV "PERF_EXEC_PATH"
 #define RESOLV_CONF_DIRECTORY_MODE 0755
 #define RESOLV_CONF_FILE_MODE 0644
 #define RESOLV_CONF_FILE_NAME "resolv.conf"
@@ -170,6 +172,35 @@ class RemoveMountAndEnvironmentOnScopeExit
     const char* m_mountPath = nullptr;
 };
 
+//
+// Moves a temporary mount created by mini_init into the distro namespace. The temporary mount point is
+// passed via MountEnvironmentName and its final target via PathEnvironmentName. The callback is invoked
+// with the target path once the mount has been moved.
+//
+template <typename TCallback>
+static void MoveTemporaryMount(const char* MountEnvironmentName, const char* PathEnvironmentName, const TCallback& Callback)
+try
+{
+    auto tempMount = RemoveMountAndEnvironmentOnScopeExit(MountEnvironmentName);
+    const char* target = tempMount ? getenv(PathEnvironmentName) : nullptr;
+    if (target == nullptr)
+    {
+        return;
+    }
+
+    const std::string targetPath{target};
+    if (unsetenv(PathEnvironmentName) < 0)
+    {
+        LOG_ERROR("unsetenv({}) failed {}", PathEnvironmentName, errno);
+    }
+
+    if (tempMount.MoveMount(targetPath.c_str()))
+    {
+        Callback(targetPath);
+    }
+}
+CATCH_LOG()
+
 constexpr auto HostsFileFormatString = LX_INIT_AUTO_GENERATED_FILE_HEADER
     "# [network]\n"
     "# generateHosts = false\n"
@@ -1110,20 +1141,74 @@ Return Value:
     }
     CATCH_LOG()
 
-    try
-    {
-        auto tempMount = RemoveMountAndEnvironmentOnScopeExit(LX_WSL2_KERNEL_MODULES_MOUNT_ENV);
-        if (tempMount)
+    std::string kernelModulesPath;
+    MoveTemporaryMount(LX_WSL2_KERNEL_MODULES_MOUNT_ENV, LX_WSL2_KERNEL_MODULES_PATH_ENV, [&](const std::string& target) {
+        kernelModulesPath = target;
+    });
+
+    MoveTemporaryMount(LX_WSL2_KERNEL_HEADERS_MOUNT_ENV, LX_WSL2_KERNEL_HEADERS_PATH_ENV, [&](const std::string& target) {
+        if (kernelModulesPath.empty())
+        {
+            return;
+        }
+
+        //
+        // Point /lib/modules/<release>/build at the kernel headers, replacing any entry that the
+        // distro may have created so that it can't shadow the headers matching the running kernel.
+        //
+        // N.B. A directory can't be replaced with a symlink (and removing it would mean a recursive
+        //      delete), so bind mount the headers over it instead.
+        //
+
+        const std::string linkPath = kernelModulesPath + "/build";
+        struct stat existing{};
+        if ((lstat(linkPath.c_str(), &existing) == 0) && S_ISDIR(existing.st_mode))
         {
-            auto target = getenv(LX_WSL2_KERNEL_MODULES_PATH_ENV);
-            if (target)
+            if (UtilMount(target.c_str(), linkPath.c_str(), nullptr, (MS_BIND | MS_REC), nullptr) < 0)
             {
-                unsetenv(LX_WSL2_KERNEL_MODULES_PATH_ENV);
-                tempMount.MoveMount(target);
+                LOG_ERROR("UtilMount({}, {}) failed {}", target, linkPath, errno);
             }
+
+            return;
         }
-    }
-    CATCH_LOG()
+
+        if ((unlink(linkPath.c_str()) < 0) && (errno != ENOENT))
+        {
+            LOG_ERROR("unlink({}) failed {}", linkPath, errno);
+        }
+
+        if (symlink(target.c_str(), linkPath.c_str()) < 0)
+        {
+            LOG_ERROR("symlink({}, {}) failed {}", target, linkPath, errno);
+        }
+    });
+
+    MoveTemporaryMount(LX_WSL2_KERNEL_PERF_MOUNT_ENV, LX_WSL2_KERNEL_PERF_PATH_ENV, [&](const std::string& target) {
+        //
+        // Expose the kernel-matched perf via the environment block (see ConfigCreateEnvironmentBlock).
+        //
+
+        Config.KernelPerfPath = target;
+
+        //
+        // If the distro ships its own perf, shadow it with a bind mount so that the binary matching the
+        // running kernel is used.
+        //
+        // N.B. The distro's file system is only modified if perf is already present as a regular file.
+        //      Distros without perf pick it up via $PATH instead.
+        //
+
+        struct stat existing{};
+        if ((stat(PERF_BINARY_PATH, &existing) == 0) && S_ISREG(existing.st_mode))
+        {
+            const std::string perfBinary = target + "/bin/perf";
+            const auto perfMountPoint = std::filesystem::canonical(PERF_BINARY_PATH);
+            if (UtilMountFile(perfBinary.c_str(), perfMountPoint.c_str()) < 0)
+            {
+                LOG_ERROR("UtilMountFile({}, {}) failed {}", perfBinary, perfMountPoint.string(), errno);
+            }
+        }
+    });
 
     //
     // Change the permission of some devtmpfs devices to be more permissive.
@@ -1664,6 +1749,25 @@ Return Value:
         {
             ConfigAppendToPath(Environment, LXSS_LIB_PATH);
         }
+
+        //
+        // Add the kernel-matched perf tools to the $PATH variable and point perf at its helper
+        // scripts since it is built with a prefix that does not match where it is mounted.
+        //
+
+        if (Config.KernelPerfPath.has_value())
+        {
+            ConfigAppendToPath(Environment, std::format("{}/bin", *Config.KernelPerfPath));
+
+            //
+            // N.B. This is only set if the user has not provided a value via WSLENV.
+            //
+
+            if (Environment.GetVariable(PERF_EXEC_PATH_ENV).empty())
+            {
+                Environment.AddVariable(PERF_EXEC_PATH_ENV, std::format("{}/libexec/perf-core", *Config.KernelPerfPath));
+            }
+        }
     }
 
     //

src/linux/init/main.cpp

@@ -85,6 +85,10 @@ Module Name:
 #define KERNEL_MODULES_PATH "/lib/modules"
 #define KERNEL_MODULES_VHD_PATH "/modules"
 #define KERNEL_MODULES_OVERLAY "/modules_overlay"
+#define KERNEL_HEADERS_TEMP_PATH "/kernel_headers"
+#define KERNEL_HEADERS_PATH_PREFIX "/usr/src/linux-headers-"
+#define KERNEL_PERF_TEMP_PATH "/kernel_perf"
+#define KERNEL_PERF_PATH_PREFIX "/usr/lib/linux-tools/"
 #define MODPROBE_PATH "/sbin/modprobe"
 #define PROCFS_PATH "/proc"
 #define RESOLV_CONF_FILE "resolv.conf"
@@ -115,6 +119,8 @@ struct VmConfiguration
     bool EnableSystemDistro = false;
     bool EnableCrashDumpCollection = false;
     std::string KernelModulesPath;
+    std::string KernelHeadersTarget;
+    std::string KernelPerfTarget;
     LX_MINI_INIT_NETWORKING_MODE NetworkingMode = LxMiniInitNetworkingModeNone;
 };
 
@@ -1621,6 +1627,31 @@ try
         AddEnvironmentVariable(LX_WSL2_KERNEL_MODULES_PATH_ENV, Config.KernelModulesPath.c_str());
     }
 
+    //
+    // If kernel headers were mounted, move them to a temporary location and pass the desired
+    // target path to the distro init via an environment variable. Distro init will move the
+    // mount to /usr/src/linux-headers-<uname -r> and create the
+    // /lib/modules/<release>/build symlink.
+    //
+
+    if (!Config.KernelHeadersTarget.empty())
+    {
+        AddTemporaryMount(LX_WSL2_KERNEL_HEADERS_MOUNT_ENV, KERNEL_HEADERS_TEMP_PATH, (MS_MOVE | MS_REC));
+        AddEnvironmentVariable(LX_WSL2_KERNEL_HEADERS_PATH_ENV, Config.KernelHeadersTarget.c_str());
+    }
+
+    //
+    // If the perf tooling was mounted, move it to a temporary location and pass the desired target
+    // path to the distro init via an environment variable. Distro init will move the mount to
+    // /usr/lib/linux-tools/<uname -r> and add it to the default $PATH.
+    //
+
+    if (!Config.KernelPerfTarget.empty())
+    {
+        AddTemporaryMount(LX_WSL2_KERNEL_PERF_MOUNT_ENV, KERNEL_PERF_TEMP_PATH, (MS_MOVE | MS_REC));
+        AddEnvironmentVariable(LX_WSL2_KERNEL_PERF_PATH_ENV, Config.KernelPerfTarget.c_str());
+    }
+
     //
     // Bind mount the init daemon into the distro namespace.
     //
@@ -3223,7 +3254,7 @@ try
         // N.B. The VHD is mounted as read-only but with a writable overlayfs layer. The modules
         //      directory must be writable for tools like depmod to work.
         //
-        // N.B. The artifacts VHD nests the modules under <release>/modules.
+        // N.B. The artifacts VHD nests its payloads under <release>/{modules,linux-headers,perf}.
         //      Older module-only VHDs place the modules tree at the filesystem root; fall back to that
         //      layout when the nested modules directory is not present.
         //
@@ -3253,7 +3284,7 @@ try
             {
                 LOG_WARNING(
                     "kernel modules VHD uses the legacy flat layout; support for the legacy modules VHD format will be "
-                    "removed in a future version");
+                    "removed in a future version; kernel headers and perf tooling are unavailable");
             }
             else if (!NestedLayout)
             {
@@ -3276,6 +3307,27 @@ try
             }
 
             Config.KernelModulesPath = std::move(Target);
+
+            //
+            // When the nested artifacts layout is present, bind mount the kernel headers and perf
+            // tooling to temporary locations. Each distro init moves them into the distro namespace
+            // at /usr/src/linux-headers-<release> and /usr/lib/linux-tools/<release> (see config.cpp).
+            //
+
+            if (NestedLayout)
+            {
+                const std::string HeadersSource = ArtifactsBase + "/linux-headers";
+                if (UtilMount(HeadersSource.c_str(), KERNEL_HEADERS_TEMP_PATH, nullptr, (MS_BIND | MS_REC), nullptr) == 0)
+                {
+                    Config.KernelHeadersTarget = std::format("{}{}", KERNEL_HEADERS_PATH_PREFIX, Release);
+                }
+
+                const std::string PerfSource = ArtifactsBase + "/perf";
+                if (UtilMount(PerfSource.c_str(), KERNEL_PERF_TEMP_PATH, nullptr, (MS_BIND | MS_REC), nullptr) == 0)
+                {
+                    Config.KernelPerfTarget = std::format("{}{}", KERNEL_PERF_PATH_PREFIX, Release);
+                }
+            }
         }
 
         //

src/linux/init/util.cpp

@@ -1738,6 +1738,10 @@ Return Value:
 int UtilMountFile(const char* Source, const char* Destination)
 try
 {
+    struct stat sourceInfo{};
+    THROW_LAST_ERROR_IF(stat(Source, &sourceInfo) < 0);
+    THROW_ERRNO_IF(EINVAL, !S_ISREG(sourceInfo.st_mode));
+
     // Is the file is a symlink, delete it since that would break the mount.
     if (std::filesystem::is_symlink(Destination))
     {

src/shared/inc/lxinitshared.h

@@ -261,6 +261,10 @@ Module Name:
 #define LX_WSL2_GUI_APP_SUPPORT_ENV "WSL2_GUI_APPS_ENABLED"
 #define LX_WSL2_KERNEL_MODULES_MOUNT_ENV "WSL2_KERNEL_MODULES_MOUNT"
 #define LX_WSL2_KERNEL_MODULES_PATH_ENV "WSL2_KERNEL_MODULES_PATH"
+#define LX_WSL2_KERNEL_HEADERS_MOUNT_ENV "WSL2_KERNEL_HEADERS_MOUNT"
+#define LX_WSL2_KERNEL_HEADERS_PATH_ENV "WSL2_KERNEL_HEADERS_PATH"
+#define LX_WSL2_KERNEL_PERF_MOUNT_ENV "WSL2_KERNEL_PERF_MOUNT"
+#define LX_WSL2_KERNEL_PERF_PATH_ENV "WSL2_KERNEL_PERF_PATH"
 #define LX_WSL2_SYSTEM_DISTRO_SHARE_ENV "WSL2_SYSTEM_DISTRO_SHARE"
 #define LX_WSL2_GPU_SHARE_ENV "WSL2_GPU_SHARE_ENV_"
 #define LX_WSL2_SHARED_MEMORY_OB_DIRECTORY "WSL2_SHARED_MEMORY_OB_DIRECTORY"

test/windows/UnitTests.cpp

@@ -3003,6 +3003,100 @@ Error code: Wsl/InstallDistro/WSL_E_DISTRO_NOT_FOUND
         ValidateOutput(L"dmesg | grep -iF \"failed to load module 'not-found'\" | wc -l", L"1\n", L"", 0);
     }
 
+    WSL2_TEST_METHOD(KernelArtifacts)
+    {
+        // The unified kernel artifacts VHD provides the kernel headers and the perf tooling
+        // alongside the kernel modules. Headers are mounted at /usr/src/linux-headers-$(uname -r)
+        // with /lib/modules/$(uname -r)/build symlinked to that directory; perf is mounted at
+        // /usr/lib/linux-tools/$(uname -r) and exposed via $PATH.
+
+        // Headers: the build symlink and a representative uapi header are present.
+        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(L"test -L /lib/modules/$(uname -r)/build", nullptr, nullptr, nullptr, nullptr), 0u);
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(L"test -s /lib/modules/$(uname -r)/build/include/linux/version.h", nullptr, nullptr, nullptr, nullptr), 0u);
+
+        // Headers are usable: compile and run a tiny program that includes recent uapi headers. The
+        // identifiers below fail to compile if the headers are missing or too old (BPF_PROG_TYPE_NETFILTER
+        // added in 6.4, IORING_OP_FUTEX_WAKE added in 6.7). Their numeric values are not a stable API
+        // contract, so the program only checks that <linux/version.h> matches the running kernel.
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(
+                LR"BASH(bash -ec '
+                    d=$(mktemp -d)
+                    trap "rm -rf $d" EXIT
+                    cat > "$d/t.c" <<EOF
+#include <stdio.h>
+#include <linux/version.h>
+#include <linux/bpf.h>
+#include <linux/io_uring.h>
+int main(void){
+    (void)BPF_PROG_TYPE_NETFILTER;
+    (void)IORING_OP_FUTEX_WAKE;
+    printf("%u.%u.%u\n",
+        LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL, LINUX_VERSION_SUBLEVEL);
+    return 0;
+}
+EOF
+                    cc -isystem /lib/modules/$(uname -r)/build/include -o "$d/t" "$d/t.c"
+                    v=$("$d/t")
+                    case "$(uname -r)" in "$v"*) exit 0 ;; *) exit 8 ;; esac
+                ')BASH",
+                nullptr,
+                nullptr,
+                nullptr,
+                nullptr),
+            0u);
+
+        // perf: leave no trace in the distro's file system and make the versioned binary reachable
+        // via $PATH, with PERF_EXEC_PATH pointing at its helper scripts.
+        //
+        // N.B. The test distro does not ship perf, so nothing should be created at /usr/bin/perf.
+        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(L"test -x /usr/lib/linux-tools/$(uname -r)/bin/perf", nullptr, nullptr, nullptr, nullptr), 0u);
+        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(L"test ! -e /usr/bin/perf", nullptr, nullptr, nullptr, nullptr), 0u);
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(L"test \"$(command -v perf)\" = \"/usr/lib/linux-tools/$(uname -r)/bin/perf\"", nullptr, nullptr, nullptr, nullptr), 0u);
+        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(L"perf --version", nullptr, nullptr, nullptr, nullptr), 0u);
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(L"test \"$(perf --exec-path)\" = \"/usr/lib/linux-tools/$(uname -r)/libexec/perf-core\"", nullptr, nullptr, nullptr, nullptr),
+            0u);
+
+        // Stale distro-provided artifacts are replaced or hidden after the VM restarts. A distro
+        // provided perf is shadowed by the binary matching the running kernel.
+        //
+        // N.B. The cleanup is registered before the distro's file system is modified so that a
+        //      failure can't leave a perf binary or a broken build symlink behind, which would
+        //      break subsequent runs.
+        auto cleanup = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&]() {
+            LxsstuLaunchWsl(
+                L"umount /usr/bin/distro-perf 2>/dev/null; rm -f /usr/bin/perf /usr/bin/distro-perf; ln -snf"
+                L" /usr/src/linux-headers-$(uname -r) /lib/modules/$(uname -r)/build",
+                nullptr,
+                nullptr,
+                nullptr,
+                nullptr);
+        });
+
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(
+                L"rm /lib/modules/$(uname -r)/build && ln -s /tmp /lib/modules/$(uname -r)/build && printf old-perf >"
+                L" /usr/bin/distro-perf && ln -s distro-perf /usr/bin/perf",
+                nullptr,
+                nullptr,
+                nullptr,
+                nullptr),
+            0u);
+        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(L"--shutdown"), 0u);
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(L"test \"$(readlink /lib/modules/$(uname -r)/build)\" = \"/usr/src/linux-headers-$(uname -r)\"", nullptr, nullptr, nullptr, nullptr),
+            0u);
+        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(L"test \"$(readlink /usr/bin/perf)\" = \"distro-perf\"", nullptr, nullptr, nullptr, nullptr), 0u);
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(
+                L"test \"$(stat -Lc %d:%i /usr/bin/perf)\" = \"$(stat -Lc %d:%i /usr/lib/linux-tools/$(uname -r)/bin/perf)\"", nullptr, nullptr, nullptr, nullptr),
+            0u);
+        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(L"/usr/bin/perf --version", nullptr, nullptr, nullptr, nullptr), 0u);
+    }
+
     WSL2_TEST_METHOD(CrashCollection)
     {
         const auto folder = std::filesystem::absolute(L"test-crash-dumps");