From 5f9b10fea92dfae2520619c93553656a6cacb788 Mon Sep 17 00:00:00 2001 From: Blue Date: Thu, 20 Aug 2026 16:22:46 -0700 Subject: [PATCH] Create a script to generate the Microsoft.WSL.TestData nuget package --- tools/test/download-test-data-rpms.ps1 | 73 ------- tools/test/generate-test-data.ps1 | 239 +++++++++++++++++++++ tools/test/images/wslc-registry/Dockerfile | 2 +- tools/test/pack-test-data.ps1 | 35 --- 4 files changed, 240 insertions(+), 109 deletions(-) delete mode 100644 tools/test/download-test-data-rpms.ps1 create mode 100644 tools/test/generate-test-data.ps1 delete mode 100644 tools/test/pack-test-data.ps1 diff --git a/tools/test/download-test-data-rpms.ps1 b/tools/test/download-test-data-rpms.ps1 deleted file mode 100644 index 539323f2d4..0000000000 --- a/tools/test/download-test-data-rpms.ps1 +++ /dev/null @@ -1,73 +0,0 @@ - -[CmdletBinding(PositionalBinding = $False)] -param ( - [Parameter(Mandatory = $true)][string]$Target, - [string]$BaseUrl = "https://packages.microsoft.com/azurelinux/3.0/prod/base" -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version Latest -$ProgressPreference = "SilentlyContinue" - -$archMap = @{ - "x64" = "x86_64" - "arm64" = "aarch64" -} - -$packages = @("socat", "readline", "ncurses-libs") - -function Get-LatestRpmUrl { - param ( - [string]$RepoUrl, - [string]$Arch, - [string]$Package - ) - - $letter = $Package.Substring(0, 1).ToLowerInvariant() - $listingUrl = "$RepoUrl/$Arch/Packages/$letter/" - - $listing = curl.exe --fail -sSL $listingUrl - if ($LASTEXITCODE -ne 0) { - throw "Failed to list packages at $listingUrl" - } - - $pattern = 'href="(?' + [regex]::Escape($Package) + '-(?[0-9][0-9.]*)-(?[0-9]+)\.azl[0-9]+\.' + $Arch + '\.rpm)"' - $matches = [regex]::Matches($listing, $pattern) - if ($matches.Count -eq 0) { - throw "No '$Package' rpm found for '$Arch' at $listingUrl" - } - - $latest = $matches | - Sort-Object -Property ` - @{ Expression = { [version]$_.Groups["ver"].Value } }, ` - @{ Expression = { [int]$_.Groups["rel"].Value } } | - Select-Object -Last 1 - - return "$listingUrl$($latest.Groups['file'].Value)" -} - -foreach ($archEntry in $archMap.GetEnumerator()) { - $nugetArch = $archEntry.Key - $repoArch = $archEntry.Value - - $archInput = Join-Path $Target $nugetArch - if (-not (Test-Path -Path $archInput -PathType Container)) { - Write-Warning "Skipping '$nugetArch': '$archInput' does not exist." - continue - } - - $packagesDir = Join-Path $archInput "packages" - New-Item -ItemType Directory -Path $packagesDir -Force | Out-Null - - foreach ($package in $packages) { - $url = Get-LatestRpmUrl -RepoUrl $BaseUrl -Arch $repoArch -Package $package - $fileName = Split-Path -Path $url -Leaf - $destination = Join-Path $packagesDir $fileName - - Write-Output "[$nugetArch] Downloading $url" - curl.exe --fail -sSL -o $destination $url - if ($LASTEXITCODE -ne 0) { - throw "Failed to download $url" - } - } -} \ No newline at end of file diff --git a/tools/test/generate-test-data.ps1 b/tools/test/generate-test-data.ps1 new file mode 100644 index 0000000000..f94a4021a5 --- /dev/null +++ b/tools/test/generate-test-data.ps1 @@ -0,0 +1,239 @@ +[CmdletBinding(PositionalBinding = $false)] +param ( + [Parameter(Mandatory)] + [string]$Version, + [string]$Distribution +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$WorkingDirectory = Join-Path $PSScriptRoot "test-data" +$OutputDirectory = $PWD.Path + +$wslArguments = @() +if (-not [string]::IsNullOrEmpty($Distribution)) { + $wslArguments += @("--distribution", $Distribution) +} +$wslArguments += "--exec" + +function Invoke-WslCommand { + param ( + [Parameter(Mandatory = $true)][string[]]$Command, + [switch]$CaptureOutput + ) + + if ($CaptureOutput) { + $output = & wsl.exe @wslArguments @Command + if ($LASTEXITCODE -ne 0) { + throw "WSL command failed with exit code $LASTEXITCODE`: $($Command -join ' ')" + } + + return $output + } + + & wsl.exe @wslArguments @Command + if ($LASTEXITCODE -ne 0) { + throw "WSL command failed with exit code $LASTEXITCODE`: $($Command -join ' ')" + } +} + +function ConvertTo-WslPath { + param ( + [Parameter(Mandatory = $true)][string]$Path + ) + + $result = Invoke-WslCommand -Command @("wslpath", "-a", $Path) -CaptureOutput + return ($result | Select-Object -Last 1).Trim() +} + +function Enable-ForeignArchitectureBuilds { + $nativeArchitecture = ( + Invoke-WslCommand -Command @("docker", "version", "--format", "{{.Server.Arch}}") -CaptureOutput | + Select-Object -Last 1 + ).Trim() + + foreach ($architecture in @("amd64", "arm64")) { + if ($architecture -eq $nativeArchitecture) { + continue + } + + Write-Host "Installing Docker binfmt support for $architecture" + Invoke-WslCommand -Command @( + "docker", "container", "run", + "--privileged", + "--rm", + "tonistiigi/binfmt", + "--install", $architecture + ) + } +} + +function Download-TestDataRpms { + param ( + [Parameter(Mandatory = $true)][string]$NugetArchitecture, + [Parameter(Mandatory = $true)][string]$DockerArchitecture + ) + + $packagesDirectory = Join-Path $WorkingDirectory "$NugetArchitecture\packages" + New-Item -ItemType Directory -Path $packagesDirectory -Force | Out-Null + $packagesPath = ConvertTo-WslPath $packagesDirectory + + Write-Host "[$NugetArchitecture] Downloading Azure Linux RPMs" + $downloadCommand = @( + "tdnf reinstall -y --downloadonly --downloaddir=/packages readline ncurses-libs", + "tdnf install -y --downloadonly --downloaddir=/packages socat" + ) -join " && " + + Invoke-WslCommand -Command @( + "docker", "container", "run", + "--rm", + "--platform", "linux/$DockerArchitecture", + "--volume", "${packagesPath}:/packages", + "mcr.microsoft.com/azurelinux/base/core:3.0", + "sh", "-c", $downloadCommand + ) + + $downloadedPackages = @(Get-ChildItem -LiteralPath $packagesDirectory -Filter "*.rpm" -File) + if ($downloadedPackages.Count -ne 3) { + throw "Expected 3 RPMs for $NugetArchitecture, found $($downloadedPackages.Count)." + } +} + +function Export-DockerImage { + param ( + [Parameter(Mandatory = $true)][string]$Image, + [Parameter(Mandatory = $true)][string]$DockerArchitecture, + [Parameter(Mandatory = $true)][string]$OutputPath, + [string]$Source + ) + + if ([string]::IsNullOrEmpty($Source)) { + $Source = "docker-daemon:$Image" + } + + $outputDirectory = Split-Path -Parent $OutputPath + $outputFileName = Split-Path -Leaf $OutputPath + $wslOutputDirectory = ConvertTo-WslPath $outputDirectory + + Invoke-WslCommand -Command @( + "docker", "container", "run", + "--rm", + "--volume", "/var/run/docker.sock:/var/run/docker.sock", + "--volume", "${wslOutputDirectory}:/output", + "quay.io/skopeo/stable", + "copy", + "--override-arch", $DockerArchitecture, + $Source, + "docker-archive:/output/${outputFileName}:$Image" + ) + + # Adjust the tar padding to match docker's behavior so the "wslc image save" tests match the size of the tars in the testdata package. + $archive = Get-Item -LiteralPath $OutputPath + [long]$tarRecordSize = 10KB + $paddedLength = [Math]::Ceiling($archive.Length / $tarRecordSize) * $tarRecordSize + if ($paddedLength -ne $archive.Length) { + $stream = [System.IO.File]::OpenWrite($archive.FullName) + try { + $stream.SetLength($paddedLength) + } + finally { + $stream.Dispose() + } + } +} + +function Export-TestImages { + param ( + [Parameter(Mandatory = $true)][string]$NugetArchitecture, + [Parameter(Mandatory = $true)][string]$DockerArchitecture + ) + + $architectureDirectory = Join-Path $WorkingDirectory $NugetArchitecture + New-Item -ItemType Directory -Path $architectureDirectory -Force | Out-Null + + $images = [ordered]@{ + "alpine:latest" = "alpine-latest.tar" + "debian:latest" = "debian-latest.tar" + "hello-world:latest" = "HelloWorldSaved.tar" + "python:3.12-alpine" = "python-3_12-alpine.tar" + } + + foreach ($entry in $images.GetEnumerator()) { + Write-Host "[$NugetArchitecture] Downloading $($entry.Key)" + Export-DockerImage ` + -Image $entry.Key ` + -DockerArchitecture $DockerArchitecture ` + -OutputPath (Join-Path $architectureDirectory $entry.Value) ` + -Source "docker://docker.io/library/$($entry.Key)" + } + + Invoke-WslCommand -Command @("docker", "image", "pull", "--platform", "linux/$DockerArchitecture", "hello-world:latest") + + $containerId = $null + try { + $containerId = ( + Invoke-WslCommand -Command @( + "docker", "container", "create", "--platform", "linux/$DockerArchitecture", "hello-world:latest" + ) -CaptureOutput | + Select-Object -Last 1 + ).Trim() + + $exportPath = ConvertTo-WslPath (Join-Path $architectureDirectory "HelloWorldExported.tar") + Invoke-WslCommand -Command @("docker", "container", "export", "--output", $exportPath, $containerId) + } + finally { + if (-not [string]::IsNullOrEmpty($containerId)) { + Invoke-WslCommand -Command @("docker", "container", "rm", "--force", $containerId) + } + } + + Write-Host "[$NugetArchitecture] Building wslc-registry:latest" + $buildContext = ConvertTo-WslPath (Join-Path $PSScriptRoot "images\wslc-registry") + Invoke-WslCommand -Command @( + "docker", "image", "build", + "--platform", "linux/$DockerArchitecture", + "--pull", + "--tag", "wslc-registry:latest", + $buildContext + ) + + Export-DockerImage ` + -Image "wslc-registry:latest" ` + -DockerArchitecture $DockerArchitecture ` + -OutputPath (Join-Path $architectureDirectory "wslc-registry.tar") +} + +try { + if (Test-Path -LiteralPath $WorkingDirectory) { + Remove-Item -LiteralPath $WorkingDirectory -Recurse -Force + } + + New-Item -ItemType Directory -Path $WorkingDirectory -Force | Out-Null + + Write-Host "Checking Docker in WSL" + Invoke-WslCommand -Command @("docker", "version") + Enable-ForeignArchitectureBuilds + + Export-TestImages -NugetArchitecture "x64" -DockerArchitecture "amd64" + Export-TestImages -NugetArchitecture "arm64" -DockerArchitecture "arm64" + + Download-TestDataRpms -NugetArchitecture "x64" -DockerArchitecture "amd64" + Download-TestDataRpms -NugetArchitecture "arm64" -DockerArchitecture "arm64" + + $nuspecPath = Join-Path $WorkingDirectory "Microsoft.WSL.TestData.nuspec" + Copy-Item -LiteralPath (Join-Path $PSScriptRoot "Microsoft.WSL.TestData.nuspec") -Destination $nuspecPath -Force + + Write-Host "Building test data NuGet. Input: $WorkingDirectory. Version: $Version" + & (Join-Path $PSScriptRoot "..\..\_deps\nuget.exe") pack $nuspecPath ` + -Properties "version=$Version" ` + -OutputDirectory $OutputDirectory + if ($LASTEXITCODE -ne 0) { + throw "NuGet pack failed with exit code $LASTEXITCODE." + } +} +finally { + if (Test-Path -LiteralPath $WorkingDirectory) { + Remove-Item -LiteralPath $WorkingDirectory -Recurse -Force + } +} diff --git a/tools/test/images/wslc-registry/Dockerfile b/tools/test/images/wslc-registry/Dockerfile index 6d50848103..dcb50a918e 100644 --- a/tools/test/images/wslc-registry/Dockerfile +++ b/tools/test/images/wslc-registry/Dockerfile @@ -3,6 +3,6 @@ FROM registry:3 RUN apk add --no-cache apache2-utils COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +RUN sed -i 's/\r$//' /entrypoint.sh && chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/tools/test/pack-test-data.ps1 b/tools/test/pack-test-data.ps1 deleted file mode 100644 index f05e22b28e..0000000000 --- a/tools/test/pack-test-data.ps1 +++ /dev/null @@ -1,35 +0,0 @@ -<# -.SYNOPSIS - Helper to pack WSL test data nuget. -.PARAMETER InputDirectory - Directory containing arch-specific subdirectories (x64, arm64) with test data. -.PARAMETER Version - Nuget package version. -.PARAMETER OutputDirectory - Directory to place the packaged nuget file. Default to current working directory. -#> - -[CmdletBinding(PositionalBinding=$False, DefaultParameterSetName='vm')] -param ( - [Parameter(Mandatory = $true)][string]$InputDirectory, - [Parameter(Mandatory = $true)][string]$Version, - [string]$OutputDirectory = $PWD.Path -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version Latest - -if (-not (Test-Path -Path $InputDirectory -PathType Container)) { - throw("The path '$InputDirectory' is not an existing directory.") -} - -$hasArch = (Test-Path "$InputDirectory\x64") -or (Test-Path "$InputDirectory\arm64") -if (-not $hasArch) { - throw("The input directory must contain at least one architecture subdirectory (x64, arm64).") -} - -echo "Building test data nuget. Input: $InputDirectory. Version: $Version" - -Copy-Item -Path "$PSScriptRoot\Microsoft.WSL.TestData.nuspec" -Destination "$InputDirectory" -Force - -& "$PSScriptRoot\..\..\_deps\nuget.exe" pack "$InputDirectory\Microsoft.WSL.TestData.nuspec" -Properties "version=$Version" -OutputDirectory "$OutputDirectory" \ No newline at end of file