From 9a9030a3cf106c9f12a6fba469ad8cc7ce031c25 Mon Sep 17 00:00:00 2001 From: imiller31 <39311954+imiller31@users.noreply.github.com> Date: Wed, 8 Jul 2026 09:47:30 -0400 Subject: [PATCH 1/5] Add metalman RAID1 and HTTP boot support --- .github/workflows/smoke-metalman.yaml | 30 +- api/machina/v1alpha3/machine_types.go | 84 +- api/machina/v1alpha3/machine_types_test.go | 159 ++++ api/machina/v1alpha3/zz_generated.deepcopy.go | 25 + cmd/metalman/README.md | 2 +- .../crd/unbounded-cloud.io_machines.yaml | 39 +- docs/content/guides/pxe.md | 43 +- docs/content/reference/machina-crd.md | 45 +- hack/smoke-metalman.py | 700 ++++++++++++++-- images/netboot/Containerfile | 33 +- images/netboot/assets/grub.cfg.tmpl | 24 +- images/netboot/assets/init | 756 ++++++++++++++---- internal/metalman/commands/serve_pxe.go | 1 + internal/metalman/dhcp/dhcp.go | 38 +- internal/metalman/dhcp/dhcp_test.go | 87 ++ internal/metalman/netboot/http.go | 10 +- internal/metalman/netboot/netboot.go | 46 ++ internal/metalman/netboot/netboot_test.go | 97 ++- 18 files changed, 1943 insertions(+), 276 deletions(-) diff --git a/.github/workflows/smoke-metalman.yaml b/.github/workflows/smoke-metalman.yaml index efbaa9341..7fbcf3715 100644 --- a/.github/workflows/smoke-metalman.yaml +++ b/.github/workflows/smoke-metalman.yaml @@ -11,9 +11,11 @@ on: - internal/metalman/** - api/machina/v1alpha3/** - deploy/machina/crd/** + - hack/smoke-metalman.py - images/agent-ubuntu2404/** - images/host-ubuntu2404/** - images/netboot/** + - .github/workflows/smoke-metalman.yaml push: branches: [main] paths: @@ -22,9 +24,11 @@ on: - internal/metalman/** - api/machina/v1alpha3/** - deploy/machina/crd/** + - hack/smoke-metalman.py - images/agent-ubuntu2404/** - images/host-ubuntu2404/** - images/netboot/** + - .github/workflows/smoke-metalman.yaml schedule: - cron: "0 7 * * *" workflow_dispatch: @@ -34,8 +38,22 @@ permissions: jobs: smoke: + name: smoke (${{ matrix.install-mode }}, ${{ matrix.boot-protocol }}) runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - install-mode: Raw + boot-protocol: PXE + - install-mode: RAID1 + boot-protocol: PXE + - install-mode: RAID1 + boot-protocol: HTTP + env: + SMOKE_INSTALL_MODE: ${{ matrix.install-mode }} + SMOKE_BOOT_PROTOCOL: ${{ matrix.boot-protocol }} steps: - name: Free disk space run: | @@ -82,6 +100,16 @@ jobs: dnsmasq qemu-kvm libvirt-daemon-system libvirt-clients virtinst \ ovmf qemu-utils swtpm swtpm-tools \ python3-pip python3-yaml cpio iptables + EXTRA_PKGS=() + if [ "${SMOKE_INSTALL_MODE}" = "RAID1" ] || [ "${SMOKE_BOOT_PROTOCOL}" = "HTTP" ]; then + EXTRA_PKGS+=(libguestfs-tools) + fi + if [ "${SMOKE_INSTALL_MODE}" = "RAID1" ]; then + EXTRA_PKGS+=(zstd) + fi + if [ "${#EXTRA_PKGS[@]}" -gt 0 ]; then + sudo apt-get install -y --no-install-recommends "${EXTRA_PKGS[@]}" + fi sudo systemctl start libvirtd wait "$PIP_PID" diff --git a/api/machina/v1alpha3/machine_types.go b/api/machina/v1alpha3/machine_types.go index 0393cae24..6e78ce9f7 100644 --- a/api/machina/v1alpha3/machine_types.go +++ b/api/machina/v1alpha3/machine_types.go @@ -4,6 +4,8 @@ package v1alpha3 import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -273,8 +275,9 @@ type RedfishSpec struct { // PXESpec defines PXE boot configuration for a Machine. type PXESpec struct { - // Image is an OCI image reference containing the machine disk image. - // The image must contain /disk/disk.img.gz. + // Image is an OCI image reference containing machine install artifacts. + // Raw installs require /disk/disk.img.gz. RAID1 installs require + // /disk/rootfs.tar.zst and /disk/esp.tar.zst. // Example: "ghcr.io/azure/host-ubuntu2404:v1" // +kubebuilder:validation:Required Image string `json:"image"` @@ -307,8 +310,9 @@ type PXESpec struct { NetbootPullSecretRef *NamespacedSecretReference `json:"netbootPullSecretRef,omitempty"` // BootProtocol selects how metalman should trigger network boot for - // repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - // HTTP boot with a URL derived from the netboot image metadata. + // repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses UEFI HTTP boot: + // Redfish-capable machines receive a one-time HTTP boot URL, and DHCP + // HTTPClient firmware requests receive an absolute HTTP bootfile URL. // +kubebuilder:validation:Enum=PXE;HTTP // +kubebuilder:default=PXE // +optional @@ -324,6 +328,11 @@ type PXESpec struct { // +optional TargetDisk string `json:"targetDisk,omitempty"` + // Install configures how the PXE installer writes the machine image. + // When omitted, the installer uses the legacy raw disk image flow. + // +optional + Install *PXEInstallSpec `json:"install,omitempty"` + // Redfish configures optional Redfish BMC access. // +optional Redfish *RedfishSpec `json:"redfish,omitempty"` @@ -334,10 +343,29 @@ type PXESpec struct { CloudInit *CloudInitSpec `json:"cloudInit,omitempty"` } +// PXEInstallSpec configures how the PXE installer writes a machine image. +// +kubebuilder:validation:XValidation:rule="!has(self.mode) || self.mode != 'RAID1' || (has(self.targetDisks) && size(self.targetDisks) == 2)",message="RAID1 install requires exactly two targetDisks" +type PXEInstallSpec struct { + // Mode selects the install workflow used by the PXE initrd. + // Raw writes /disk/disk.img.gz directly to one disk. RAID1 creates a + // mirrored root disk from rootfs and ESP tarball machine artifacts. + // +kubebuilder:validation:Enum=Raw;RAID1 + // +kubebuilder:default=Raw + // +optional + Mode string `json:"mode,omitempty"` + + // TargetDisks are explicit whole-disk device paths used by the selected + // install mode. RAID1 requires exactly two paths. Raw uses the first path + // when set and otherwise falls back to targetDisk or automatic selection. + // Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... + // +optional + TargetDisks []string `json:"targetDisks,omitempty"` +} + const ( // PXEBootProtocolPXE uses DHCP/TFTP PXE boot. PXEBootProtocolPXE = "PXE" - // PXEBootProtocolHTTP uses Redfish UEFI HTTP boot. + // PXEBootProtocolHTTP uses UEFI HTTP boot. PXEBootProtocolHTTP = "HTTP" // PXEArchitectureAMD64 is the x86_64 target architecture for PXE boot. PXEArchitectureAMD64 = "amd64" @@ -347,6 +375,12 @@ const ( DefaultPXEArchitecture = PXEArchitectureAMD64 // DefaultPXEBootProtocol is used when spec.pxe.bootProtocol is omitted. DefaultPXEBootProtocol = PXEBootProtocolPXE + // PXEInstallModeRaw writes a raw disk image to a single target disk. + PXEInstallModeRaw = "Raw" + // PXEInstallModeRAID1 creates a mirrored root disk from machine artifacts. + PXEInstallModeRAID1 = "RAID1" + // DefaultPXEInstallMode is used when spec.pxe.install.mode is omitted. + DefaultPXEInstallMode = PXEInstallModeRaw ) // TargetArchitecture returns the effective PXE target architecture. @@ -367,6 +401,46 @@ func (p *PXESpec) TargetBootProtocol() string { return p.BootProtocol } +// TargetInstallMode returns the effective PXE install mode. +func (p *PXESpec) TargetInstallMode() string { + if p == nil || p.Install == nil || p.Install.Mode == "" { + return DefaultPXEInstallMode + } + + return p.Install.Mode +} + +// InstallTargetDisks returns the effective installer target disk list. +func (p *PXESpec) InstallTargetDisks() []string { + if p == nil { + return nil + } + + if p.Install != nil && len(p.Install.TargetDisks) > 0 { + return p.Install.TargetDisks + } + + if p.TargetDisk != "" { + return []string{p.TargetDisk} + } + + return nil +} + +// ValidateInstall returns an error when the PXE install configuration cannot +// be rendered into a safe installer command line. +func (p *PXESpec) ValidateInstall() error { + switch p.TargetInstallMode() { + case PXEInstallModeRAID1: + disks := p.InstallTargetDisks() + if len(disks) != 2 { + return fmt.Errorf("RAID1 install requires exactly two targetDisks, got %d", len(disks)) + } + } + + return nil +} + // CloudInitSpec defines cloud-init customization for PXE-booted machines. // Cloud-init merges vendor-data (managed by unbounded-kube) with user-data // (managed by the cluster operator). This spec controls the user-data diff --git a/api/machina/v1alpha3/machine_types_test.go b/api/machina/v1alpha3/machine_types_test.go index df6746e7f..3e2a94e5c 100644 --- a/api/machina/v1alpha3/machine_types_test.go +++ b/api/machina/v1alpha3/machine_types_test.go @@ -27,3 +27,162 @@ func TestKubernetesSpecOmitsUnsetBootstrapTokenRef(t *testing.T) { t.Fatalf("marshaled KubernetesSpec = %s, want bootstrapTokenRef omitted", data) } } + +func TestPXESpecTargetInstallMode(t *testing.T) { + tests := []struct { + name string + pxe *PXESpec + want string + }{ + { + name: "nil pxe defaults raw", + want: PXEInstallModeRaw, + }, + { + name: "nil install defaults raw", + pxe: &PXESpec{}, + want: PXEInstallModeRaw, + }, + { + name: "empty install mode defaults raw", + pxe: &PXESpec{Install: &PXEInstallSpec{}}, + want: PXEInstallModeRaw, + }, + { + name: "explicit raid1", + pxe: &PXESpec{Install: &PXEInstallSpec{Mode: PXEInstallModeRAID1}}, + want: PXEInstallModeRAID1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.pxe.TargetInstallMode(); got != tt.want { + t.Fatalf("TargetInstallMode() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestPXESpecInstallTargetDisks(t *testing.T) { + tests := []struct { + name string + pxe *PXESpec + want []string + }{ + { + name: "nil pxe", + }, + { + name: "legacy target disk", + pxe: &PXESpec{TargetDisk: "/dev/disk/by-id/os"}, + want: []string{"/dev/disk/by-id/os"}, + }, + { + name: "install target disks override legacy target disk", + pxe: &PXESpec{ + TargetDisk: "/dev/disk/by-id/legacy", + Install: &PXEInstallSpec{ + TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}, + }, + }, + want: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.pxe.InstallTargetDisks() + if len(got) != len(tt.want) { + t.Fatalf("InstallTargetDisks() = %v, want %v", got, tt.want) + } + + for i := range tt.want { + if got[i] != tt.want[i] { + t.Fatalf("InstallTargetDisks() = %v, want %v", got, tt.want) + } + } + }) + } +} + +func TestPXESpecValidateInstall(t *testing.T) { + tests := []struct { + name string + pxe *PXESpec + wantErr bool + }{ + { + name: "nil pxe defaults raw", + }, + { + name: "raw allows no disks", + pxe: &PXESpec{}, + }, + { + name: "raw allows explicit disk", + pxe: &PXESpec{ + Install: &PXEInstallSpec{ + TargetDisks: []string{"/dev/disk/by-id/os"}, + }, + }, + }, + { + name: "raid1 requires disks", + pxe: &PXESpec{ + Install: &PXEInstallSpec{Mode: PXEInstallModeRAID1}, + }, + wantErr: true, + }, + { + name: "raid1 rejects one disk", + pxe: &PXESpec{ + Install: &PXEInstallSpec{ + Mode: PXEInstallModeRAID1, + TargetDisks: []string{"/dev/disk/by-id/a"}, + }, + }, + wantErr: true, + }, + { + name: "raid1 accepts two disks", + pxe: &PXESpec{ + Install: &PXEInstallSpec{ + Mode: PXEInstallModeRAID1, + TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}, + }, + }, + }, + { + name: "raid1 rejects three disks", + pxe: &PXESpec{ + Install: &PXEInstallSpec{ + Mode: PXEInstallModeRAID1, + TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b", "/dev/disk/by-id/c"}, + }, + }, + wantErr: true, + }, + { + name: "raid1 does not use legacy single disk", + pxe: &PXESpec{ + TargetDisk: "/dev/disk/by-id/legacy", + Install: &PXEInstallSpec{Mode: PXEInstallModeRAID1}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.pxe.ValidateInstall() + if tt.wantErr && err == nil { + t.Fatalf("ValidateInstall() succeeded, want error") + } + + if !tt.wantErr && err != nil { + t.Fatalf("ValidateInstall() error = %v", err) + } + }) + } +} diff --git a/api/machina/v1alpha3/zz_generated.deepcopy.go b/api/machina/v1alpha3/zz_generated.deepcopy.go index 6807cd37c..9415c6ca2 100644 --- a/api/machina/v1alpha3/zz_generated.deepcopy.go +++ b/api/machina/v1alpha3/zz_generated.deepcopy.go @@ -1014,6 +1014,26 @@ func (in *OperationsStatus) DeepCopy() *OperationsStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PXEInstallSpec) DeepCopyInto(out *PXEInstallSpec) { + *out = *in + if in.TargetDisks != nil { + in, out := &in.TargetDisks, &out.TargetDisks + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PXEInstallSpec. +func (in *PXEInstallSpec) DeepCopy() *PXEInstallSpec { + if in == nil { + return nil + } + out := new(PXEInstallSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PXESpec) DeepCopyInto(out *PXESpec) { *out = *in @@ -1034,6 +1054,11 @@ func (in *PXESpec) DeepCopyInto(out *PXESpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Install != nil { + in, out := &in.Install, &out.Install + *out = new(PXEInstallSpec) + (*in).DeepCopyInto(*out) + } if in.Redfish != nil { in, out := &in.Redfish, &out.Redfish *out = new(RedfishSpec) diff --git a/cmd/metalman/README.md b/cmd/metalman/README.md index 569109e98..3c4563b64 100644 --- a/cmd/metalman/README.md +++ b/cmd/metalman/README.md @@ -187,7 +187,7 @@ metadata: spec: pxe: image: ghcr.io/azure/host-ubuntu2404:v1 - # Defaults to PXE. Set to HTTP to use Redfish UEFI HTTP boot. + # Defaults to PXE. Set to HTTP to use UEFI HTTP boot. bootProtocol: PXE # Optional. Recommended when the host has multiple disks. targetDisk: /dev/disk/by-id/example-os-disk diff --git a/deploy/machina/crd/unbounded-cloud.io_machines.yaml b/deploy/machina/crd/unbounded-cloud.io_machines.yaml index f96484975..ae00d886e 100644 --- a/deploy/machina/crd/unbounded-cloud.io_machines.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_machines.yaml @@ -332,8 +332,9 @@ spec: default: PXE description: |- BootProtocol selects how metalman should trigger network boot for - repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - HTTP boot with a URL derived from the netboot image metadata. + repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses UEFI HTTP boot: + Redfish-capable machines receive a one-time HTTP boot URL, and DHCP + HTTPClient firmware requests receive an absolute HTTP bootfile URL. enum: - PXE - HTTP @@ -396,10 +397,40 @@ spec: type: array image: description: |- - Image is an OCI image reference containing the machine disk image. - The image must contain /disk/disk.img.gz. + Image is an OCI image reference containing machine install artifacts. + Raw installs require /disk/disk.img.gz. RAID1 installs require + /disk/rootfs.tar.zst and /disk/esp.tar.zst. Example: "ghcr.io/azure/host-ubuntu2404:v1" type: string + install: + description: |- + Install configures how the PXE installer writes the machine image. + When omitted, the installer uses the legacy raw disk image flow. + properties: + mode: + default: Raw + description: |- + Mode selects the install workflow used by the PXE initrd. + Raw writes /disk/disk.img.gz directly to one disk. RAID1 creates a + mirrored root disk from rootfs and ESP tarball machine artifacts. + enum: + - Raw + - RAID1 + type: string + targetDisks: + description: |- + TargetDisks are explicit whole-disk device paths used by the selected + install mode. RAID1 requires exactly two paths. Raw uses the first path + when set and otherwise falls back to targetDisk or automatic selection. + Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... + items: + type: string + type: array + type: object + x-kubernetes-validations: + - message: RAID1 install requires exactly two targetDisks + rule: '!has(self.mode) || self.mode != ''RAID1'' || (has(self.targetDisks) + && size(self.targetDisks) == 2)' netbootImage: description: |- NetbootImage is an OCI image reference containing the PXE boot diff --git a/docs/content/guides/pxe.md b/docs/content/guides/pxe.md index bb0ed0cfa..80db3fe82 100644 --- a/docs/content/guides/pxe.md +++ b/docs/content/guides/pxe.md @@ -48,17 +48,23 @@ When `--dhcp-interface` is set, metalman binds to the interface for broadcast DH Metalman uses a machine image and a netboot image for each PXE repave. -- `spec.pxe.image` is the machine image. It contains `/disk/disk.img.gz`, a - gzip-compressed raw disk image written to the target disk. +- `spec.pxe.image` is the machine image. For the default `Raw` install mode, + it contains `/disk/disk.img.gz`, a gzip-compressed raw disk image written to + the target disk. For `RAID1`, it contains `/disk/rootfs.tar.zst` and + `/disk/esp.tar.zst`. - `spec.pxe.architecture` selects the target architecture (`amd64` or `arm64`) used when pulling machine and netboot image platform manifests. It defaults to `amd64`. +- `spec.pxe.install` selects the install workflow. When omitted, metalman uses + the legacy raw single-disk workflow. `RAID1` requires exactly two explicit + whole-disk paths in `spec.pxe.install.targetDisks`. - `spec.pxe.netbootImage` is the reusable PXE boot environment. It contains bootloaders, kernel, initrd, templates, metadata, and `unbounded-agent`. If omitted, Metalman uses the release-matched `--default-netboot-image`. - `spec.pxe.bootProtocol` selects the network boot trigger. `PXE` is the - default and uses DHCP/TFTP bootfile options. `HTTP` uses Redfish UEFI HTTP - boot and requires a Redfish block. + default and uses DHCP/TFTP bootfile options. `HTTP` uses UEFI HTTP boot. With + Redfish, metalman sets a one-time HTTP boot URL; DHCP `HTTPClient` firmware + requests also receive an absolute HTTP bootfile URL. Both images are standard OCI container images built `FROM scratch` with artifacts under `/disk/`. Files with a `.tmpl` suffix in the netboot image are @@ -66,6 +72,10 @@ Go templates rendered per-machine at serve time; other files are served verbatim. A `metadata.yaml` file in the netboot image provides image-level configuration such as `dhcpBootImageName` and `httpBootPath`. +At serve time, metalman keeps the legacy raw disk URL at `/disk.img.gz` and +serves v2 machine image artifacts under `/machine/`, for example +`/machine/rootfs.tar.zst` and `/machine/esp.tar.zst`. + Images are built, tagged, and pushed using standard container tooling: ```bash @@ -75,6 +85,11 @@ docker push ghcr.io/azure/host-ubuntu2404:v1 docker push ghcr.io/azure/netboot:v1 ``` +The default netboot image advertises shim for PXE/TFTP and GRUB for UEFI HTTP +boot. Keeping separate `dhcpBootImageName` and `httpBootPath` values lets HTTP +firmware fetch a single bootloader artifact from metalman without requiring +shim companion files. + Template context includes `.Machine`, `.ApiserverURL`, `.ServeURL`, `.KubernetesVersion`, and `.ClusterDNS`. See the [CRD Reference]({{< relref "/reference/machina-crd" >}}) for the full Machine spec. @@ -94,12 +109,18 @@ spec: pxe: image: ghcr.io/azure/host-ubuntu2404:v1 architecture: amd64 - # Optional. Defaults to PXE. Set to HTTP for Redfish UEFI HTTP boot. + # Optional. Defaults to PXE. Set to HTTP for UEFI HTTP boot. bootProtocol: PXE # Optional. Omit to use Metalman's default netboot image. netbootImage: ghcr.io/azure/netboot:v1 - # Optional. Recommended on hosts with multiple disks. - targetDisk: /dev/disk/by-id/example-os-disk + # Raw mode only. Recommended on hosts with multiple disks. + # targetDisk: /dev/disk/by-id/example-os-disk + # Optional. Omit for the legacy Raw single-disk install mode. + install: + mode: RAID1 + targetDisks: + - /dev/disk/by-id/example-os-disk-a + - /dev/disk/by-id/example-os-disk-b dhcpLeases: - ipv4: "10.10.0.50" mac: "aa:bb:cc:dd:ee:ff" @@ -173,13 +194,13 @@ If the referenced ConfigMap does not exist, metalman falls back to the default m ## Boot Flow -1. **Machine CR created.** The Redfish reconciler sets the boot device and power-cycles the server (ForceOff → On). For `bootProtocol: PXE`, it selects PXE boot. For `bootProtocol: HTTP`, it sets a one-time Redfish UEFI HTTP boot URL from the netboot image metadata. -2. **Network boot.** DHCP assigns the static IP by MAC. In PXE mode, DHCP also advertises the TFTP bootfile and TFTP serves `shimx64.efi`. In HTTP mode, the firmware downloads the Redfish-supplied URL from metalman's HTTP server. +1. **Machine CR created.** If Redfish is configured, the Redfish reconciler sets the boot device and power-cycles the server (ForceOff → On). For `bootProtocol: PXE`, it selects PXE boot. For `bootProtocol: HTTP`, it sets a one-time Redfish UEFI HTTP boot URL from the netboot image metadata. Without Redfish, the machine must be booted into the selected network boot path out of band. +2. **Network boot.** DHCP assigns the static IP by MAC. In PXE mode, DHCP also advertises the TFTP bootfile and TFTP serves `shimx64.efi`. In HTTP mode, firmware either downloads the Redfish-supplied URL or, when it identifies as `HTTPClient` during DHCP, receives an absolute HTTP bootfile URL from metalman's DHCP server. 3. **GRUB decision.** A rendered `grub.cfg` (from a `.tmpl` file in the netboot image) checks `repaveCounter` against status: if counter is ahead, boot the PXE installer; otherwise chainload the local OS. When a Machine has multiple DHCP leases, metalman renders the lease matching the request source IP and passes that lease's MAC as `unbounded.boot_mac`. 4. **Installer (initrd overlay).** An init script in the initrd: - Loads storage and network drivers, selects the provisioning NIC by MAC, and configures the static IP from kernel cmdline. - - Downloads the gzip-compressed raw disk image from the machine image over HTTP (retries up to 120 times). - - Writes the image to `spec.pxe.targetDisk` when set, otherwise to an automatically selected block device. + - In `Raw` mode, downloads the gzip-compressed raw disk image from the machine image over HTTP and writes it to `spec.pxe.targetDisk`, `spec.pxe.install.targetDisks[0]`, or an automatically selected block device. + - In `RAID1` mode, requires two explicit whole-disk paths, creates an ESP on each disk, creates a mirrored root array, extracts the rootfs and ESP artifacts, writes `mdadm.conf` and `fstab`, and refreshes host boot artifacts. The host image must include mdadm, initramfs, and GRUB tooling or the installer fails before disabling PXE. - Mounts the root filesystem and injects cloud-init config and the agent configuration. - Calls `/pxe/disable` on metalman to signal completion, then reboots. 5. **First boot.** cloud-init downloads the `unbounded-agent` binary from metalman and runs `unbounded-agent start`. diff --git a/docs/content/reference/machina-crd.md b/docs/content/reference/machina-crd.md index 797dd079f..7b4765d22 100644 --- a/docs/content/reference/machina-crd.md +++ b/docs/content/reference/machina-crd.md @@ -52,17 +52,20 @@ PXE boot configuration consumed by the metalman controller. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| | `pxe` | PXESpec | No | - | PXE boot configuration. | -| `pxe.image` | string | Yes | - | OCI machine image reference containing `/disk/disk.img.gz` (e.g. `"ghcr.io/azure/host-ubuntu2404:v1"`). | +| `pxe.image` | string | Yes | - | OCI machine image reference. `Raw` installs read `/disk/disk.img.gz`; `RAID1` installs read `/disk/rootfs.tar.zst` and `/disk/esp.tar.zst`. | | `pxe.architecture` | string | No | `amd64` | Target CPU architecture for PXE boot artifacts and machine images. Allowed values: `amd64`, `arm64`. | | `pxe.netbootImage` | string | No | Metalman default | OCI netboot image reference containing PXE boot artifacts. | -| `pxe.bootProtocol` | string | No | `PXE` | Network boot trigger protocol for repaves. `PXE` uses DHCP/TFTP bootfile options. `HTTP` uses Redfish UEFI HTTP boot with a URL derived from the netboot image metadata. Allowed values: `PXE`, `HTTP`. | +| `pxe.bootProtocol` | string | No | `PXE` | Network boot trigger protocol for repaves. `PXE` uses DHCP/TFTP bootfile options. `HTTP` uses UEFI HTTP boot with a URL derived from the netboot image metadata. Redfish-capable machines receive a one-time HTTP boot URL; DHCP `HTTPClient` firmware requests receive an absolute HTTP bootfile URL. Allowed values: `PXE`, `HTTP`. | | `pxe.dhcpLeases` | []DHCPLease | No | - | Static DHCP leases served during PXE boot. | | `pxe.dhcpLeases[].ipv4` | string | Yes | - | Static IPv4 address to assign. | | `pxe.dhcpLeases[].mac` | string | Yes | - | NIC MAC address (matched case-insensitively). | | `pxe.dhcpLeases[].subnetMask` | string | Yes | - | Subnet mask. | | `pxe.dhcpLeases[].gateway` | string | Yes | - | Default gateway. | | `pxe.dhcpLeases[].dns` | []string | No | - | DNS server addresses. | -| `pxe.targetDisk` | string | No | Installer-selected | Block device the installer writes the machine image to, such as `/dev/nvme0n1` or `/dev/disk/by-id/...`. When omitted, the initrd selects a disk automatically. | +| `pxe.targetDisk` | string | No | Installer-selected | Legacy raw-mode block device override, such as `/dev/nvme0n1` or `/dev/disk/by-id/...`. When omitted in `Raw` mode, the initrd selects a disk automatically. | +| `pxe.install` | PXEInstallSpec | No | `Raw` | Installer workflow configuration. Omit for the legacy raw single-disk workflow. | +| `pxe.install.mode` | string | No | `Raw` | Install workflow. Allowed values: `Raw`, `RAID1`. | +| `pxe.install.targetDisks` | []string | No | - | Explicit whole-disk device paths. `RAID1` requires exactly two paths and does not fall back to automatic disk selection. `Raw` uses the first path when set. | | `pxe.redfish` | RedfishSpec | No | - | BMC access via the Redfish API. | | `pxe.redfish.url` | string | Yes | - | Redfish endpoint URL. | | `pxe.redfish.username` | string | Yes | - | Redfish username. | @@ -406,15 +409,21 @@ spec: ## PXE OCI Images Metalman uses a machine image and a netboot image for PXE repaves. The machine -image is referenced by `spec.pxe.image` and contains `/disk/disk.img.gz`. The -netboot image is referenced by `spec.pxe.netbootImage`, or by Metalman's default -when that field is omitted, and contains the reusable PXE boot environment. -`spec.pxe.architecture` selects the OCI platform manifest to pull for both -images and defaults to `amd64`. +image is referenced by `spec.pxe.image`. In `Raw` mode it contains +`/disk/disk.img.gz`. In `RAID1` mode it contains `/disk/rootfs.tar.zst` and +`/disk/esp.tar.zst`, with `/disk/install.yaml` reserved for future metadata. +The netboot image is referenced by `spec.pxe.netbootImage`, or by Metalman's +default when that field is omitted, and contains the reusable PXE boot +environment. `spec.pxe.architecture` selects the OCI platform manifest to pull +for both images and defaults to `amd64`. Both images are standard OCI container images built `FROM scratch` with artifacts under `/disk/`. This follows the kubevirt containerDisk convention. +Metalman serves the legacy raw artifact at `/disk.img.gz` and serves v2 machine +image artifacts under `/machine/`, for example `/machine/rootfs.tar.zst` and +`/machine/esp.tar.zst`. + Files with a `.tmpl` suffix in the netboot image are Go templates rendered per-machine at serve time; other files are served verbatim. A `metadata.yaml` file in the netboot image provides image-level configuration such as @@ -436,12 +445,19 @@ Templates receive the following data object: | `.ServeURL` | string | External metalman HTTP URL. | | `.KubernetesVersion` | string | Resolved Kubernetes version for the machine. | | `.ClusterDNS` | string | Cluster DNS service IP. | +| `.InstallMode` | string | Effective PXE install mode. Defaults to `Raw`. | +| `.InstallKernelArgs` | []string | Kernel arguments that pass install mode and target disks to the initrd. | The default netboot template passes `.BootLease.MAC` as `unbounded.boot_mac`. The installer initrd uses that MAC address to configure the provisioning interface instead of relying on kernel interface names such as `eth0`. -If `spec.pxe.targetDisk` is set, the template passes it as `unbounded.disk`; -otherwise the installer falls back to automatic disk selection. +In `Raw` mode, the template passes a configured disk as `unbounded.disk` and +the installer falls back to automatic disk selection when no disk is configured. +In `RAID1` mode, the template passes `unbounded.disk0` and `unbounded.disk1` +from `spec.pxe.install.targetDisks`; the installer fails closed when either +disk is missing or invalid. RAID1 host images must include mdadm, +update-initramfs, and GRUB tooling so the installer can refresh boot artifacts +for the mirrored root before disabling PXE. ### Building images @@ -461,15 +477,18 @@ reusable netboot image Containerfile. ```yaml dhcpBootImageName: shimx64.efi -httpBootPath: shimx64.efi +httpBootPath: grubx64.efi ``` The `dhcpBootImageName` field specifies the boot filename included in DHCP responses (option 67) for `spec.pxe.bootProtocol: PXE`. The `httpBootPath` field specifies the file path, relative to metalman's HTTP -artifact server, used for `spec.pxe.bootProtocol: HTTP`. If `httpBootPath` is -omitted, metalman falls back to `dhcpBootImageName` for the UEFI HTTP boot URL. +artifact server, used for `spec.pxe.bootProtocol: HTTP`. The default netboot +image uses shim for PXE/TFTP and GRUB for HTTP boot because some UEFI HTTP +clients fetch shim companion files that are not part of Ubuntu's netboot +artifact set. If `httpBootPath` is omitted, metalman falls back to +`dhcpBootImageName` for the UEFI HTTP boot URL. --- diff --git a/hack/smoke-metalman.py b/hack/smoke-metalman.py index 900c9c51f..e28e3aa71 100755 --- a/hack/smoke-metalman.py +++ b/hack/smoke-metalman.py @@ -6,6 +6,7 @@ import atexit import base64 +import gzip import json import os import signal @@ -22,7 +23,9 @@ from typing import Any REPO_ROOT = Path(__file__).resolve().parent.parent -TMPDIR = Path(tempfile.mkdtemp()) +TMP_ROOT = Path(os.environ.get("SMOKE_TMP_ROOT", tempfile.gettempdir())) +TMP_ROOT.mkdir(parents=True, exist_ok=True) +TMPDIR = Path(tempfile.mkdtemp(prefix="unbounded-metal-smoke-", dir=TMP_ROOT)) os.chmod(TMPDIR, 0o755) SITE = "smoke" NODE_NAME = "smoke-node" @@ -52,7 +55,8 @@ AGENT_DOWNLOAD_URL = f"http://{SERVER_IP}:{AGENT_DOWNLOAD_PORT}/{AGENT_TARBALL.name}" REGISTRY_PORT = 5555 REGISTRY_CONTAINER = "unbounded-smoke-registry" -IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/host-ubuntu2404:smoke" +RAW_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/host-ubuntu2404:smoke" +RAID_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/host-ubuntu2404-raid:smoke" NETBOOT_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/netboot:smoke" AGENT_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/agent-ubuntu2404:smoke" # The agent runs inside a VM on an isolated libvirt network. "localhost" inside @@ -64,6 +68,27 @@ KUBECTL_UNBOUNDED = REPO_ROOT / "bin" / "kubectl-unbounded" SERIAL_SOCK = TMPDIR / "console.sock" QGA_SOCK = TMPDIR / "qga.sock" +INSTALL_MODE = os.environ.get("SMOKE_INSTALL_MODE", "Raw") +if INSTALL_MODE.lower() == "raw": + INSTALL_MODE = "Raw" +elif INSTALL_MODE.lower() == "raid1": + INSTALL_MODE = "RAID1" +else: + print(f"FAIL: unsupported SMOKE_INSTALL_MODE={INSTALL_MODE!r}; expected Raw or RAID1", file=sys.stderr) + sys.exit(1) +BOOT_PROTOCOL = os.environ.get("SMOKE_BOOT_PROTOCOL", "PXE") +if BOOT_PROTOCOL.lower() == "pxe": + BOOT_PROTOCOL = "PXE" +elif BOOT_PROTOCOL.lower() == "http": + BOOT_PROTOCOL = "HTTP" +else: + print(f"FAIL: unsupported SMOKE_BOOT_PROTOCOL={BOOT_PROTOCOL!r}; expected PXE or HTTP", file=sys.stderr) + sys.exit(1) +IMAGE_NAME = RAID_IMAGE_NAME if INSTALL_MODE == "RAID1" else RAW_IMAGE_NAME +RAID_DISK_A_SERIAL = "unbounded-raid-a" +RAID_DISK_B_SERIAL = "unbounded-raid-b" +RAID_DISK_A_PATH = f"/dev/disk/by-id/virtio-{RAID_DISK_A_SERIAL}" +RAID_DISK_B_PATH = f"/dev/disk/by-id/virtio-{RAID_DISK_B_SERIAL}" # The nspawn machine name used by the agent (must match the constant in # cmd/agent/internal/goalstates/constants.go - NSpawnMachineKube1). NSPAWN_MACHINE = "kube1" @@ -74,6 +99,14 @@ _procs: list[subprocess.Popen[Any]] = [] +_TRANSIENT_QGA_ERRORS = ( + "domain is not running", + "guest agent is not responding", + "qemu guest agent is not connected", + "qemu guest agent is not configured", + "guest agent is not available", +) + def log(msg: str) -> None: print(f"==> {msg}", file=sys.stderr) @@ -88,6 +121,21 @@ def die(msg: str) -> None: sys.exit(1) +def find_ovmf_firmware() -> tuple[Path, Path]: + candidates = [ + (Path("/usr/share/OVMF/OVMF_CODE_4M.fd"), Path("/usr/share/OVMF/OVMF_VARS_4M.fd")), + (Path("/usr/share/OVMF/OVMF_CODE.fd"), Path("/usr/share/OVMF/OVMF_VARS.fd")), + (Path("/usr/share/edk2/x64/OVMF_CODE.4m.fd"), Path("/usr/share/edk2/x64/OVMF_VARS.4m.fd")), + (Path("/usr/share/edk2/x64/OVMF_CODE.fd"), Path("/usr/share/edk2/x64/OVMF_VARS.fd")), + (Path("/usr/share/edk2/ovmf/OVMF_CODE.fd"), Path("/usr/share/edk2/ovmf/OVMF_VARS.fd")), + ] + for code, vars_template in candidates: + if code.is_file() and vars_template.is_file(): + return code, vars_template + + die("could not find OVMF firmware; install ovmf or edk2-ovmf") + + def run(args: list[str], **kw: Any) -> subprocess.CompletedProcess[str]: return subprocess.run(args, check=True, **kw) @@ -161,6 +209,11 @@ def forward_console(sock_path: Path) -> None: time.sleep(1) +def _is_transient_qga_error(stderr: str) -> bool: + err = stderr.lower() + return any(marker in err for marker in _TRANSIENT_QGA_ERRORS) + + def guest_exec(command: str, timeout: int = 30) -> tuple[int, str, str]: """Execute a command inside the VM via the QEMU guest agent. @@ -168,45 +221,75 @@ def guest_exec(command: str, timeout: int = 30) -> tuple[int, str, str]: to be configured on the VM and the qemu-guest-agent service running inside the guest. """ - exec_req = json.dumps({ - "execute": "guest-exec", - "arguments": { - "path": "/bin/bash", - "arg": ["-c", command], - "capture-output": True, - }, - }) - result = subprocess.run( - [*VIRSH, "qemu-agent-command", VM_NAME, exec_req], - capture_output=True, text=True, timeout=10, - ) - if result.returncode != 0: - raise RuntimeError(f"guest-exec failed: {result.stderr.strip()}") - pid = json.loads(result.stdout)["return"]["pid"] - - # Poll guest-exec-status until the process exits. deadline = time.monotonic() + timeout - while True: - status_req = json.dumps({ - "execute": "guest-exec-status", - "arguments": {"pid": pid}, + last_error = "" + + while time.monotonic() < deadline: + if not _vm_is_running(): + time.sleep(1) + continue + + exec_req = json.dumps({ + "execute": "guest-exec", + "arguments": { + "path": "/bin/bash", + "arg": ["-c", command], + "capture-output": True, + }, }) result = subprocess.run( - [*VIRSH, "qemu-agent-command", VM_NAME, status_req], - capture_output=True, text=True, timeout=10, + [*VIRSH, "qemu-agent-command", VM_NAME, exec_req], + capture_output=True, text=True, + timeout=max(1, min(10, deadline - time.monotonic())), ) if result.returncode != 0: - raise RuntimeError(f"guest-exec-status failed: {result.stderr.strip()}") - status = json.loads(result.stdout)["return"] - if status.get("exited"): - exit_code = status.get("exitcode", -1) - stdout = base64.b64decode(status.get("out-data", "")).decode("utf-8", errors="replace") - stderr = base64.b64decode(status.get("err-data", "")).decode("utf-8", errors="replace") - return exit_code, stdout, stderr - if time.monotonic() > deadline: - raise TimeoutError(f"guest-exec pid {pid} did not exit within {timeout}s") - time.sleep(0.5) + last_error = result.stderr.strip() + if _is_transient_qga_error(last_error): + time.sleep(1) + continue + raise RuntimeError(f"guest-exec failed: {last_error}") + + pid = json.loads(result.stdout)["return"]["pid"] + + # Poll guest-exec-status until the process exits. If the guest + # reboots while the command is active, restart the command. + restart_command = False + while time.monotonic() < deadline: + status_req = json.dumps({ + "execute": "guest-exec-status", + "arguments": {"pid": pid}, + }) + result = subprocess.run( + [*VIRSH, "qemu-agent-command", VM_NAME, status_req], + capture_output=True, text=True, + timeout=max(1, min(10, deadline - time.monotonic())), + ) + if result.returncode != 0: + last_error = result.stderr.strip() + if _is_transient_qga_error(last_error): + restart_command = True + break + raise RuntimeError(f"guest-exec-status failed: {last_error}") + status = json.loads(result.stdout)["return"] + if status.get("exited"): + exit_code = status.get("exitcode", -1) + stdout = base64.b64decode(status.get("out-data", "")).decode("utf-8", errors="replace") + stderr = base64.b64decode(status.get("err-data", "")).decode("utf-8", errors="replace") + return exit_code, stdout, stderr + + time.sleep(0.5) + + if restart_command: + time.sleep(1) + continue + + break + + if last_error: + raise TimeoutError(f"guest-exec did not complete within {timeout}s; last error: {last_error}") + + raise TimeoutError(f"guest-exec did not complete within {timeout}s") def collect_debug_logs() -> None: """Use the QEMU guest agent to dump kubelet and agent debug information. @@ -219,6 +302,11 @@ def collect_debug_logs() -> None: ("resolv.conf", "cat /etc/resolv.conf"), ("ip addr", "ip -4 addr show"), ("ip route", "ip route show"), + ("block devices", "lsblk -o NAME,PATH,SIZE,FSTYPE,TYPE,MOUNTPOINTS"), + ("fstab", "cat /etc/fstab"), + ("mdstat", "cat /proc/mdstat 2>/dev/null || true"), + ("mdadm detail", "mdadm --detail /dev/md/unbounded-root 2>/dev/null || mdadm --detail /dev/md* 2>/dev/null || true"), + ("efibootmgr", "efibootmgr -v 2>/dev/null || true"), ("dns test (dl.k8s.io)", "timeout 5 getent hosts dl.k8s.io || echo 'DNS FAILED'"), ("curl test (dl.k8s.io)", "timeout 10 curl -sS -o /dev/null -w '%{http_code}' https://dl.k8s.io/ || echo 'CURL FAILED'"), ("dns test (github.com)", "timeout 5 getent hosts github.com || echo 'DNS FAILED'"), @@ -395,6 +483,7 @@ def cleanup() -> None: # Remove iptables rules that were added for VM ↔ kind connectivity. # Use check=False so these are best-effort (rules may not exist if setup # failed before they were inserted). + run_quiet(["sudo", "iptables", "-D", "INPUT", "-i", "virbr-smoke", "-j", "ACCEPT"], check=False) run_quiet(["sudo", "iptables", "-D", "FORWARD", "-i", "virbr-smoke", "-j", "ACCEPT"], check=False) run_quiet(["sudo", "iptables", "-D", "FORWARD", "-o", "virbr-smoke", "-j", "ACCEPT"], check=False) run_quiet(["sudo", "iptables", "-t", "raw", "-D", "PREROUTING", @@ -791,12 +880,53 @@ def assert_cloud_init_done(timeout: int = 900) -> None: return if status == "False" and reason == "Failed": die(f"Cloud-init failed: {message}") + if BOOT_PROTOCOL == "HTTP" and elapsed > 15 and not _vm_is_running(): + die("VM powered off during HTTP boot before CloudInitDone") if elapsed > 0 and elapsed % 30 == 0: log(f" ({elapsed}s) CloudInitDone status={status or 'not set'} reason={reason or 'not set'}") time.sleep(1) die(f"Timed out waiting for CloudInitDone condition on Machine '{NODE_NAME}'") +def assert_machine_condition( + condition_type: str, + expected_status: str, + expected_reason: str | None = None, + timeout: int = 300, +) -> None: + """Assert a Machine condition reaches the requested status and reason.""" + log(f" Waiting for Machine '{NODE_NAME}' condition {condition_type}={expected_status}...") + last: str | None = None + for elapsed in range(timeout): + check_procs() + result = subprocess.run( + [KUBECTL, "get", f"machines.{API_GROUP}", NODE_NAME, "-o", "json"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + machine = json.loads(result.stdout) + for cond in machine.get("status", {}).get("conditions", []): + if cond.get("type") != condition_type: + continue + status = cond.get("status", "") + reason = cond.get("reason", "") + current = f"{status}/{reason}" + if status == expected_status and (expected_reason is None or reason == expected_reason): + log(f" Machine condition {condition_type} is {current}") + return + if current != last: + last = current + log(f" ({elapsed}s) {condition_type}={current}") + elif elapsed % 15 == 0: + log(f" ({elapsed}s) Machine not readable yet: {result.stderr.strip()}") + time.sleep(1) + + if expected_reason is None: + die(f"Timed out waiting for Machine condition {condition_type}={expected_status}") + die(f"Timed out waiting for Machine condition {condition_type}={expected_status}/{expected_reason}") + + def wait_machine_operation_complete(name: str, timeout: int = 1800) -> None: log(f" Waiting for MachineOperation '{name}' to complete...") for elapsed in range(timeout): @@ -882,6 +1012,350 @@ def assert_log_contains(path: Path, needle: str) -> None: die(f"Expected {path} to contain {needle!r}") +def guestfs_env() -> dict[str, str]: + env = os.environ.copy() + env.setdefault("LIBGUESTFS_BACKEND", "direct") + env["TMPDIR"] = str(TMPDIR) + env["TEMP"] = str(TMPDIR) + env["TMP"] = str(TMPDIR) + return env + + +def require_tools(tools: list[str]) -> None: + missing = [tool for tool in tools if shutil.which(tool) is None] + if missing: + die("Missing required tool(s): " + ", ".join(missing)) + + +def docker_copy_from_image(image: str, src: str, dest: Path) -> None: + result = run( + ["docker", "create", image, "/copy-only"], + capture_output=True, + text=True, + ) + cid = result.stdout.strip() + if not cid: + die(f"docker create {image} did not return a container id") + + try: + run(["docker", "cp", f"{cid}:{src}", str(dest)]) + finally: + run_quiet(["docker", "rm", "-f", cid], check=False) + + +HTTP_BOOT_SELECTOR_SCRIPT = """\ +#!/bin/bash +set -euxo pipefail + +log_file=/var/log/metalman-httpboot-selector.log +touch "${log_file}" +chmod 0644 "${log_file}" +if [ -w /dev/ttyS0 ]; then + exec > >(tee -a "${log_file}" >/dev/ttyS0) 2>&1 +else + exec >>"${log_file}" 2>&1 +fi + +state_dir=/var/lib/metalman-httpboot-selector +state_file="${state_dir}/requested" +mkdir -p "${state_dir}" + +echo "=== metalman HTTP boot selector ===" +date -u + +if [ -f "${state_file}" ]; then + echo "=== returned to helper disk after requesting HTTP BootNext ===" + echo "Firmware HTTP boot likely failed or fell through to disk." + efibootmgr -v || true + systemctl poweroff --force + exit 0 +fi + +echo requested >"${state_file}" + +for attempt in $(seq 1 10); do + echo "=== efibootmgr before, attempt ${attempt} ===" + efibootmgr -v || true + entry="$(efibootmgr -v 2>/dev/null | sed -n 's/^Boot\\([0-9A-Fa-f]\\{4\\}\\).*UEFI HTTPv4.*/\\1/p' | head -n1)" + if [ -n "${entry}" ]; then + break + fi + sleep 1 +done + +if [ -z "${entry:-}" ]; then + echo "=== no UEFI HTTPv4 boot entry found ===" + systemctl poweroff --force + exit 1 +fi + +echo "=== setting BootNext to UEFI HTTPv4 Boot${entry} ===" +efibootmgr -n "${entry}" +efibootmgr -v || true +echo "=== rebooting into UEFI HTTPv4 ===" +systemctl reboot --force +exit 0 +""" + + +HTTP_BOOT_SELECTOR_SERVICE = """\ +[Unit] +Description=One-shot metalman UEFI HTTP boot selector +After=local-fs.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/metalman-httpboot-selector.sh + +[Install] +WantedBy=multi-user.target +""" + + +def prepare_http_boot_helper_disk(raw_image: str, dest: Path) -> None: + log(f"Preparing HTTP boot helper disk {dest.name}") + require_tools(["docker", "virt-customize", "qemu-img"]) + + workdir = TMPDIR / "http-boot-helper" + workdir.mkdir(parents=True, exist_ok=True) + disk_gz = workdir / "disk.img.gz" + disk_raw = workdir / "disk.img" + script = workdir / "metalman-httpboot-selector.sh" + service = workdir / "metalman-httpboot-selector.service" + + log(f" Extracting /disk/disk.img.gz from {raw_image}") + docker_copy_from_image(raw_image, "/disk/disk.img.gz", disk_gz) + with gzip.open(disk_gz, "rb") as src, disk_raw.open("wb") as dst: + shutil.copyfileobj(src, dst) + disk_gz.unlink() + + script.write_text(HTTP_BOOT_SELECTOR_SCRIPT, encoding="utf-8") + service.write_text(HTTP_BOOT_SELECTOR_SERVICE, encoding="utf-8") + + log(" Injecting one-shot efibootmgr selector") + run([ + "virt-customize", + "-a", str(disk_raw), + "--copy-in", f"{script}:/usr/local/sbin", + "--copy-in", f"{service}:/etc/systemd/system", + "--run-command", "chmod 0755 /usr/local/sbin/metalman-httpboot-selector.sh", + "--run-command", "mkdir -p /etc/cloud && touch /etc/cloud/cloud-init.disabled", + "--run-command", "rm -rf /var/lib/metalman-httpboot-selector", + "--run-command", "systemctl enable metalman-httpboot-selector.service", + ], env=guestfs_env()) + + log(" Converting helper disk to qcow2") + run(["qemu-img", "convert", "-f", "raw", "-O", "qcow2", str(disk_raw), str(dest)]) + run(["qemu-img", "resize", str(dest), "20G"]) + + +def virt_filesystems(image: Path) -> list[tuple[str, str, str]]: + result = run( + ["virt-filesystems", "-a", str(image), "--filesystems", "--long", "--no-title"], + capture_output=True, + text=True, + env=guestfs_env(), + ) + filesystems: list[tuple[str, str, str]] = [] + for line in result.stdout.splitlines(): + fields = line.split() + if len(fields) < 4: + continue + filesystems.append((fields[0], fields[2].lower(), fields[3])) + if not filesystems: + die(f"virt-filesystems found no filesystems in {image}") + return filesystems + + +def virt_has_file(image: Path, device: str, path: str) -> bool: + result = subprocess.run( + ["virt-cat", "-a", str(image), "-m", device, path], + stdout=DEVNULL, + stderr=DEVNULL, + env=guestfs_env(), + ) + return result.returncode == 0 + + +def virt_has_dir(image: Path, device: str, path: str) -> bool: + result = subprocess.run( + ["virt-ls", "-a", str(image), "-m", device, path], + stdout=DEVNULL, + stderr=DEVNULL, + env=guestfs_env(), + ) + return result.returncode == 0 + + +def virt_list(image: Path, device: str, path: str) -> list[str]: + result = subprocess.run( + ["virt-ls", "-a", str(image), "-m", device, path], + capture_output=True, + text=True, + env=guestfs_env(), + ) + if result.returncode != 0: + return [] + return result.stdout.splitlines() + + +def find_root_partition(image: Path, filesystems: list[tuple[str, str, str]]) -> str: + for device, vfs, _label in filesystems: + if vfs not in ("ext2", "ext3", "ext4", "xfs", "btrfs"): + continue + if virt_has_file(image, device, "/etc/os-release"): + return device + die(f"Could not identify root filesystem in {image}") + + +def find_esp_partition(image: Path, filesystems: list[tuple[str, str, str]]) -> str: + for device, vfs, _label in filesystems: + if vfs not in ("vfat", "fat", "msdos"): + continue + if virt_has_dir(image, device, "/EFI"): + return device + die(f"Could not identify EFI system partition in {image}") + + +def find_boot_partition(image: Path, filesystems: list[tuple[str, str, str]], root_device: str) -> str | None: + candidates: list[tuple[str, str, str]] = [] + for device, vfs, label in filesystems: + if device == root_device: + continue + if vfs in ("ext2", "ext3", "ext4", "xfs", "btrfs"): + candidates.append((device, vfs, label)) + + for device, _vfs, label in candidates: + if label.lower() == "boot": + return device + + for device, _vfs, _label in candidates: + names = virt_list(image, device, "/") + if any(name.startswith("vmlinuz-") for name in names) and "grub" in names: + return device + + return None + + +def virt_tar_zst( + image: Path, + mount_device: str, + dest: Path, + extra_mounts: list[tuple[str, str]] | None = None, +) -> None: + extra_mounts = extra_mounts or [] + mount_args = ["-m", f"{mount_device}:/"] + for device, mountpoint in extra_mounts: + mount_args.extend(["-m", f"{device}:{mountpoint}"]) + + mount_desc = ", ".join([mount_device] + [f"{device} at {mountpoint}" for device, mountpoint in extra_mounts]) + log(f" Exporting {mount_desc} to {dest.name}") + tar_proc = subprocess.Popen( + ["virt-tar-out", "--ro", "--no-sync", "-a", str(image), *mount_args, "/", "-"], + stdout=subprocess.PIPE, + env=guestfs_env(), + ) + if tar_proc.stdout is None: + die("virt-tar-out did not provide stdout") + zstd_proc = subprocess.Popen( + ["zstd", "-T0", "-q", "-f", "-o", str(dest), "-"], + stdin=tar_proc.stdout, + ) + tar_proc.stdout.close() + zstd_rc = zstd_proc.wait() + tar_rc = tar_proc.wait() + if tar_rc != 0 or zstd_rc != 0: + die(f"Failed to export {mount_device}: virt-tar-out={tar_rc}, zstd={zstd_rc}") + + +def prepare_raid_machine_image(raw_image: str, raid_image: str) -> None: + log("Preparing RAID1 machine image from host-ubuntu2404") + require_tools([ + "docker", "virt-customize", "virt-filesystems", "virt-cat", "virt-ls", + "virt-tar-out", "zstd", + ]) + + workdir = TMPDIR / "raid-machine-image" + workdir.mkdir(parents=True, exist_ok=True) + disk_gz = workdir / "disk.img.gz" + disk_raw = workdir / "disk.img" + + log(f" Extracting /disk/disk.img.gz from {raw_image}") + docker_copy_from_image(raw_image, "/disk/disk.img.gz", disk_gz) + with gzip.open(disk_gz, "rb") as src, disk_raw.open("wb") as dst: + shutil.copyfileobj(src, dst) + disk_gz.unlink() + + log(" Installing RAID boot dependencies into copied host image") + run([ + "virt-customize", + "-a", str(disk_raw), + "--run-command", "apt-get update", + "--install", "mdadm,grub-efi-amd64,grub-efi-amd64-bin,shim-signed,qemu-guest-agent", + "--run-command", "systemctl enable qemu-guest-agent || true", + ], env=guestfs_env()) + + filesystems = virt_filesystems(disk_raw) + root_part = find_root_partition(disk_raw, filesystems) + esp_part = find_esp_partition(disk_raw, filesystems) + boot_part = find_boot_partition(disk_raw, filesystems, root_part) + boot_mounts = [(boot_part, "/boot")] if boot_part else [] + log(f" Identified root={root_part} boot={boot_part or 'inline'} esp={esp_part}") + + virt_tar_zst(disk_raw, root_part, workdir / "rootfs.tar.zst", boot_mounts) + virt_tar_zst(disk_raw, esp_part, workdir / "esp.tar.zst") + (workdir / "install.yaml").write_text("version: 1\nmode: RAID1\n", encoding="utf-8") + (workdir / "Containerfile").write_text(textwrap.dedent("""\ + FROM scratch + COPY rootfs.tar.zst /disk/rootfs.tar.zst + COPY esp.tar.zst /disk/esp.tar.zst + COPY install.yaml /disk/install.yaml + """), encoding="utf-8") + + log(f" Building {raid_image}") + run(["docker", "build", "-t", raid_image, "-f", str(workdir / "Containerfile"), str(workdir)]) + + +def assert_raid1_install() -> None: + if INSTALL_MODE != "RAID1": + return + + log("Verifying RAID1 install inside guest") + wait_vm_state("running", timeout=180) + wait_guest_agent(timeout=300) + + command = textwrap.dedent(f"""\ + set -euo pipefail + root_src="$(findmnt -n -o SOURCE /)" + case "$root_src" in + /dev/md*|/dev/mapper/*) ;; + *) echo "root filesystem is not on md device: $root_src" >&2; exit 1 ;; + esac + test -e {RAID_DISK_A_PATH} + test -e {RAID_DISK_B_PATH} + grep -q unbounded-root /etc/mdadm/mdadm.conf + grep -q '/boot/efi-secondary' /etc/fstab + grep -q '\\[UU\\]' /proc/mdstat + mdadm --detail "$root_src" | grep -q 'Raid Level : raid1' + mdadm --detail "$root_src" | grep -q 'Raid Devices : 2' + findmnt /boot/efi >/dev/null + lsblk -o NAME,PATH,SIZE,FSTYPE,TYPE,MOUNTPOINTS + cat /proc/mdstat + mdadm --detail "$root_src" + """) + exit_code, stdout, stderr = guest_exec(command, timeout=60) + if stdout: + sys.stderr.write(stdout) + sys.stderr.flush() + if stderr: + sys.stderr.write(stderr) + sys.stderr.flush() + if exit_code != 0: + die(f"RAID1 guest verification failed with exit code {exit_code}") + + log(" RAID1 guest verification passed") + + def run_operation_smoke_suite() -> None: log("Running bare-metal MachineOperation smoke suite") @@ -914,6 +1388,9 @@ def main() -> None: signal.signal(signal.SIGINT, _sigint_handler) atexit.register(cleanup) + log(f"Metalman smoke install mode: {INSTALL_MODE}") + log(f"Metalman smoke boot protocol: {BOOT_PROTOCOL}") + log("Cleaning up stale libvirt resources") clean_libvirt() @@ -933,6 +1410,9 @@ def main() -> None: # Allow the VM to reach the kind Docker network (Docker's bridge # isolation rules block cross-bridge traffic by default). log("Adding iptables rules for VM ↔ kind connectivity") + # Some developer machines default-drop INPUT through UFW. The VM must + # reach metalman's DHCP, TFTP, HTTP, and health endpoints on this bridge. + run(["sudo", "iptables", "-I", "INPUT", "-i", "virbr-smoke", "-j", "ACCEPT"]) run(["sudo", "iptables", "-I", "FORWARD", "-i", "virbr-smoke", "-j", "ACCEPT"]) run(["sudo", "iptables", "-I", "FORWARD", "-o", "virbr-smoke", "-j", "ACCEPT"]) # Docker may insert a raw PREROUTING DROP rule that blocks non-Docker @@ -990,17 +1470,34 @@ def main() -> None: "--type=strategic", "-p", patch]) log("Creating UEFI VM (powered off, with TPM)") + ovmf_code, ovmf_vars_template = find_ovmf_firmware() + log(f" Using OVMF loader {ovmf_code}") ovmf_vars = TMPDIR / "OVMF_VARS.fd" - shutil.copy2("/usr/share/OVMF/OVMF_VARS_4M.fd", ovmf_vars) - disk = str(TMPDIR / "disk.qcow2") - run_quiet(["qemu-img", "create", "-f", "qcow2", disk, "20G"], check=True) + shutil.copy2(ovmf_vars_template, ovmf_vars) + disk_args: list[str] = [] + if INSTALL_MODE == "RAID1": + for index, (name, serial) in enumerate((("disk-a.qcow2", RAID_DISK_A_SERIAL), ("disk-b.qcow2", RAID_DISK_B_SERIAL))): + disk_path = TMPDIR / name + if BOOT_PROTOCOL == "HTTP" and index == 0: + prepare_http_boot_helper_disk(RAW_IMAGE_NAME, disk_path) + else: + run_quiet(["qemu-img", "create", "-f", "qcow2", str(disk_path), "20G"], check=True) + disk_args.extend(["--disk", f"path={disk_path},format=qcow2,bus=virtio,serial={serial}"]) + else: + disk_path = TMPDIR / "disk.qcow2" + if BOOT_PROTOCOL == "HTTP": + prepare_http_boot_helper_disk(RAW_IMAGE_NAME, disk_path) + else: + run_quiet(["qemu-img", "create", "-f", "qcow2", str(disk_path), "20G"], check=True) + disk_args.extend(["--disk", f"path={disk_path},format=qcow2,bus=virtio"]) + run_quiet([ "virt-install", "--connect", "qemu:///system", "--name", VM_NAME, "--ram", "4096", "--vcpus", "2", - "--disk", f"path={disk},format=qcow2,bus=virtio", + *disk_args, "--network", f"network={NET_NAME},mac={MAC_ADDRESS}", - "--boot", f"uefi,loader=/usr/share/OVMF/OVMF_CODE_4M.fd,nvram={ovmf_vars},hd,network", + "--boot", f"uefi,loader={ovmf_code},nvram={ovmf_vars},hd,network", "--tpm", "backend.type=emulator,backend.version=2.0", "--serial", f"unix,path={SERIAL_SOCK},mode=bind", "--channel", f"unix,path={QGA_SOCK},mode=bind,target.type=virtio,target.name=org.qemu.guest_agent.0", @@ -1015,25 +1512,26 @@ def main() -> None: ) console_thread.start() - log("Starting sushy-emulator") - run_quiet([ - "openssl", "req", "-x509", "-newkey", "rsa:2048", - "-keyout", str(TMPDIR / "sushy.key"), - "-out", str(TMPDIR / "sushy.crt"), - "-days", "1", "-nodes", - "-subj", "/CN=sushy-emulator", - "-addext", "subjectAltName=IP:127.0.0.1", - ], check=True) - sushy_url = f"https://127.0.0.1:{SUSHY_PORT}" - proc = spawn([ - "sushy-emulator", "--libvirt-uri", "qemu:///system", - "-i", "127.0.0.1", "-p", str(SUSHY_PORT), - "--ssl-certificate", str(TMPDIR / "sushy.crt"), - "--ssl-key", str(TMPDIR / "sushy.key"), - ], TMPDIR / "sushy.log") - log(f" sushy-emulator PID={proc.pid}") - time.sleep(2) - check_procs() + if BOOT_PROTOCOL == "PXE": + log("Starting sushy-emulator") + run_quiet([ + "openssl", "req", "-x509", "-newkey", "rsa:2048", + "-keyout", str(TMPDIR / "sushy.key"), + "-out", str(TMPDIR / "sushy.crt"), + "-days", "1", "-nodes", + "-subj", "/CN=sushy-emulator", + "-addext", "subjectAltName=IP:127.0.0.1", + ], check=True) + sushy_url = f"https://127.0.0.1:{SUSHY_PORT}" + proc = spawn([ + "sushy-emulator", "--libvirt-uri", "qemu:///system", + "-i", "127.0.0.1", "-p", str(SUSHY_PORT), + "--ssl-certificate", str(TMPDIR / "sushy.crt"), + "--ssl-key", str(TMPDIR / "sushy.key"), + ], TMPDIR / "sushy.log") + log(f" sushy-emulator PID={proc.pid}") + time.sleep(2) + check_procs() # Start Go builds in the background so they overlap with Kubernetes # setup and Docker image builds. Both targets share the Go build @@ -1098,7 +1596,7 @@ def main() -> None: # layer caching. They are already loaded into the local Docker daemon # with the correct tags. log("Verifying pre-built OCI images are available") - for name, tag in [("host-ubuntu2404", IMAGE_NAME), + for name, tag in [("host-ubuntu2404", RAW_IMAGE_NAME), ("netboot", NETBOOT_IMAGE_NAME), ("agent-ubuntu2404", AGENT_IMAGE_NAME)]: result = subprocess.run( @@ -1110,6 +1608,9 @@ def main() -> None: "Ensure the workflow builds it before running this script.") log(f" {name} image found: {tag}") + if INSTALL_MODE == "RAID1": + prepare_raid_machine_image(RAW_IMAGE_NAME, RAID_IMAGE_NAME) + # Wait for Go builds (likely already finished during k8s setup). for name, proc in go_builds: rc = proc.wait() @@ -1146,6 +1647,26 @@ def main() -> None: server_url = apiserver_url() log(f" API server URL: {server_url}") + pxe_spec: dict[str, Any] = { + "image": IMAGE_NAME, + "dhcpLeases": [{ + "mac": MAC_ADDRESS, + "ipv4": NODE_IP, + "subnetMask": "255.255.255.0", + "gateway": GATEWAY, + "dns": [DNS_SERVER], + }], + } + if BOOT_PROTOCOL == "HTTP": + pxe_spec["bootProtocol"] = "HTTP" + else: + pxe_spec["redfish"] = { + "url": sushy_url, + "username": "", + "deviceID": VM_NAME, + "passwordRef": {"name": "bmc-pass", "key": "password", "namespace": NODE_NS}, + } + protonode = { "apiVersion": API_VERSION, "kind": "Machine", @@ -1154,22 +1675,7 @@ def main() -> None: "labels": {f"{API_GROUP}/site": SITE}, }, "spec": { - "pxe": { - "image": IMAGE_NAME, - "redfish": { - "url": sushy_url, - "username": "", - "deviceID": VM_NAME, - "passwordRef": {"name": "bmc-pass", "key": "password", "namespace": NODE_NS}, - }, - "dhcpLeases": [{ - "mac": MAC_ADDRESS, - "ipv4": NODE_IP, - "subnetMask": "255.255.255.0", - "gateway": GATEWAY, - "dns": [DNS_SERVER], - }], - }, + "pxe": pxe_spec, "agent": { "image": AGENT_IMAGE_NAME_VM, "url": AGENT_DOWNLOAD_URL, @@ -1179,6 +1685,16 @@ def main() -> None: }, }, } + if INSTALL_MODE == "RAID1": + protonode["spec"]["pxe"]["install"] = { + "mode": "RAID1", + "targetDisks": [RAID_DISK_A_PATH, RAID_DISK_B_PATH], + } + if BOOT_PROTOCOL == "HTTP": + protonode["spec"]["operations"] = { + "rebootCounter": 1, + "repaveCounter": 1, + } kubectl(["apply", "-f", "-"], input=json.dumps(protonode).encode(), stdout=DEVNULL) log(" Resources created") @@ -1204,12 +1720,19 @@ def main() -> None: time.sleep(2) check_procs() - log("Triggering HostReplace through kubectl-unbounded") - operation_log = TMPDIR / "kubectl-host-replace.log" - operation_proc = run_kubectl_unbounded_operation( - ["replace", NODE_NAME, "--force", "--ttl=3600"], - operation_log.name, - ) + operation_log: Path | None = None + operation_proc: subprocess.Popen[Any] | None = None + if BOOT_PROTOCOL == "HTTP": + log("Starting VM for out-of-band UEFI HTTP boot") + run([*VIRSH, "start", VM_NAME]) + wait_vm_state("running", timeout=180) + else: + log("Triggering HostReplace through kubectl-unbounded") + operation_log = TMPDIR / "kubectl-host-replace.log" + operation_proc = run_kubectl_unbounded_operation( + ["replace", NODE_NAME, "--force", "--ttl=3600"], + operation_log.name, + ) # Log free space so we can correlate disk exhaustion with VM failures. df = subprocess.run(["df", "-h", str(TMPDIR)], capture_output=True, text=True) @@ -1218,15 +1741,22 @@ def main() -> None: log("Waiting for cloud-init to complete...") assert_cloud_init_done(timeout=900) - wait_process_success(operation_proc, timeout=900) - assert_log_contains(operation_log, "Condition CloudInitDone: True/Succeeded") + if operation_proc is not None and operation_log is not None: + wait_process_success(operation_proc, timeout=900) + assert_log_contains(operation_log, "Condition CloudInitDone: True/Succeeded") + else: + assert_machine_condition("Repaved", "True", "Succeeded", timeout=300) log("Waiting for kubelet to join the cluster...") wait_k8s_node(NODE_NAME, timeout=900) assert_node_ready(NODE_NAME, timeout=720) assert_node_label(NODE_NAME, NODE_LABEL_KEY, NODE_LABEL_VALUE) + assert_raid1_install() - run_operation_smoke_suite() + if BOOT_PROTOCOL == "PXE": + run_operation_smoke_suite() + else: + log("Skipping MachineOperation smoke suite for out-of-band HTTP boot") log("") log("Smoke test PASSED") diff --git a/images/netboot/Containerfile b/images/netboot/Containerfile index ad9dc8a07..6411b730c 100644 --- a/images/netboot/Containerfile +++ b/images/netboot/Containerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ ok=false && \ for attempt in 1 2 3 4 5; do \ if apt-get install -y --no-install-recommends \ - ca-certificates cpio curl efibootmgr kmod linux-image-generic xz-utils; then \ + ca-certificates cpio curl dosfstools e2fsprogs efibootmgr gdisk kmod linux-image-generic mdadm parted tar xz-utils zstd; then \ ok=true; break; \ fi; \ echo "apt-get install failed (attempt $attempt/5), retrying in 30s..."; \ @@ -45,29 +45,42 @@ RUN mkdir -p /artifacts && \ COPY images/netboot/assets/init /tmp/initrd-root/init RUN chmod 755 /tmp/initrd-root/init && \ mkdir -p /tmp/initrd-root/usr/bin /tmp/initrd-root/usr/lib && \ - cp /usr/bin/efibootmgr /tmp/initrd-root/usr/bin/ && \ - for lib in $(ldd /usr/bin/efibootmgr | awk '/=>/{print $3}'); do \ - mkdir -p "/tmp/initrd-root$(dirname "$lib")" && \ - cp "$lib" "/tmp/initrd-root${lib}"; \ + for bin in efibootmgr mdadm sgdisk partprobe mkfs.ext4 mkfs.vfat zstd tar blkid chroot; do \ + src="$(command -v "$bin")" || { echo "missing required binary: $bin" >&2; exit 1; }; \ + mkdir -p "/tmp/initrd-root$(dirname "$src")" && \ + cp -L "$src" "/tmp/initrd-root${src}" && \ + for lib in $(ldd "$src" | awk '/=>/{print $3} /^\//{print $1}'); do \ + [ -n "$lib" ] || continue; \ + mkdir -p "/tmp/initrd-root$(dirname "$lib")" && \ + cp -L "$lib" "/tmp/initrd-root${lib}"; \ + done && \ + interp="$(ldd "$src" | awk '/ld-linux/{print $1; exit}')" && \ + if [ -n "$interp" ]; then \ + mkdir -p "/tmp/initrd-root$(dirname "$interp")" && \ + cp -L "$interp" "/tmp/initrd-root${interp}"; \ + fi; \ done && \ - INTERP=$(ldd /usr/bin/efibootmgr | awk '/ld-linux/{print $1}') && \ - mkdir -p "/tmp/initrd-root$(dirname "$INTERP")" && \ - cp "$INTERP" "/tmp/initrd-root${INTERP}" && \ KVER=$(ls /lib/modules | head -1) && \ KMOD="/tmp/initrd-root/lib/modules/${KVER}" && \ mkdir -p "${KMOD}/kernel/drivers/net" \ "${KMOD}/kernel/drivers/scsi" \ "${KMOD}/kernel/drivers/ata" \ "${KMOD}/kernel/drivers/nvme" \ + "${KMOD}/kernel/drivers/md" \ "${KMOD}/kernel/drivers/virtio" \ "${KMOD}/kernel/drivers/block" \ + "${KMOD}/kernel/crypto" \ + "${KMOD}/kernel/lib" \ "${KMOD}/kernel/fs" && \ cp -a "/lib/modules/${KVER}/kernel/drivers/net/." "${KMOD}/kernel/drivers/net/" && \ cp -a "/lib/modules/${KVER}/kernel/drivers/scsi/." "${KMOD}/kernel/drivers/scsi/" && \ cp -a "/lib/modules/${KVER}/kernel/drivers/ata/." "${KMOD}/kernel/drivers/ata/" && \ cp -a "/lib/modules/${KVER}/kernel/drivers/nvme/." "${KMOD}/kernel/drivers/nvme/" && \ + cp -a "/lib/modules/${KVER}/kernel/drivers/md/." "${KMOD}/kernel/drivers/md/" && \ cp -a "/lib/modules/${KVER}/kernel/drivers/virtio/." "${KMOD}/kernel/drivers/virtio/" && \ cp -a "/lib/modules/${KVER}/kernel/drivers/block/." "${KMOD}/kernel/drivers/block/" && \ + cp -a "/lib/modules/${KVER}/kernel/crypto/." "${KMOD}/kernel/crypto/" && \ + cp -a "/lib/modules/${KVER}/kernel/lib/." "${KMOD}/kernel/lib/" && \ cp -a "/lib/modules/${KVER}/kernel/fs/." "${KMOD}/kernel/fs/" && \ depmod -b /tmp/initrd-root "${KVER}" && \ cd /tmp/initrd-root && \ @@ -77,10 +90,10 @@ RUN mkdir -p /disk/grub /disk/cloud-init && \ cp /artifacts/vmlinuz /artifacts/initrd /artifacts/init.cpio /disk/ && \ if [ "${TARGETARCH}" = "amd64" ]; then \ cp /artifacts/shimx64.efi /artifacts/grubx64.efi /disk/ && \ - printf '%s\n' "dhcpBootImageName: shimx64.efi" "httpBootPath: shimx64.efi" > /disk/metadata.yaml; \ + printf '%s\n' "dhcpBootImageName: shimx64.efi" "httpBootPath: grubx64.efi" > /disk/metadata.yaml; \ elif [ "${TARGETARCH}" = "arm64" ]; then \ cp /artifacts/shimaa64.efi /artifacts/grubaa64.efi /disk/ && \ - printf '%s\n' "dhcpBootImageName: shimaa64.efi" "httpBootPath: shimaa64.efi" > /disk/metadata.yaml; \ + printf '%s\n' "dhcpBootImageName: shimaa64.efi" "httpBootPath: grubaa64.efi" > /disk/metadata.yaml; \ fi COPY images/netboot/assets/grub.cfg.tmpl /disk/grub/grub.cfg.tmpl diff --git a/images/netboot/assets/grub.cfg.tmpl b/images/netboot/assets/grub.cfg.tmpl index 43404e106..d1d315b2e 100644 --- a/images/netboot/assets/grub.cfg.tmpl +++ b/images/netboot/assets/grub.cfg.tmpl @@ -3,8 +3,28 @@ insmod part_gpt insmod fat insmod search_fs_file +insmod configfile insmod chain +if search --no-floppy --set=root --file /unbounded/grub.cfg; then + configfile /unbounded/grub.cfg +fi +if search --no-floppy --set=root --file /EFI/BOOT/grubx64.efi; then + chainloader /EFI/BOOT/grubx64.efi + boot +fi +if search --no-floppy --set=root --file /EFI/ubuntu/grubx64.efi; then + chainloader /EFI/ubuntu/grubx64.efi + boot +fi +if search --no-floppy --set=root --file /EFI/BOOT/grubaa64.efi; then + chainloader /EFI/BOOT/grubaa64.efi + boot +fi +if search --no-floppy --set=root --file /EFI/ubuntu/grubaa64.efi; then + chainloader /EFI/ubuntu/grubaa64.efi + boot +fi if search --no-floppy --set=root --file /EFI/BOOT/BOOTX64.EFI; then chainloader /EFI/BOOT/BOOTX64.EFI boot @@ -38,8 +58,8 @@ menuentry "Unbounded Metal Install" { unbounded.boot_mac={{ .BootLease.MAC }} \ ip={{ .BootLease.IPv4 }}::{{ .BootLease.Gateway }}:{{ .BootLease.SubnetMask }}:::none \ {{- end }} - {{- if .Machine.Spec.PXE.TargetDisk }} - unbounded.disk={{ .Machine.Spec.PXE.TargetDisk }} \ + {{- range .InstallKernelArgs }} + {{ . }} \ {{- end }} console=tty0 console=ttyS0,115200n8 \ --- diff --git a/images/netboot/assets/init b/images/netboot/assets/init index a41cfbe28..6d63e7c34 100644 --- a/images/netboot/assets/init +++ b/images/netboot/assets/init @@ -1,8 +1,8 @@ #!/bin/sh # Unbounded Metal installer initrd /init script. # Overlaid on top of the Ubuntu netboot initrd via a second cpio archive. -# Downloads a compressed raw disk image over HTTP, writes it to disk, -# injects cloud-init datasource configuration, and reboots. +# Installs a machine image to local disks, injects cloud-init datasource +# configuration, disables PXE, and reboots. set -e @@ -15,9 +15,18 @@ mount -t devtmpfs devtmpfs /dev 2>/dev/null || true exec >/dev/kmsg 2>&1 export PATH=/usr/sbin:/usr/bin:/sbin:/bin +export DEBIAN_FRONTEND=noninteractive -log() { echo "unbounded-metal: $*"; } -fatal() { log "FATAL: $*"; exec sh; } +log() { + msg="unbounded-metal: $*" + echo "$msg" >/dev/kmsg 2>/dev/null || true + echo "$msg" >/dev/console 2>/dev/null || true +} + +fatal() { + log "FATAL: $*" + exec sh /dev/console 2>&1 +} retry() { local n="$1" delay="$2" desc="$3" i @@ -33,6 +42,23 @@ retry() { done } +run_logged() { + desc="$1" + shift + log_file="/tmp/unbounded-metal-command.log" + log "starting: $desc" + if "$@" >"$log_file" 2>&1; then + rm -f "$log_file" + else + log "output from failed command:" + while IFS= read -r line; do + log " $line" + done <"$log_file" + fatal "failed: $desc" + fi + log "completed: $desc" +} + log "installer starting" CMDLINE=$(cat /proc/cmdline) @@ -77,34 +103,602 @@ log_disks() { esac size=$(cat "$disk/size" 2>/dev/null || echo 0) model=$(cat "$disk/device/model" 2>/dev/null || true) - serial=$(cat "$disk/device/serial" 2>/dev/null || true) + serial=$(cat "$disk/serial" 2>/dev/null || cat "$disk/device/serial" 2>/dev/null || true) removable=$(cat "$disk/removable" 2>/dev/null || true) log " /dev/$(basename "$disk") sectors=$size model=${model:-unknown} serial=${serial:-unknown} removable=${removable:-unknown}" done } target_disk_sysfs() { - base=$(basename "$TARGET_DISK") + disk="$1" + base=$(basename "$disk") sysdisk=$(readlink -f "/sys/class/block/$base" 2>/dev/null || true) if [ -z "$sysdisk" ]; then sysdisk="/sys/block/$base" fi [ -d "$sysdisk" ] || return 1 - [ ! -e "$sysdisk/partition" ] || fatal "target disk $TARGET_DISK is a partition, expected whole disk" + [ ! -e "$sysdisk/partition" ] || fatal "target disk $disk is a partition, expected whole disk" echo "$sysdisk" } +normalize_disk_id() { + printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -c '[:alnum:]' '_' +} + +disk_id_candidate_matches() { + wanted="$1" + candidate="$2" + candidate_id=$(normalize_disk_id "$candidate") + [ -n "$candidate_id" ] || return 1 + [ "$wanted" = "$candidate_id" ] +} + +disk_id_matches_prefixed_value() { + wanted="$1" + value="$2" + shift 2 + value_id=$(normalize_disk_id "$value") + [ -n "$value_id" ] || return 1 + + for prefix in "$@"; do + disk_id_candidate_matches "$wanted" "${prefix}_${value_id}" && return 0 + done + + return 1 +} + +disk_id_matches_wwid() { + wanted="$1" + value="$2" + value_id=$(normalize_disk_id "$value") + [ -n "$value_id" ] || return 1 + + disk_id_candidate_matches "$wanted" "wwn_${value_id}" && return 0 + disk_id_candidate_matches "$wanted" "scsi_${value_id}" && return 0 + disk_id_candidate_matches "$wanted" "nvme_${value_id}" && return 0 + + case "$value_id" in + 0x*) ;; + *) disk_id_candidate_matches "$wanted" "wwn_0x${value_id}" && return 0 ;; + esac + + return 1 +} + +disk_identity_matches() { + wanted="$1" + sysdisk="$2" + + for attr in "$sysdisk/serial" "$sysdisk/device/serial"; do + [ -r "$attr" ] || continue + value=$(cat "$attr" 2>/dev/null || true) + disk_id_matches_prefixed_value "$wanted" "$value" virtio ata scsi nvme usb \ + && return 0 + done + + for attr in "$sysdisk/wwid" "$sysdisk/device/wwid"; do + [ -r "$attr" ] || continue + value=$(cat "$attr" 2>/dev/null || true) + disk_id_matches_wwid "$wanted" "$value" && return 0 + done + + for attr in "$sysdisk/device/eui"; do + [ -r "$attr" ] || continue + value=$(cat "$attr" 2>/dev/null || true) + disk_id_matches_prefixed_value "$wanted" "$value" nvme_eui eui && return 0 + done + + for attr in "$sysdisk/device/nguid"; do + [ -r "$attr" ] || continue + value=$(cat "$attr" 2>/dev/null || true) + disk_id_matches_prefixed_value "$wanted" "$value" nvme_nguid nguid && return 0 + done + + model=$(cat "$sysdisk/device/model" 2>/dev/null || true) + serial=$(cat "$sysdisk/serial" 2>/dev/null || cat "$sysdisk/device/serial" 2>/dev/null || true) + if [ -n "$model" ] && [ -n "$serial" ]; then + disk_id_matches_prefixed_value "$wanted" "${model}_${serial}" ata scsi nvme usb \ + && return 0 + fi + + return 1 +} + +resolve_disk_by_id_path() { + path="$1" + wanted=$(normalize_disk_id "$(basename "$path")") + DISK_BY_ID_RESOLVED="" + matches="" + count=0 + + for sysdisk in /sys/block/sd* /sys/block/nvme*n* /sys/block/vd*; do + [ -e "$sysdisk" ] || continue + [ ! -e "$sysdisk/partition" ] || continue + base=$(basename "$sysdisk") + [ -b "/dev/$base" ] || continue + + if disk_identity_matches "$wanted" "$sysdisk"; then + count=$((count + 1)) + DISK_BY_ID_RESOLVED="/dev/$base" + matches="${matches} /dev/$base" + fi + done + + case "$count" in + 0) return 1 ;; + 1) return 0 ;; + *) fatal "target disk $path matched multiple disks:${matches}" ;; + esac +} + +resolve_whole_disk() { + disk="$1" + RESOLVED_DISK="" + [ -n "$disk" ] || return 1 + resolved="$disk" + + if [ ! -e "$resolved" ]; then + case "$disk" in + /dev/disk/by-id/*) + if resolve_disk_by_id_path "$disk"; then + resolved="$DISK_BY_ID_RESOLVED" + log "resolved $disk to $resolved from sysfs identity" + fi + ;; + esac + fi + + [ -e "$resolved" ] || return 1 + + resolved=$(readlink -f "$resolved" 2>/dev/null || echo "$resolved") + [ -b "$resolved" ] || fatal "target disk $disk is not a block device" + sysdisk=$(target_disk_sysfs "$resolved") || fatal "target disk $disk does not exist in sysfs" + + removable=$(cat "$sysdisk/removable" 2>/dev/null || echo 0) + [ "$removable" = "0" ] || fatal "target disk $disk is removable" + + RESOLVED_DISK="$resolved" +} + +resolve_required_whole_disk() { + disk="$1" + retry 30 1 "resolve target disk $disk" resolve_whole_disk "$disk" \ + || fatal "target disk $disk does not exist" +} + partition_number() { part=$(basename "$1") cat "/sys/class/block/$part/partition" 2>/dev/null || true } +parts_for_disk() { + sysdisk=$(target_disk_sysfs "$1") || return 0 + for p in "$sysdisk"/*; do + [ -e "$p" ] || continue + [ -e "$p/partition" ] || continue + echo "/dev/$(basename "$p")" + done +} + +part_path() { + disk="$1" + number="$2" + case "$disk" in + /dev/nvme*n*|/dev/mmcblk*|/dev/loop*) echo "${disk}p${number}" ;; + *) echo "${disk}${number}" ;; + esac +} + +block_exists() { + [ -b "$1" ] +} + +download_file() { + url="$1" + dest="$2" + wget -q -O "$dest" "$url" +} + +extract_tar_zst() { + archive="$1" + dest="$2" + zstd -dc "$archive" | tar -xpf - -C "$dest" +} + +efi_loader_candidates() { + cat < "$root/etc/cloud/cloud.cfg.d/99-unbounded-metal.cfg" < "$root/etc/unbounded-metal/config" </dev/null || mount -n --rbind /dev /mnt/dev + mountpoint -q /mnt/proc 2>/dev/null || mount -n -t proc proc /mnt/proc + mountpoint -q /mnt/sys 2>/dev/null || mount -n --rbind /sys /mnt/sys + mountpoint -q /mnt/run 2>/dev/null || mount -n --bind /run /mnt/run +} + +umount_chroot_filesystems() { + umount -l /mnt/run 2>/dev/null || true + umount -l /mnt/sys 2>/dev/null || true + umount -l /mnt/proc 2>/dev/null || true + umount -l /mnt/dev 2>/dev/null || true +} + +grub_target() { + case "$(uname -m)" in + x86_64) echo "x86_64-efi" ;; + aarch64|arm64) echo "arm64-efi" ;; + *) echo "" ;; + esac +} + +write_esp_grub_config() { + root_uuid="$1" + esp_root="$2" + + [ -n "$root_uuid" ] || fatal "root UUID is required for ESP GRUB config" + mkdir -p "$esp_root/EFI/ubuntu" "$esp_root/EFI/BOOT" "$esp_root/unbounded" + { + echo "insmod part_gpt" + echo "insmod fat" + echo "insmod gzio" + echo "insmod linux" + echo "search --no-floppy --file --set=root /unbounded/vmlinuz" + echo "linux /unbounded/vmlinuz root=UUID=${root_uuid} ro console=tty1 console=ttyS0" + echo "initrd /unbounded/initrd.img" + echo "boot" + } > "$esp_root/unbounded/grub.cfg" + cp "$esp_root/unbounded/grub.cfg" "$esp_root/EFI/ubuntu/grub.cfg" + cp "$esp_root/EFI/ubuntu/grub.cfg" "$esp_root/EFI/BOOT/grub.cfg" +} + +copy_grub_modules() { + target="$1" + src="/mnt/usr/lib/grub/$target" + dest="/mnt/boot/grub/$target" + + [ -d "$src" ] || fatal "host image does not contain GRUB modules for $target" + rm -rf "$dest" + mkdir -p "$dest" + run_logged "copying GRUB modules for $target" cp -a "$src/." "$dest/" +} + +copy_esp_boot_artifacts() { + esp_root="$1" + + [ -f /mnt/boot/vmlinuz ] || fatal "host image does not contain /boot/vmlinuz" + [ -f /mnt/boot/initrd.img ] || fatal "host image does not contain /boot/initrd.img" + mkdir -p "$esp_root/unbounded" + run_logged "copying kernel to ESP" cp -L /mnt/boot/vmlinuz "$esp_root/unbounded/vmlinuz" + run_logged "copying initramfs to ESP" cp -L /mnt/boot/initrd.img "$esp_root/unbounded/initrd.img" +} + +refresh_host_boot() { + esp0="$1" + esp1="$2" + root_uuid="$3" + + mount_chroot_filesystems + mkdir -p /mnt/boot/efi /mnt/boot/grub + run_logged "mounting primary ESP $esp0 for host boot refresh" \ + mount -n "$esp0" /mnt/boot/efi + + [ -x /mnt/usr/sbin/update-initramfs ] \ + || fatal "RAID1 host image must contain update-initramfs" + run_logged "refreshing host initramfs" \ + chroot /mnt /usr/sbin/update-initramfs -u -k all + + [ -x /mnt/usr/sbin/update-grub ] \ + || fatal "RAID1 host image must contain update-grub" + run_logged "refreshing host GRUB config" \ + chroot /mnt /usr/sbin/update-grub + + target=$(grub_target) + [ -n "$target" ] || fatal "unsupported architecture for grub-install: $(uname -m)" + [ -x /mnt/usr/sbin/grub-install ] \ + || fatal "RAID1 host image must contain grub-install" + copy_grub_modules "$target" + + for esp in "$esp0" "$esp1"; do + if [ "$esp" != "$esp0" ]; then + umount /mnt/boot/efi 2>/dev/null || true + run_logged "mounting ESP $esp for boot refresh" mount -n "$esp" /mnt/boot/efi + fi + + run_logged "installing GRUB fallback loader on $esp" \ + chroot /mnt /usr/sbin/grub-install \ + --target="$target" \ + --efi-directory=/boot/efi \ + --bootloader-id=ubuntu \ + --modules="part_gpt fat ext2 mdraid1x diskfilter search search_fs_uuid normal configfile linux gzio" \ + --no-nvram \ + --removable + + copy_esp_boot_artifacts /mnt/boot/efi + write_esp_grub_config "$root_uuid" /mnt/boot/efi + find_loader_in_mount /mnt/boot/efi >/dev/null \ + || fatal "ESP $esp does not contain a known EFI loader" + sync + umount /mnt/boot/efi + done + + umount_chroot_filesystems +} + +find_root_part() { + disk="$1" + ROOT_PART="" + for part in $(parts_for_disk "$disk"); do + if mount -n "$part" /mnt 2>/dev/null; then + if [ -d /mnt/etc ] && [ -d /mnt/var ]; then + ROOT_PART="$part" + umount /mnt + return 0 + fi + umount /mnt + fi + done + return 1 +} + +install_raw() { + [ -n "$IMAGE_URL" ] || fatal "unbounded.image_url not set" + + if [ -z "$TARGET_DISK" ] && [ -n "$TARGET_DISK0" ]; then + TARGET_DISK="$TARGET_DISK0" + fi + + if [ -z "$TARGET_DISK" ]; then + log "waiting for block devices" + find_disk() { + TARGET_DISK="" + MAX_SIZE=0 + for disk in /sys/block/sd* /sys/block/nvme*n* /sys/block/vd*; do + [ -e "$disk" ] || continue + [ -e "$disk/partition" ] && continue + SIZE=$(cat "$disk/size" 2>/dev/null || echo 0) + if [ "$SIZE" -gt "$MAX_SIZE" ]; then + MAX_SIZE=$SIZE + TARGET_DISK="/dev/$(basename "$disk")" + fi + done + [ -n "$TARGET_DISK" ] + } + log_disks + retry 30 1 "find block device" find_disk || fatal "no target disk found" + log "WARNING: target disk was not specified, selected largest disk" + fi + + resolve_required_whole_disk "$TARGET_DISK" + TARGET_DISK="$RESOLVED_DISK" + UEFI_DISKS="$TARGET_DISK" + log "target disk: $TARGET_DISK" + + log "downloading disk image from $IMAGE_URL" + retry 120 5 "download disk image" download_file "$IMAGE_URL" /tmp/disk.img.gz \ + || fatal "failed to download disk image" + + log "writing disk image to $TARGET_DISK" + write_image() { gunzip -c /tmp/disk.img.gz | dd of="$TARGET_DISK" bs=4194304; } + retry 3 5 "write disk image" write_image || fatal "failed to write disk image" + rm -f /tmp/disk.img.gz + sync + + retry 5 2 "re-read partition table" blockdev --rereadpt "$TARGET_DISK" \ + || log "WARNING: could not re-read partition table" + sleep 2 + + if [ -n "$DS_URL" ]; then + retry 20 2 "find root partition" find_root_part "$TARGET_DISK" \ + || fatal "no root partition found on $TARGET_DISK" + retry 5 2 "mount $ROOT_PART" mount -n "$ROOT_PART" /mnt \ + || fatal "failed to mount $ROOT_PART" + configure_cloud_init /mnt + sync + umount /mnt + log "cloud-init configured on $ROOT_PART" + fi +} + +prepare_raid_disk() { + disk="$1" + log "partitioning $disk for RAID1 install" + sgdisk --zap-all "$disk" + sgdisk \ + -n 1:1MiB:+1024MiB -t 1:ef00 -c 1:unbounded-efi \ + -n 2:0:0 -t 2:fd00 -c 2:unbounded-root \ + "$disk" + partprobe "$disk" 2>/dev/null || blockdev --rereadpt "$disk" \ + || log "WARNING: could not re-read partition table on $disk" +} + +install_raid1() { + [ -n "$SERVE_URL" ] || fatal "unbounded.serve_url is required for RAID1 install" + [ -n "$TARGET_DISK0" ] || fatal "RAID1 install requires unbounded.disk0" + [ -n "$TARGET_DISK1" ] || fatal "RAID1 install requires unbounded.disk1" + + resolve_required_whole_disk "$TARGET_DISK0" + DISK0="$RESOLVED_DISK" + resolve_required_whole_disk "$TARGET_DISK1" + DISK1="$RESOLVED_DISK" + [ "$DISK0" != "$DISK1" ] || fatal "RAID1 install requires two distinct disks" + + UEFI_DISKS="$DISK0 $DISK1" + log_disks + log "target disks: $DISK0 $DISK1" + + machine_url="${SERVE_URL%/}/machine" + log "downloading rootfs from ${machine_url}/rootfs.tar.zst" + retry 120 5 "download rootfs" download_file "${machine_url}/rootfs.tar.zst" /tmp/rootfs.tar.zst \ + || fatal "failed to download rootfs.tar.zst" + log "downloading ESP from ${machine_url}/esp.tar.zst" + retry 120 5 "download ESP" download_file "${machine_url}/esp.tar.zst" /tmp/esp.tar.zst \ + || fatal "failed to download esp.tar.zst" + + for disk in "$DISK0" "$DISK1"; do + prepare_raid_disk "$disk" + done + sleep 2 + + ESP0=$(part_path "$DISK0" 1) + ROOT0=$(part_path "$DISK0" 2) + ESP1=$(part_path "$DISK1" 1) + ROOT1=$(part_path "$DISK1" 2) + retry 30 1 "wait for $ESP0" block_exists "$ESP0" || fatal "$ESP0 did not appear" + retry 30 1 "wait for $ROOT0" block_exists "$ROOT0" || fatal "$ROOT0 did not appear" + retry 30 1 "wait for $ESP1" block_exists "$ESP1" || fatal "$ESP1 did not appear" + retry 30 1 "wait for $ROOT1" block_exists "$ROOT1" || fatal "$ROOT1 did not appear" + + mdadm --stop /dev/md/unbounded-root 2>/dev/null || true + mdadm --zero-superblock --force "$ROOT0" "$ROOT1" 2>/dev/null || true + + mkdir -p /dev/md + log "creating RAID1 array /dev/md/unbounded-root" + mdadm --create /dev/md/unbounded-root \ + --metadata=1.2 \ + --level=1 \ + --raid-devices=2 \ + --force \ + "$ROOT0" "$ROOT1" \ + --run \ + || fatal "failed to create RAID1 array" + retry 30 1 "wait for RAID array" block_exists /dev/md/unbounded-root \ + || fatal "/dev/md/unbounded-root did not appear" + + run_logged "formatting RAID root filesystem" mkfs.ext4 -F -L unbounded-root /dev/md/unbounded-root + run_logged "formatting primary ESP $ESP0" mkfs.vfat -F 32 -n UBNDEFI0 "$ESP0" + run_logged "formatting secondary ESP $ESP1" mkfs.vfat -F 32 -n UBNDEFI1 "$ESP1" + + run_logged "mounting RAID root filesystem" mount -n /dev/md/unbounded-root /mnt + log "extracting rootfs" + extract_tar_zst /tmp/rootfs.tar.zst /mnt || fatal "failed to extract rootfs" + + mkdir -p /mnt/boot/efi /mnt/boot/efi-secondary + run_logged "mounting primary ESP $ESP0" mount -n "$ESP0" /mnt/boot/efi + log "extracting ESP to $ESP0" + extract_tar_zst /tmp/esp.tar.zst /mnt/boot/efi || fatal "failed to extract ESP to $ESP0" + umount /mnt/boot/efi + + run_logged "mounting secondary ESP $ESP1" mount -n "$ESP1" /mnt/boot/efi-secondary + log "extracting ESP to $ESP1" + extract_tar_zst /tmp/esp.tar.zst /mnt/boot/efi-secondary || fatal "failed to extract ESP to $ESP1" + umount /mnt/boot/efi-secondary + + mkdir -p /mnt/etc/mdadm + mdadm --detail --scan > /mnt/etc/mdadm/mdadm.conf + + ROOT_UUID=$(blkid -s UUID -o value /dev/md/unbounded-root) + ESP0_UUID=$(blkid -s UUID -o value "$ESP0") + ESP1_UUID=$(blkid -s UUID -o value "$ESP1") + [ -n "$ROOT_UUID" ] || fatal "could not read RAID root UUID" + [ -n "$ESP0_UUID" ] || fatal "could not read primary ESP UUID" + [ -n "$ESP1_UUID" ] || fatal "could not read secondary ESP UUID" + + if [ -f /mnt/etc/fstab ]; then + cp /mnt/etc/fstab /mnt/etc/fstab.unbounded.bak + fi + cat > /mnt/etc/fstab </dev/null || true + rm -f /tmp/rootfs.tar.zst /tmp/esp.tar.zst +} + +create_uefi_boot_entries() { + [ -n "$UEFI_DISKS" ] || return 0 + [ -d /sys/firmware/efi ] || return 0 + + log "creating UEFI boot entries for local disks" + if ! mountpoint -q /sys/firmware/efi/efivars 2>/dev/null; then + mount -n -t efivarfs efivarfs /sys/firmware/efi/efivars 2>/dev/null || true + fi + + for disk in $UEFI_DISKS; do + for part in $(parts_for_disk "$disk"); do + if mount -n -t vfat "$part" /mnt/esp 2>/dev/null; then + LOADER=$(find_loader_in_mount /mnt/esp || true) + umount /mnt/esp + + if [ -n "$LOADER" ] && command -v efibootmgr >/dev/null 2>&1; then + ESP_NUM=$(partition_number "$part") + if [ -n "$ESP_NUM" ]; then + LOADER_EFI=$(echo "$LOADER" | tr '/' '\\') + efibootmgr --create --disk "$disk" --part "$ESP_NUM" \ + --loader "$LOADER_EFI" --label "unbounded" 2>&1 \ + && log "UEFI boot entry created (${LOADER} on $disk part ${ESP_NUM})" \ + || log "WARNING: efibootmgr failed, PXE chainloader will be used as fallback" + break + fi + fi + fi + done + done +} + +disable_pxe() { + [ -n "$SERVE_URL" ] || return 0 + + log "disabling PXE boot" + retry 5 2 "disable PXE" wget -q -O /dev/null "$SERVE_URL/pxe/disable" \ + || log "WARNING: failed to disable PXE boot" +} + KVER=$(uname -r) for mod in virtio_pci virtio_blk ahci sd_mod nvme xfs ext4; do modprobe "$mod" 2>/dev/null || true done +for mod in md_mod raid1 linear multipath; do + modprobe "$mod" 2>/dev/null || true +done for mod in virtio_net e1000 e1000e igb ixgbe i40e ice mlx5_core mlx4_core bnxt_en tg3 be2net ena; do modprobe "$mod" 2>/dev/null || true done @@ -119,15 +713,27 @@ for mod in nls_cp437 nls_ascii nls_utf8 fat vfat efivarfs; do modprobe "$mod" 2>/dev/null || true done -IMAGE_URL=$(get_param unbounded.image_url) || fatal "unbounded.image_url not set" +IMAGE_URL=$(get_param unbounded.image_url || true) SERVE_URL=$(get_param unbounded.serve_url || true) +INSTALL_MODE=$(get_param unbounded.install_mode || echo Raw) TARGET_DISK=$(get_param unbounded.disk || true) +TARGET_DISK0=$(get_param unbounded.disk0 || true) +TARGET_DISK1=$(get_param unbounded.disk1 || true) BOOT_MAC=$(get_param unbounded.boot_mac || true) BOOTIF=$(get_param BOOTIF || true) DS_URL=$(get_param unbounded.ds_url || true) NODE_NAME=$(get_param unbounded.node_name || true) NODE_NAMESPACE=$(get_param unbounded.node_namespace || true) APISERVER_URL=$(get_param unbounded.apiserver_url || true) +UEFI_DISKS="" + +case "$INSTALL_MODE" in + ""|Raw|raw|RAW) INSTALL_MODE=Raw ;; + RAID1|raid1|Raid1) INSTALL_MODE=RAID1 ;; + *) fatal "unsupported install mode $INSTALL_MODE" ;; +esac +log "install mode: $INSTALL_MODE" + BOOT_MAC=$(normalize_mac "$BOOT_MAC") if [ -z "$BOOT_MAC" ] && [ -n "$BOOTIF" ]; then BOOT_MAC=$(bootif_to_mac "$BOOTIF") @@ -210,135 +816,13 @@ if [ -n "$IP_PARAM" ]; then retry 3 1 "add default route" ip route add default via "$GW" dev "$IFACE" || true fi -if [ -z "$TARGET_DISK" ]; then - log "waiting for block devices" - find_disk() { - TARGET_DISK="" - MAX_SIZE=0 - for disk in /sys/block/sd* /sys/block/nvme*n* /sys/block/vd*; do - [ -e "$disk" ] || continue - [ -e "$disk/partition" ] && continue - SIZE=$(cat "$disk/size" 2>/dev/null || echo 0) - if [ "$SIZE" -gt "$MAX_SIZE" ]; then - MAX_SIZE=$SIZE - TARGET_DISK="/dev/$(basename "$disk")" - fi - done - [ -n "$TARGET_DISK" ] - } - log_disks - retry 30 1 "find block device" find_disk || fatal "no target disk found" - log "WARNING: target disk was not specified, selected largest disk" -else - if [ ! -e "$TARGET_DISK" ]; then - fatal "target disk $TARGET_DISK does not exist" - fi -fi -TARGET_DISK=$(readlink -f "$TARGET_DISK" 2>/dev/null || echo "$TARGET_DISK") -log "target disk: $TARGET_DISK" - -log "downloading disk image from $IMAGE_URL" -retry 120 5 "download disk image" wget -q -O /tmp/disk.img.gz "$IMAGE_URL" \ - || fatal "failed to download disk image" - -log "writing disk image to $TARGET_DISK" -write_image() { gunzip -c /tmp/disk.img.gz | dd of="$TARGET_DISK" bs=4194304; } -retry 3 5 "write disk image" write_image || fatal "failed to write disk image" -rm -f /tmp/disk.img.gz -sync - -retry 5 2 "re-read partition table" blockdev --rereadpt "$TARGET_DISK" \ - || log "WARNING: could not re-read partition table" -sleep 2 - -parts_for_disk() { - sysdisk=$(target_disk_sysfs) || return 0 - for p in "$sysdisk"/*; do - [ -e "$p" ] || continue - [ -e "$p/partition" ] || continue - echo "/dev/$(basename "$p")" - done -} - -find_root_part() { - ROOT_PART="" - for part in $(parts_for_disk); do - if mount "$part" /mnt 2>/dev/null; then - if [ -d /mnt/etc ] && [ -d /mnt/var ]; then - ROOT_PART="$part" - umount /mnt - return 0 - fi - umount /mnt - fi - done - return 1 -} +case "$INSTALL_MODE" in + Raw) install_raw ;; + RAID1) install_raid1 ;; +esac -if [ -n "$DS_URL" ]; then - log "injecting cloud-init datasource: $DS_URL" - retry 20 2 "find root partition" find_root_part || fatal "no root partition found on $TARGET_DISK" - retry 5 2 "mount $ROOT_PART" mount "$ROOT_PART" /mnt || fatal "failed to mount $ROOT_PART" - - mkdir -p /mnt/etc/cloud/cloud.cfg.d /mnt/etc/unbounded-metal - cat > /mnt/etc/cloud/cloud.cfg.d/99-unbounded-metal.cfg < /mnt/etc/unbounded-metal/config </dev/null; then - mount -t efivarfs efivarfs /sys/firmware/efi/efivars 2>/dev/null || true - fi - - for part in $(parts_for_disk); do - if mount -t vfat "$part" /mnt/esp 2>/dev/null; then - LOADER="" - for candidate in \ - /EFI/BOOT/BOOTX64.EFI \ - /EFI/BOOT/BOOTAA64.EFI \ - /EFI/ubuntu/shimx64.efi \ - /EFI/ubuntu/shimaa64.efi; do - if [ -f "/mnt/esp${candidate}" ]; then - LOADER="$(echo "$candidate" | tr '/' '\\')" - break - fi - done - umount /mnt/esp - - if [ -n "$LOADER" ] && command -v efibootmgr >/dev/null 2>&1; then - ESP_NUM=$(partition_number "$part") - if [ -n "$ESP_NUM" ]; then - efibootmgr --create --disk "$TARGET_DISK" --part "$ESP_NUM" \ - --loader "$LOADER" --label "unbounded" 2>&1 \ - && log "UEFI boot entry created (${LOADER} on part ${ESP_NUM})" \ - || log "WARNING: efibootmgr failed, PXE chainloader will be used as fallback" - break - fi - fi - fi - done -fi - -if [ -n "$SERVE_URL" ]; then - log "disabling PXE boot" - retry 5 2 "disable PXE" wget -q -O /dev/null "$SERVE_URL/pxe/disable" \ - || log "WARNING: failed to disable PXE boot" -fi +create_uefi_boot_entries +disable_pxe log "installation complete, rebooting" sleep 2 diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index bcbc3adf6..d4b9f06ef 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -275,6 +275,7 @@ func ServePXECmd() *cobra.Command { Port: dhcpPort, Reader: mgr.GetClient(), ServerIP: dhcpServerIP, + ServeURL: serveURL, OCICache: ociCache, DefaultNetbootRef: defaultNetbootImage, } diff --git a/internal/metalman/dhcp/dhcp.go b/internal/metalman/dhcp/dhcp.go index f1a1b17c3..652f9ff47 100644 --- a/internal/metalman/dhcp/dhcp.go +++ b/internal/metalman/dhcp/dhcp.go @@ -26,6 +26,7 @@ type Server struct { Port int Reader client.Reader ServerIP net.IP + ServeURL string OCICache *netboot.OCICache DefaultNetbootRef string } @@ -120,7 +121,7 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { } mac := strings.ToLower(m.ClientHWAddr.String()) - log := slog.With("mac", mac, "type", m.MessageType().String()) + log := slog.With("mac", mac, "type", m.MessageType().String(), "class", m.ClassIdentifier()) ctx := context.Background() @@ -192,13 +193,29 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { netbootImage = s.DefaultNetbootRef } - if node.Spec.PXE.TargetBootProtocol() != v1alpha3.PXEBootProtocolHTTP && netbootImage != "" && s.OCICache != nil { + if netbootImage != "" && s.OCICache != nil { architecture := node.Spec.PXE.TargetArchitecture() meta, err := s.OCICache.MetadataForRefArchitecture(netbootImage, architecture) if err != nil { log.Warn("OCI image metadata not available", "image", netbootImage, "architecture", architecture, "err", err) - } else if meta.DHCPBootImageName != "" { + } else if node.Spec.PXE.TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP && isHTTPBootClient(m) { + bootPath := netboot.HTTPBootPathFromMetadata(meta) + + bootURL, err := netboot.JoinServeURLPath(s.ServeURL, bootPath) + if err != nil { + log.Warn("HTTP boot URL not available", "image", netbootImage, "architecture", architecture, "err", err) + } else { + resp.UpdateOption(dhcpv4.OptBootFileName(bootURL)) + resp.UpdateOption(dhcpv4.OptClassIdentifier("HTTPClient")) + + if len(bootURL) <= 127 { + resp.BootFileName = bootURL + } else { + log.Warn("HTTP boot URL too long for BOOTP file field; setting DHCP option only", "bootURL", bootURL) + } + } + } else if node.Spec.PXE.TargetBootProtocol() != v1alpha3.PXEBootProtocolHTTP && meta.DHCPBootImageName != "" { resp.UpdateOption(dhcpv4.OptTFTPServerName(s.ServerIP.String())) resp.UpdateOption(dhcpv4.OptBootFileName(meta.DHCPBootImageName)) resp.ServerIPAddr = s.ServerIP @@ -219,9 +236,22 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { dest = peer } - log.Info("sending DHCP response", "node", node.Name, "ip", lease.IPv4, "response", resp.MessageType().String()) + log.Info( + "sending DHCP response", + "node", node.Name, + "ip", lease.IPv4, + "response", resp.MessageType().String(), + "bootfileOption", resp.BootFileNameOption(), + "bootfile", resp.BootFileName, + ) if _, err := conn.WriteTo(resp.ToBytes(), dest); err != nil { log.Error("sending DHCP response", "err", err) } } + +func isHTTPBootClient(m *dhcpv4.DHCPv4) bool { + classID := m.ClassIdentifier() + + return strings.HasPrefix(classID, "HTTPClient") +} diff --git a/internal/metalman/dhcp/dhcp_test.go b/internal/metalman/dhcp/dhcp_test.go index b2e169923..d2f94abd7 100644 --- a/internal/metalman/dhcp/dhcp_test.go +++ b/internal/metalman/dhcp/dhcp_test.go @@ -331,6 +331,93 @@ func TestDHCPHandlerHTTPBootSuppressesPXEBootOptions(t *testing.T) { } } +func TestDHCPHandlerHTTPBootClientGetsBootURL(t *testing.T) { + mac, _ := net.ParseMAC("aa:bb:cc:dd:ee:f4") + serverIP := net.ParseIP("10.0.1.254").To4() + netbootImageRef := "ghcr.io/test/netboot:v1" + + node := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "node-http-client"}, + Spec: v1alpha3.MachineSpec{ + PXE: &v1alpha3.PXESpec{ + NetbootImage: netbootImageRef, + BootProtocol: v1alpha3.PXEBootProtocolHTTP, + DHCPLeases: []v1alpha3.DHCPLease{{ + MAC: "aa:bb:cc:dd:ee:f4", + IPv4: "10.0.1.14", + SubnetMask: "255.255.255.0", + }}, + }, + }, + } + + cacheDir := t.TempDir() + ociCache := netboot.NewOCICache(cacheDir) + + digest := "sha256:httpbootclient123456" + ociCache.SetDigest(netbootImageRef, digest) + + diskDir := filepath.Join(ociCache.DiskDir(digest)) + if err := os.MkdirAll(diskDir, 0o755); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(diskDir, "metadata.yaml"), []byte("dhcpBootImageName: shimx64.efi\nhttpBootPath: efi/grubx64.efi\n"), 0o644); err != nil { + t.Fatal(err) + } + + reader := newFakeReader(t, node) + srv := &Server{ + Interface: "eth0", + Reader: reader, + ServerIP: serverIP, + ServeURL: "http://10.0.1.254:8880/base/", + OCICache: ociCache, + } + + discover, err := dhcpv4.NewDiscovery(mac) + if err != nil { + t.Fatal(err) + } + + discover.UpdateOption(dhcpv4.OptClassIdentifier("HTTPClient:Arch:00016:UNDI:003016")) + + conn := &fakePacketConn{} + peer := &net.UDPAddr{IP: net.ParseIP("10.0.1.14"), Port: 68} + + srv.handler(conn, peer, discover) + + if conn.written == nil { + t.Fatal("expected DHCP response, got none") + } + + resp, err := dhcpv4.FromBytes(conn.written) + if err != nil { + t.Fatal(err) + } + + if tftpServer := resp.TFTPServerName(); tftpServer != "" { + t.Errorf("expected no TFTP server for HTTP boot, got %s", tftpServer) + } + + wantBootfile := "http://10.0.1.254:8880/base/efi/grubx64.efi" + if bootfile := resp.BootFileNameOption(); bootfile != wantBootfile { + t.Errorf("expected HTTP boot URL, got %s", bootfile) + } + + if bootfile := resp.BootFileName; bootfile != wantBootfile { + t.Errorf("expected BOOTP file field HTTP boot URL, got %s", bootfile) + } + + if classID := resp.ClassIdentifier(); classID != "HTTPClient" { + t.Errorf("expected HTTPClient class identifier in response, got %s", classID) + } + + if !resp.YourIPAddr.Equal(net.ParseIP("10.0.1.14")) { + t.Errorf("expected YourIP 10.0.1.14, got %s", resp.YourIPAddr) + } +} + func TestDHCPHandlerUnknownMAC(t *testing.T) { mac, _ := net.ParseMAC("ff:ff:ff:ff:ff:ff") serverIP := net.ParseIP("10.0.1.254").To4() diff --git a/internal/metalman/netboot/http.go b/internal/metalman/netboot/http.go index 4c468e8d3..a4ab9f878 100644 --- a/internal/metalman/netboot/http.go +++ b/internal/metalman/netboot/http.go @@ -112,18 +112,22 @@ func (h *HTTPServer) handleFile(w http.ResponseWriter, r *http.Request) { } imageRef := node.Spec.PXE.Image - if path != "disk.img.gz" { + + resolvePath := path + if strings.HasPrefix(path, "machine/") { + resolvePath = strings.TrimPrefix(path, "machine/") + } else if path != "disk.img.gz" { imageRef = h.NetbootImageRef(node) } - if imageRef == "" { + if imageRef == "" || resolvePath == "" { log.Warn("node has no image for requested path", "node", node.Name) http.NotFound(w, r) return } - resolved, err := h.ResolveFileByPathForIP(r.Context(), path, node, imageRef, ip) + resolved, err := h.ResolveFileByPathForIP(r.Context(), resolvePath, node, imageRef, ip) if err != nil { if errors.Is(err, ErrNotYetDownloaded) { log.Info("file not yet downloaded", "node", node.Name) diff --git a/internal/metalman/netboot/netboot.go b/internal/metalman/netboot/netboot.go index daa70e45a..960c2a789 100644 --- a/internal/metalman/netboot/netboot.go +++ b/internal/metalman/netboot/netboot.go @@ -219,6 +219,12 @@ func (f *FileResolver) ResolveFileByPathForIP(ctx context.Context, path string, } if node != nil { + if path == "grub/grub.cfg" && repavePending(node) && node.Spec.PXE != nil { + if err := node.Spec.PXE.ValidateInstall(); err != nil { + return nil, fmt.Errorf("invalid PXE install config: %w", err) + } + } + ci := f.Cluster.ClusterInfo() agentConfig := provision.BuildAgentConfig(provision.BuildAgentConfigParams{ @@ -258,6 +264,21 @@ func (f *FileResolver) ResolveFileByPathForIP(ctx context.Context, path string, return &ResolvedFile{DiskPath: diskPath}, nil } +func repavePending(node *v1alpha3.Machine) bool { + if node == nil || node.Spec.Operations == nil { + return false + } + + var statusRepave int64 + + specRepave := node.Spec.Operations.RepaveCounter + if node.Status.Operations != nil { + statusRepave = node.Status.Operations.RepaveCounter + } + + return specRepave > statusRepave +} + func (f *FileResolver) resolveUserDataFromConfigMap(ctx context.Context, node *v1alpha3.Machine) ([]byte, bool, error) { if node.Spec.PXE == nil || node.Spec.PXE.CloudInit == nil || node.Spec.PXE.CloudInit.UserDataConfigMapRef == nil { return nil, false, nil @@ -301,6 +322,8 @@ type templateData struct { AgentConfigJSON string InstallScript string InstallEnv []string + InstallMode string + InstallKernelArgs []string SpecRepaveCounter int64 StatusRepaveCounter int64 } @@ -311,10 +334,12 @@ func newTemplateData(node *v1alpha3.Machine, ci ClusterInfo, serveURL, agentConf var ( agent *v1alpha3.AgentSpec bootLease *v1alpha3.DHCPLease + pxe *v1alpha3.PXESpec ) if node != nil { agent = node.Spec.Agent + pxe = node.Spec.PXE bootLease = selectBootLease(node, requestIP) if node.Spec.Operations != nil { @@ -334,11 +359,32 @@ func newTemplateData(node *v1alpha3.Machine, ci ClusterInfo, serveURL, agentConf AgentConfigJSON: agentConfigJSON, InstallScript: provision.UnboundedAgentInstallScript(), InstallEnv: provision.AgentInstallEnv(agent), + InstallMode: pxe.TargetInstallMode(), + InstallKernelArgs: installKernelArgs(pxe), SpecRepaveCounter: specRepave, StatusRepaveCounter: statusRepave, } } +func installKernelArgs(pxe *v1alpha3.PXESpec) []string { + mode := pxe.TargetInstallMode() + args := []string{fmt.Sprintf("unbounded.install_mode=%s", mode)} + disks := pxe.InstallTargetDisks() + + switch mode { + case v1alpha3.PXEInstallModeRAID1: + for i, disk := range disks { + args = append(args, fmt.Sprintf("unbounded.disk%d=%s", i, disk)) + } + default: + if len(disks) > 0 { + args = append(args, fmt.Sprintf("unbounded.disk=%s", disks[0])) + } + } + + return args +} + func selectBootLease(node *v1alpha3.Machine, requestIP string) *v1alpha3.DHCPLease { if node == nil || node.Spec.PXE == nil || len(node.Spec.PXE.DHCPLeases) == 0 { return nil diff --git a/internal/metalman/netboot/netboot_test.go b/internal/metalman/netboot/netboot_test.go index 0345d8ae3..f3920d0f1 100644 --- a/internal/metalman/netboot/netboot_test.go +++ b/internal/metalman/netboot/netboot_test.go @@ -841,6 +841,7 @@ func TestGrubTemplate_MissingOperationsCounters(t *testing.T) { "unbounded.image_url=http://10.0.1.1:8080/disk.img.gz", "unbounded.node_name=node-no-operations", "unbounded.boot_mac=aa:bb:cc:dd:ee:20", + "unbounded.install_mode=Raw", "unbounded.disk=/dev/disk/by-id/test-os-disk", "ip=10.0.1.20::10.0.1.1:255.255.255.0:::none", } { @@ -849,7 +850,92 @@ func TestGrubTemplate_MissingOperationsCounters(t *testing.T) { } } + require.Contains(t, body, "unbounded.install_mode=Raw \\\n unbounded.disk=/dev/disk/by-id/test-os-disk") require.NotContains(t, body, "eth0") + + espConfig := strings.Index(body, "configfile /unbounded/grub.cfg") + directGRUB := strings.Index(body, "chainloader /EFI/BOOT/grubx64.efi") + shim := strings.Index(body, "chainloader /EFI/BOOT/BOOTX64.EFI") + + require.NotEqual(t, -1, espConfig) + require.NotEqual(t, -1, directGRUB) + require.NotEqual(t, -1, shim) + require.Less(t, espConfig, directGRUB) + require.Less(t, directGRUB, shim) +} + +func TestGrubTemplate_RAID1InstallArgs(t *testing.T) { + grubTmpl, err := os.ReadFile(filepath.Join("..", "..", "..", "images", "netboot", "assets", "grub.cfg.tmpl")) + require.NoError(t, err) + + node := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "node-raid", Namespace: "default"}, + Spec: v1alpha3.MachineSpec{ + PXE: &v1alpha3.PXESpec{ + Install: &v1alpha3.PXEInstallSpec{ + Mode: v1alpha3.PXEInstallModeRAID1, + TargetDisks: []string{ + "/dev/disk/by-id/nvme-a", + "/dev/disk/by-id/nvme-b", + }, + }, + }, + Operations: &v1alpha3.OperationsSpec{RepaveCounter: 1}, + }, + } + + data := newTemplateData( + node, + ClusterInfo{ApiserverURL: "https://k8s.example.com"}, + "http://10.0.1.1:8080", + "", + "", + ) + + result, err := renderTemplate(string(grubTmpl), data) + require.NoError(t, err) + + body := string(result) + for _, want := range []string{ + "unbounded.install_mode=RAID1", + "unbounded.disk0=/dev/disk/by-id/nvme-a", + "unbounded.disk1=/dev/disk/by-id/nvme-b", + "unbounded.serve_url=http://10.0.1.1:8080", + } { + require.Contains(t, body, want) + } + + require.Contains(t, body, "unbounded.install_mode=RAID1 \\\n unbounded.disk0=/dev/disk/by-id/nvme-a") + require.Contains(t, body, "unbounded.disk1=/dev/disk/by-id/nvme-b \\\n console=tty0") + require.NotContains(t, body, "unbounded.disk=/dev/disk") +} + +func TestFileResolverRejectsInvalidRAID1InstallBeforeRenderingGRUB(t *testing.T) { + imageRef := "ghcr.io/test/image:v1" + cache := setupOCICache(t, imageRef, "badraid123", map[string][]byte{ + "grub/grub.cfg.tmpl": []byte(`{{ range .InstallKernelArgs }}{{ . }} {{ end }}`), + }) + resolver := FileResolver{ + Cache: cache, + Cluster: &StaticClusterInfo{Info: ClusterInfo{ApiserverURL: "https://k8s.example.com"}}, + ServeURL: "http://10.0.1.1:8080", + } + node := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "node-raid", Namespace: "default"}, + Spec: v1alpha3.MachineSpec{ + PXE: &v1alpha3.PXESpec{ + Install: &v1alpha3.PXEInstallSpec{ + Mode: v1alpha3.PXEInstallModeRAID1, + TargetDisks: []string{"/dev/disk/by-id/nvme-a"}, + }, + }, + Operations: &v1alpha3.OperationsSpec{RepaveCounter: 1}, + }, + } + + _, err := resolver.ResolveFileByPath(t.Context(), "grub/grub.cfg", node, imageRef) + require.ErrorContains(t, err, "invalid PXE install config") + require.ErrorContains(t, err, "RAID1 install requires exactly two targetDisks") } func TestGrubTemplate_SelectsBootLeaseByRequestIP(t *testing.T) { @@ -1939,12 +2025,20 @@ menuentry "Install {{ .Machine.Name }}" { func TestHTTPServer_RoutesDiskFromMachineImageAndBootFromNetbootImage(t *testing.T) { machineData := []byte("machine-disk-data") + rootfsData := []byte("machine-rootfs-data") netbootData := []byte("netboot-kernel-data") cacheDir := t.TempDir() cache := NewOCICache(cacheDir) - if err := populateOCICache(cacheDir, "machine123", map[string][]byte{"disk.img.gz": machineData}); err != nil { + if err := populateOCICache(cacheDir, "machine123", map[string][]byte{ + "disk.img.gz": machineData, + "rootfs.tar.zst": rootfsData, + "esp.tar.zst": []byte("machine-esp-data"), + "install.yaml": []byte("version: 1\n"), + "grub/grub.cfg": []byte("machine-grub-data"), + "grub/grub.cfg.1": []byte("machine-grub-data-1"), + }); err != nil { t.Fatal(err) } @@ -1986,6 +2080,7 @@ func TestHTTPServer_RoutesDiskFromMachineImageAndBootFromNetbootImage(t *testing want []byte }{ {path: "/disk.img.gz", want: machineData}, + {path: "/machine/rootfs.tar.zst", want: rootfsData}, {path: "/vmlinuz", want: netbootData}, } { req, _ := http.NewRequest("GET", ts.URL+tt.path, nil) From 3d7f684f64fb856a01c94cb868982fdc5d566717 Mon Sep 17 00:00:00 2001 From: imiller31 <39311954+imiller31@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:20:18 -0400 Subject: [PATCH 2/5] Address metalman RAID API review --- api/machina/v1alpha3/machine_types.go | 73 +++++++-------- api/machina/v1alpha3/machine_types_test.go | 85 +++++++++++------- api/machina/v1alpha3/zz_generated.deepcopy.go | 12 +-- cmd/metalman/README.md | 9 +- .../crd/unbounded-cloud.io_machines.yaml | 42 ++++----- docs/content/concepts/bare-metal.md | 7 +- docs/content/guides/pxe.md | 21 ++--- docs/content/reference/machina-crd.md | 9 +- hack/smoke-metalman.py | 90 +++++++++++++++++-- internal/metalman/netboot/netboot_test.go | 16 ++-- 10 files changed, 228 insertions(+), 136 deletions(-) diff --git a/api/machina/v1alpha3/machine_types.go b/api/machina/v1alpha3/machine_types.go index 6e78ce9f7..c24d2c61b 100644 --- a/api/machina/v1alpha3/machine_types.go +++ b/api/machina/v1alpha3/machine_types.go @@ -274,6 +274,7 @@ type RedfishSpec struct { } // PXESpec defines PXE boot configuration for a Machine. +// +kubebuilder:validation:XValidation:rule="!has(self.install) || !has(self.install.raidMode) || self.install.raidMode != 'RAID1' || (has(self.targetDisks) && size(self.targetDisks) == 2)",message="RAID1 install requires exactly two targetDisks" type PXESpec struct { // Image is an OCI image reference containing machine install artifacts. // Raw installs require /disk/disk.img.gz. RAID1 installs require @@ -322,11 +323,13 @@ type PXESpec struct { // +optional DHCPLeases []DHCPLease `json:"dhcpLeases,omitempty"` - // TargetDisk is the block device the installer writes the machine image to. + // TargetDisks are explicit whole-disk device paths the installer can write + // the machine image to. Raw installs use the first path when set and + // otherwise choose a disk automatically. RAID1 installs require exactly two + // paths and mirror the root filesystem across them. // Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... - // When omitted, the installer chooses a target disk automatically. // +optional - TargetDisk string `json:"targetDisk,omitempty"` + TargetDisks []string `json:"targetDisks,omitempty"` // Install configures how the PXE installer writes the machine image. // When omitted, the installer uses the legacy raw disk image flow. @@ -343,23 +346,15 @@ type PXESpec struct { CloudInit *CloudInitSpec `json:"cloudInit,omitempty"` } -// PXEInstallSpec configures how the PXE installer writes a machine image. -// +kubebuilder:validation:XValidation:rule="!has(self.mode) || self.mode != 'RAID1' || (has(self.targetDisks) && size(self.targetDisks) == 2)",message="RAID1 install requires exactly two targetDisks" +// PXEInstallSpec configures optional install layout behavior for PXE images. type PXEInstallSpec struct { - // Mode selects the install workflow used by the PXE initrd. - // Raw writes /disk/disk.img.gz directly to one disk. RAID1 creates a - // mirrored root disk from rootfs and ESP tarball machine artifacts. - // +kubebuilder:validation:Enum=Raw;RAID1 - // +kubebuilder:default=Raw - // +optional - Mode string `json:"mode,omitempty"` - - // TargetDisks are explicit whole-disk device paths used by the selected - // install mode. RAID1 requires exactly two paths. Raw uses the first path - // when set and otherwise falls back to targetDisk or automatic selection. - // Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... + // RAIDMode selects the software RAID layout used by the PXE initrd. None + // writes /disk/disk.img.gz directly to one disk. RAID1 creates a mirrored + // root disk from rootfs and ESP tarball machine artifacts. + // +kubebuilder:validation:Enum=None;RAID1 + // +kubebuilder:default=None // +optional - TargetDisks []string `json:"targetDisks,omitempty"` + RAIDMode string `json:"raidMode,omitempty"` } const ( @@ -375,12 +370,18 @@ const ( DefaultPXEArchitecture = PXEArchitectureAMD64 // DefaultPXEBootProtocol is used when spec.pxe.bootProtocol is omitted. DefaultPXEBootProtocol = PXEBootProtocolPXE - // PXEInstallModeRaw writes a raw disk image to a single target disk. + // PXERAIDModeNone disables software RAID for PXE installs. + PXERAIDModeNone = "None" + // PXERAIDModeRAID1 mirrors the PXE-installed root disk. + PXERAIDModeRAID1 = "RAID1" + // DefaultPXERAIDMode is used when spec.pxe.install.raidMode is omitted. + DefaultPXERAIDMode = PXERAIDModeNone + // PXEInstallModeRaw writes a raw disk image to a single target disk. This + // is the installer kernel argument value for non-RAID installs. PXEInstallModeRaw = "Raw" // PXEInstallModeRAID1 creates a mirrored root disk from machine artifacts. + // This is the installer kernel argument value for RAID1 installs. PXEInstallModeRAID1 = "RAID1" - // DefaultPXEInstallMode is used when spec.pxe.install.mode is omitted. - DefaultPXEInstallMode = PXEInstallModeRaw ) // TargetArchitecture returns the effective PXE target architecture. @@ -403,11 +404,21 @@ func (p *PXESpec) TargetBootProtocol() string { // TargetInstallMode returns the effective PXE install mode. func (p *PXESpec) TargetInstallMode() string { - if p == nil || p.Install == nil || p.Install.Mode == "" { - return DefaultPXEInstallMode + switch p.TargetRAIDMode() { + case PXERAIDModeRAID1: + return PXEInstallModeRAID1 + default: + return PXEInstallModeRaw } +} - return p.Install.Mode +// TargetRAIDMode returns the effective PXE software RAID mode. +func (p *PXESpec) TargetRAIDMode() string { + if p == nil || p.Install == nil || p.Install.RAIDMode == "" { + return DefaultPXERAIDMode + } + + return p.Install.RAIDMode } // InstallTargetDisks returns the effective installer target disk list. @@ -416,22 +427,14 @@ func (p *PXESpec) InstallTargetDisks() []string { return nil } - if p.Install != nil && len(p.Install.TargetDisks) > 0 { - return p.Install.TargetDisks - } - - if p.TargetDisk != "" { - return []string{p.TargetDisk} - } - - return nil + return p.TargetDisks } // ValidateInstall returns an error when the PXE install configuration cannot // be rendered into a safe installer command line. func (p *PXESpec) ValidateInstall() error { - switch p.TargetInstallMode() { - case PXEInstallModeRAID1: + switch p.TargetRAIDMode() { + case PXERAIDModeRAID1: disks := p.InstallTargetDisks() if len(disks) != 2 { return fmt.Errorf("RAID1 install requires exactly two targetDisks, got %d", len(disks)) diff --git a/api/machina/v1alpha3/machine_types_test.go b/api/machina/v1alpha3/machine_types_test.go index 3e2a94e5c..7e747beac 100644 --- a/api/machina/v1alpha3/machine_types_test.go +++ b/api/machina/v1alpha3/machine_types_test.go @@ -28,29 +28,60 @@ func TestKubernetesSpecOmitsUnsetBootstrapTokenRef(t *testing.T) { } } -func TestPXESpecTargetInstallMode(t *testing.T) { +func TestPXESpecTargetRAIDMode(t *testing.T) { tests := []struct { name string pxe *PXESpec want string }{ { - name: "nil pxe defaults raw", - want: PXEInstallModeRaw, + name: "nil pxe defaults none", + want: PXERAIDModeNone, }, { - name: "nil install defaults raw", + name: "nil install defaults none", pxe: &PXESpec{}, - want: PXEInstallModeRaw, + want: PXERAIDModeNone, }, { - name: "empty install mode defaults raw", + name: "empty install raid mode defaults none", pxe: &PXESpec{Install: &PXEInstallSpec{}}, - want: PXEInstallModeRaw, + want: PXERAIDModeNone, }, { name: "explicit raid1", - pxe: &PXESpec{Install: &PXEInstallSpec{Mode: PXEInstallModeRAID1}}, + pxe: &PXESpec{Install: &PXEInstallSpec{RAIDMode: PXERAIDModeRAID1}}, + want: PXERAIDModeRAID1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.pxe.TargetRAIDMode(); got != tt.want { + t.Fatalf("TargetRAIDMode() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestPXESpecTargetInstallMode(t *testing.T) { + tests := []struct { + name string + pxe *PXESpec + want string + }{ + { + name: "nil pxe defaults raw", + want: PXEInstallModeRaw, + }, + { + name: "none raid mode maps to raw install", + pxe: &PXESpec{Install: &PXEInstallSpec{RAIDMode: PXERAIDModeNone}}, + want: PXEInstallModeRaw, + }, + { + name: "raid1 maps to raid1 install", + pxe: &PXESpec{Install: &PXEInstallSpec{RAIDMode: PXERAIDModeRAID1}}, want: PXEInstallModeRAID1, }, } @@ -74,18 +105,8 @@ func TestPXESpecInstallTargetDisks(t *testing.T) { name: "nil pxe", }, { - name: "legacy target disk", - pxe: &PXESpec{TargetDisk: "/dev/disk/by-id/os"}, - want: []string{"/dev/disk/by-id/os"}, - }, - { - name: "install target disks override legacy target disk", - pxe: &PXESpec{ - TargetDisk: "/dev/disk/by-id/legacy", - Install: &PXEInstallSpec{ - TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}, - }, - }, + name: "top-level target disks", + pxe: &PXESpec{TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}}, want: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}, }, } @@ -122,24 +143,22 @@ func TestPXESpecValidateInstall(t *testing.T) { { name: "raw allows explicit disk", pxe: &PXESpec{ - Install: &PXEInstallSpec{ - TargetDisks: []string{"/dev/disk/by-id/os"}, - }, + TargetDisks: []string{"/dev/disk/by-id/os"}, }, }, { name: "raid1 requires disks", pxe: &PXESpec{ - Install: &PXEInstallSpec{Mode: PXEInstallModeRAID1}, + Install: &PXEInstallSpec{RAIDMode: PXERAIDModeRAID1}, }, wantErr: true, }, { name: "raid1 rejects one disk", pxe: &PXESpec{ + TargetDisks: []string{"/dev/disk/by-id/a"}, Install: &PXEInstallSpec{ - Mode: PXEInstallModeRAID1, - TargetDisks: []string{"/dev/disk/by-id/a"}, + RAIDMode: PXERAIDModeRAID1, }, }, wantErr: true, @@ -147,27 +166,27 @@ func TestPXESpecValidateInstall(t *testing.T) { { name: "raid1 accepts two disks", pxe: &PXESpec{ + TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}, Install: &PXEInstallSpec{ - Mode: PXEInstallModeRAID1, - TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b"}, + RAIDMode: PXERAIDModeRAID1, }, }, }, { name: "raid1 rejects three disks", pxe: &PXESpec{ + TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b", "/dev/disk/by-id/c"}, Install: &PXEInstallSpec{ - Mode: PXEInstallModeRAID1, - TargetDisks: []string{"/dev/disk/by-id/a", "/dev/disk/by-id/b", "/dev/disk/by-id/c"}, + RAIDMode: PXERAIDModeRAID1, }, }, wantErr: true, }, { - name: "raid1 does not use legacy single disk", + name: "raid1 does not use a single top-level disk", pxe: &PXESpec{ - TargetDisk: "/dev/disk/by-id/legacy", - Install: &PXEInstallSpec{Mode: PXEInstallModeRAID1}, + TargetDisks: []string{"/dev/disk/by-id/os"}, + Install: &PXEInstallSpec{RAIDMode: PXERAIDModeRAID1}, }, wantErr: true, }, diff --git a/api/machina/v1alpha3/zz_generated.deepcopy.go b/api/machina/v1alpha3/zz_generated.deepcopy.go index 9415c6ca2..90db19c66 100644 --- a/api/machina/v1alpha3/zz_generated.deepcopy.go +++ b/api/machina/v1alpha3/zz_generated.deepcopy.go @@ -1017,11 +1017,6 @@ func (in *OperationsStatus) DeepCopy() *OperationsStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PXEInstallSpec) DeepCopyInto(out *PXEInstallSpec) { *out = *in - if in.TargetDisks != nil { - in, out := &in.TargetDisks, &out.TargetDisks - *out = make([]string, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PXEInstallSpec. @@ -1054,10 +1049,15 @@ func (in *PXESpec) DeepCopyInto(out *PXESpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.TargetDisks != nil { + in, out := &in.TargetDisks, &out.TargetDisks + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Install != nil { in, out := &in.Install, &out.Install *out = new(PXEInstallSpec) - (*in).DeepCopyInto(*out) + **out = **in } if in.Redfish != nil { in, out := &in.Redfish, &out.Redfish diff --git a/cmd/metalman/README.md b/cmd/metalman/README.md index 3c4563b64..8b0caac0b 100644 --- a/cmd/metalman/README.md +++ b/cmd/metalman/README.md @@ -190,7 +190,8 @@ spec: # Defaults to PXE. Set to HTTP to use UEFI HTTP boot. bootProtocol: PXE # Optional. Recommended when the host has multiple disks. - targetDisk: /dev/disk/by-id/example-os-disk + targetDisks: + - /dev/disk/by-id/example-os-disk dhcpLeases: - mac: "aa:bb:cc:dd:ee:01" ipv4: "10.0.0.11" @@ -205,9 +206,9 @@ manually PXE-booted (or have PXE as its default boot option). The default netboot template passes the matching DHCP lease MAC to the installer initrd, which uses it to select the provisioning NIC instead of assuming a fixed -interface name such as `eth0`. If `spec.pxe.targetDisk` is set, the installer -writes the image to that disk; otherwise it falls back to automatic disk -selection. +interface name such as `eth0`. If `spec.pxe.targetDisks` is set, raw installs +write the image to the first disk in the list; otherwise the installer falls +back to automatic disk selection. #### BMC diff --git a/deploy/machina/crd/unbounded-cloud.io_machines.yaml b/deploy/machina/crd/unbounded-cloud.io_machines.yaml index ae00d886e..73e25f352 100644 --- a/deploy/machina/crd/unbounded-cloud.io_machines.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_machines.yaml @@ -407,30 +407,17 @@ spec: Install configures how the PXE installer writes the machine image. When omitted, the installer uses the legacy raw disk image flow. properties: - mode: - default: Raw + raidMode: + default: None description: |- - Mode selects the install workflow used by the PXE initrd. - Raw writes /disk/disk.img.gz directly to one disk. RAID1 creates a - mirrored root disk from rootfs and ESP tarball machine artifacts. + RAIDMode selects the software RAID layout used by the PXE initrd. None + writes /disk/disk.img.gz directly to one disk. RAID1 creates a mirrored + root disk from rootfs and ESP tarball machine artifacts. enum: - - Raw + - None - RAID1 type: string - targetDisks: - description: |- - TargetDisks are explicit whole-disk device paths used by the selected - install mode. RAID1 requires exactly two paths. Raw uses the first path - when set and otherwise falls back to targetDisk or automatic selection. - Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... - items: - type: string - type: array type: object - x-kubernetes-validations: - - message: RAID1 install requires exactly two targetDisks - rule: '!has(self.mode) || self.mode != ''RAID1'' || (has(self.targetDisks) - && size(self.targetDisks) == 2)' netbootImage: description: |- NetbootImage is an OCI image reference containing the PXE boot @@ -512,15 +499,24 @@ spec: - url - username type: object - targetDisk: + targetDisks: description: |- - TargetDisk is the block device the installer writes the machine image to. + TargetDisks are explicit whole-disk device paths the installer can write + the machine image to. Raw installs use the first path when set and + otherwise choose a disk automatically. RAID1 installs require exactly two + paths and mirror the root filesystem across them. Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... - When omitted, the installer chooses a target disk automatically. - type: string + items: + type: string + type: array required: - image type: object + x-kubernetes-validations: + - message: RAID1 install requires exactly two targetDisks + rule: '!has(self.install) || !has(self.install.raidMode) || self.install.raidMode + != ''RAID1'' || (has(self.targetDisks) && size(self.targetDisks) + == 2)' ssh: description: |- SSH contains the SSH connection and credential details for the diff --git a/docs/content/concepts/bare-metal.md b/docs/content/concepts/bare-metal.md index 9f6e65f09..ce343a270 100644 --- a/docs/content/concepts/bare-metal.md +++ b/docs/content/concepts/bare-metal.md @@ -101,9 +101,10 @@ For PXE-provisioned machines, the `Machine` resource includes: assignment for each interface. During install, the default netboot template passes the matching lease MAC to the initrd so it can select the provisioning NIC without relying on names such as `eth0`. -- **`spec.pxe.targetDisk`** -- Optional block device path for the disk that - receives the machine image. Set this on hosts with multiple disks; when - omitted, the installer selects a disk automatically. +- **`spec.pxe.targetDisks`** -- Optional block device paths for disks that can + receive the machine image. Raw installs use the first path when set; RAID1 + installs require exactly two paths. When omitted for raw installs, the + installer selects a disk automatically. - **`spec.pxe.redfish`** -- Optional BMC connection details (endpoint, username, password secret) for remote power management. - **`spec.pxe.cloudInit`** -- Optional cloud-init customization. References a diff --git a/docs/content/guides/pxe.md b/docs/content/guides/pxe.md index 80db3fe82..0ee618f05 100644 --- a/docs/content/guides/pxe.md +++ b/docs/content/guides/pxe.md @@ -55,9 +55,11 @@ Metalman uses a machine image and a netboot image for each PXE repave. - `spec.pxe.architecture` selects the target architecture (`amd64` or `arm64`) used when pulling machine and netboot image platform manifests. It defaults to `amd64`. -- `spec.pxe.install` selects the install workflow. When omitted, metalman uses - the legacy raw single-disk workflow. `RAID1` requires exactly two explicit - whole-disk paths in `spec.pxe.install.targetDisks`. +- `spec.pxe.targetDisks` optionally selects explicit whole-disk paths. Raw + installs use the first path when set and otherwise choose a disk + automatically. `RAID1` requires exactly two paths. +- `spec.pxe.install.raidMode` selects optional software RAID layout. When + omitted or set to `None`, metalman uses the legacy raw single-disk workflow. - `spec.pxe.netbootImage` is the reusable PXE boot environment. It contains bootloaders, kernel, initrd, templates, metadata, and `unbounded-agent`. If omitted, Metalman uses the release-matched `--default-netboot-image`. @@ -113,14 +115,13 @@ spec: bootProtocol: PXE # Optional. Omit to use Metalman's default netboot image. netbootImage: ghcr.io/azure/netboot:v1 - # Raw mode only. Recommended on hosts with multiple disks. - # targetDisk: /dev/disk/by-id/example-os-disk + # Optional. Raw uses the first path; RAID1 requires exactly two paths. + targetDisks: + - /dev/disk/by-id/example-os-disk-a + - /dev/disk/by-id/example-os-disk-b # Optional. Omit for the legacy Raw single-disk install mode. install: - mode: RAID1 - targetDisks: - - /dev/disk/by-id/example-os-disk-a - - /dev/disk/by-id/example-os-disk-b + raidMode: RAID1 dhcpLeases: - ipv4: "10.10.0.50" mac: "aa:bb:cc:dd:ee:ff" @@ -199,7 +200,7 @@ If the referenced ConfigMap does not exist, metalman falls back to the default m 3. **GRUB decision.** A rendered `grub.cfg` (from a `.tmpl` file in the netboot image) checks `repaveCounter` against status: if counter is ahead, boot the PXE installer; otherwise chainload the local OS. When a Machine has multiple DHCP leases, metalman renders the lease matching the request source IP and passes that lease's MAC as `unbounded.boot_mac`. 4. **Installer (initrd overlay).** An init script in the initrd: - Loads storage and network drivers, selects the provisioning NIC by MAC, and configures the static IP from kernel cmdline. - - In `Raw` mode, downloads the gzip-compressed raw disk image from the machine image over HTTP and writes it to `spec.pxe.targetDisk`, `spec.pxe.install.targetDisks[0]`, or an automatically selected block device. + - In `Raw` mode, downloads the gzip-compressed raw disk image from the machine image over HTTP and writes it to `spec.pxe.targetDisks[0]` or an automatically selected block device. - In `RAID1` mode, requires two explicit whole-disk paths, creates an ESP on each disk, creates a mirrored root array, extracts the rootfs and ESP artifacts, writes `mdadm.conf` and `fstab`, and refreshes host boot artifacts. The host image must include mdadm, initramfs, and GRUB tooling or the installer fails before disabling PXE. - Mounts the root filesystem and injects cloud-init config and the agent configuration. - Calls `/pxe/disable` on metalman to signal completion, then reboots. diff --git a/docs/content/reference/machina-crd.md b/docs/content/reference/machina-crd.md index 7b4765d22..1daf72c2b 100644 --- a/docs/content/reference/machina-crd.md +++ b/docs/content/reference/machina-crd.md @@ -62,10 +62,9 @@ PXE boot configuration consumed by the metalman controller. | `pxe.dhcpLeases[].subnetMask` | string | Yes | - | Subnet mask. | | `pxe.dhcpLeases[].gateway` | string | Yes | - | Default gateway. | | `pxe.dhcpLeases[].dns` | []string | No | - | DNS server addresses. | -| `pxe.targetDisk` | string | No | Installer-selected | Legacy raw-mode block device override, such as `/dev/nvme0n1` or `/dev/disk/by-id/...`. When omitted in `Raw` mode, the initrd selects a disk automatically. | -| `pxe.install` | PXEInstallSpec | No | `Raw` | Installer workflow configuration. Omit for the legacy raw single-disk workflow. | -| `pxe.install.mode` | string | No | `Raw` | Install workflow. Allowed values: `Raw`, `RAID1`. | -| `pxe.install.targetDisks` | []string | No | - | Explicit whole-disk device paths. `RAID1` requires exactly two paths and does not fall back to automatic disk selection. `Raw` uses the first path when set. | +| `pxe.targetDisks` | []string | No | Installer-selected | Explicit whole-disk device paths, such as `/dev/nvme0n1` or `/dev/disk/by-id/...`. Raw installs use the first path when set. RAID1 installs require exactly two paths. | +| `pxe.install` | PXEInstallSpec | No | `None` | Optional install layout configuration. Omit for the legacy raw single-disk workflow. | +| `pxe.install.raidMode` | string | No | `None` | Software RAID layout. Allowed values: `None`, `RAID1`. | | `pxe.redfish` | RedfishSpec | No | - | BMC access via the Redfish API. | | `pxe.redfish.url` | string | Yes | - | Redfish endpoint URL. | | `pxe.redfish.username` | string | Yes | - | Redfish username. | @@ -454,7 +453,7 @@ interface instead of relying on kernel interface names such as `eth0`. In `Raw` mode, the template passes a configured disk as `unbounded.disk` and the installer falls back to automatic disk selection when no disk is configured. In `RAID1` mode, the template passes `unbounded.disk0` and `unbounded.disk1` -from `spec.pxe.install.targetDisks`; the installer fails closed when either +from `spec.pxe.targetDisks`; the installer fails closed when either disk is missing or invalid. RAID1 host images must include mdadm, update-initramfs, and GRUB tooling so the installer can refresh boot artifacts for the mirrored root before disabling PXE. diff --git a/hack/smoke-metalman.py b/hack/smoke-metalman.py index e28e3aa71..5bdc84fb7 100755 --- a/hack/smoke-metalman.py +++ b/hack/smoke-metalman.py @@ -1135,6 +1135,7 @@ def prepare_http_boot_helper_disk(raw_image: str, dest: Path) -> None: log(" Injecting one-shot efibootmgr selector") run([ "virt-customize", + "--no-network", "-a", str(disk_raw), "--copy-in", f"{script}:/usr/local/sbin", "--copy-in", f"{service}:/etc/systemd/system", @@ -1268,6 +1269,53 @@ def virt_tar_zst( die(f"Failed to export {mount_device}: virt-tar-out={tar_rc}, zstd={zstd_rc}") +def apt_dependency_closure(packages: list[str]) -> list[str]: + result = run([ + "apt-cache", + "depends", + "--recurse", + "--no-recommends", + "--no-suggests", + "--no-conflicts", + "--no-breaks", + "--no-replaces", + "--no-enhances", + *packages, + ], capture_output=True, text=True) + + deps = set(packages) + for raw_line in result.stdout.splitlines(): + line = raw_line.strip().lstrip("|").strip() + if not line or line.startswith("<"): + continue + + if ":" in line: + field, value = line.split(":", 1) + if field not in ("Depends", "PreDepends"): + continue + name = value.strip().split()[0] + else: + name = line.split()[0] + + if not name or name.startswith("<"): + continue + deps.add(name.split(":", 1)[0]) + + return sorted(deps) + + +def download_debs(packages: list[str], dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + debs = list(dest.glob("*.deb")) + if debs: + for deb in debs: + deb.unlink() + + deps = apt_dependency_closure(packages) + log(f" Downloading {len(deps)} RAID boot package(s) on the host") + run(["apt-get", "download", *deps], cwd=str(dest)) + + def prepare_raid_machine_image(raw_image: str, raid_image: str) -> None: log("Preparing RAID1 machine image from host-ubuntu2404") require_tools([ @@ -1279,6 +1327,7 @@ def prepare_raid_machine_image(raw_image: str, raid_image: str) -> None: workdir.mkdir(parents=True, exist_ok=True) disk_gz = workdir / "disk.img.gz" disk_raw = workdir / "disk.img" + deb_dir = workdir / "raid-debs" log(f" Extracting /disk/disk.img.gz from {raw_image}") docker_copy_from_image(raw_image, "/disk/disk.img.gz", disk_gz) @@ -1286,14 +1335,37 @@ def prepare_raid_machine_image(raw_image: str, raid_image: str) -> None: shutil.copyfileobj(src, dst) disk_gz.unlink() + packages = [ + "mdadm", + "grub-efi-amd64", + "grub-efi-amd64-bin", + "shim-signed", + "qemu-guest-agent", + ] + log(" Installing RAID boot dependencies into copied host image") - run([ - "virt-customize", - "-a", str(disk_raw), - "--run-command", "apt-get update", - "--install", "mdadm,grub-efi-amd64,grub-efi-amd64-bin,shim-signed,qemu-guest-agent", - "--run-command", "systemctl enable qemu-guest-agent || true", - ], env=guestfs_env()) + if shutil.which("apt-cache") and shutil.which("apt-get"): + download_debs(packages, deb_dir) + run([ + "virt-customize", + "--no-network", + "-a", str(disk_raw), + "--copy-in", f"{deb_dir}:/tmp", + "--run-command", + "DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/raid-debs/*.deb || " + "DEBIAN_FRONTEND=noninteractive apt-get -f install -y --no-download", + "--run-command", "systemctl enable qemu-guest-agent || true", + "--run-command", "rm -rf /tmp/raid-debs", + ], env=guestfs_env()) + else: + log(" Host apt tools unavailable; using guest-networked virt-customize install") + run([ + "virt-customize", + "-a", str(disk_raw), + "--run-command", "apt-get update", + "--install", ",".join(packages), + "--run-command", "systemctl enable qemu-guest-agent || true", + ], env=guestfs_env()) filesystems = virt_filesystems(disk_raw) root_part = find_root_partition(disk_raw, filesystems) @@ -1686,9 +1758,9 @@ def main() -> None: }, } if INSTALL_MODE == "RAID1": + protonode["spec"]["pxe"]["targetDisks"] = [RAID_DISK_A_PATH, RAID_DISK_B_PATH] protonode["spec"]["pxe"]["install"] = { - "mode": "RAID1", - "targetDisks": [RAID_DISK_A_PATH, RAID_DISK_B_PATH], + "raidMode": "RAID1", } if BOOT_PROTOCOL == "HTTP": protonode["spec"]["operations"] = { diff --git a/internal/metalman/netboot/netboot_test.go b/internal/metalman/netboot/netboot_test.go index f3920d0f1..d21dde3cc 100644 --- a/internal/metalman/netboot/netboot_test.go +++ b/internal/metalman/netboot/netboot_test.go @@ -811,7 +811,7 @@ func TestGrubTemplate_MissingOperationsCounters(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node-no-operations", Namespace: "default"}, Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ - TargetDisk: "/dev/disk/by-id/test-os-disk", + TargetDisks: []string{"/dev/disk/by-id/test-os-disk"}, DHCPLeases: []v1alpha3.DHCPLease{{ IPv4: "10.0.1.20", MAC: "aa:bb:cc:dd:ee:20", @@ -872,12 +872,12 @@ func TestGrubTemplate_RAID1InstallArgs(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node-raid", Namespace: "default"}, Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ + TargetDisks: []string{ + "/dev/disk/by-id/nvme-a", + "/dev/disk/by-id/nvme-b", + }, Install: &v1alpha3.PXEInstallSpec{ - Mode: v1alpha3.PXEInstallModeRAID1, - TargetDisks: []string{ - "/dev/disk/by-id/nvme-a", - "/dev/disk/by-id/nvme-b", - }, + RAIDMode: v1alpha3.PXERAIDModeRAID1, }, }, Operations: &v1alpha3.OperationsSpec{RepaveCounter: 1}, @@ -924,9 +924,9 @@ func TestFileResolverRejectsInvalidRAID1InstallBeforeRenderingGRUB(t *testing.T) ObjectMeta: metav1.ObjectMeta{Name: "node-raid", Namespace: "default"}, Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ + TargetDisks: []string{"/dev/disk/by-id/nvme-a"}, Install: &v1alpha3.PXEInstallSpec{ - Mode: v1alpha3.PXEInstallModeRAID1, - TargetDisks: []string{"/dev/disk/by-id/nvme-a"}, + RAIDMode: v1alpha3.PXERAIDModeRAID1, }, }, Operations: &v1alpha3.OperationsSpec{RepaveCounter: 1}, From 7d61ae4a2d71882d17b2250738b3d9da35f1eaca Mon Sep 17 00:00:00 2001 From: imiller31 <39311954+imiller31@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:46:19 -0400 Subject: [PATCH 3/5] Fix RAID smoke image export --- hack/smoke-metalman.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hack/smoke-metalman.py b/hack/smoke-metalman.py index 5bdc84fb7..43e667664 100755 --- a/hack/smoke-metalman.py +++ b/hack/smoke-metalman.py @@ -1238,7 +1238,7 @@ def find_boot_partition(image: Path, filesystems: list[tuple[str, str, str]], ro return None -def virt_tar_zst( +def guestfish_tar_zst( image: Path, mount_device: str, dest: Path, @@ -1252,12 +1252,12 @@ def virt_tar_zst( mount_desc = ", ".join([mount_device] + [f"{device} at {mountpoint}" for device, mountpoint in extra_mounts]) log(f" Exporting {mount_desc} to {dest.name}") tar_proc = subprocess.Popen( - ["virt-tar-out", "--ro", "--no-sync", "-a", str(image), *mount_args, "/", "-"], + ["guestfish", "--ro", "--no-sync", "-a", str(image), *mount_args, "tar-out", "/", "-"], stdout=subprocess.PIPE, env=guestfs_env(), ) if tar_proc.stdout is None: - die("virt-tar-out did not provide stdout") + die("guestfish tar-out did not provide stdout") zstd_proc = subprocess.Popen( ["zstd", "-T0", "-q", "-f", "-o", str(dest), "-"], stdin=tar_proc.stdout, @@ -1266,7 +1266,7 @@ def virt_tar_zst( zstd_rc = zstd_proc.wait() tar_rc = tar_proc.wait() if tar_rc != 0 or zstd_rc != 0: - die(f"Failed to export {mount_device}: virt-tar-out={tar_rc}, zstd={zstd_rc}") + die(f"Failed to export {mount_device}: guestfish={tar_rc}, zstd={zstd_rc}") def apt_dependency_closure(packages: list[str]) -> list[str]: @@ -1319,8 +1319,8 @@ def download_debs(packages: list[str], dest: Path) -> None: def prepare_raid_machine_image(raw_image: str, raid_image: str) -> None: log("Preparing RAID1 machine image from host-ubuntu2404") require_tools([ - "docker", "virt-customize", "virt-filesystems", "virt-cat", "virt-ls", - "virt-tar-out", "zstd", + "docker", "guestfish", "virt-customize", "virt-filesystems", "virt-cat", + "virt-ls", "zstd", ]) workdir = TMPDIR / "raid-machine-image" @@ -1374,8 +1374,8 @@ def prepare_raid_machine_image(raw_image: str, raid_image: str) -> None: boot_mounts = [(boot_part, "/boot")] if boot_part else [] log(f" Identified root={root_part} boot={boot_part or 'inline'} esp={esp_part}") - virt_tar_zst(disk_raw, root_part, workdir / "rootfs.tar.zst", boot_mounts) - virt_tar_zst(disk_raw, esp_part, workdir / "esp.tar.zst") + guestfish_tar_zst(disk_raw, root_part, workdir / "rootfs.tar.zst", boot_mounts) + guestfish_tar_zst(disk_raw, esp_part, workdir / "esp.tar.zst") (workdir / "install.yaml").write_text("version: 1\nmode: RAID1\n", encoding="utf-8") (workdir / "Containerfile").write_text(textwrap.dedent("""\ FROM scratch From 8e217ed21f8e6efbf04b4b6398891a32507d0dcc Mon Sep 17 00:00:00 2001 From: imiller31 <39311954+imiller31@users.noreply.github.com> Date: Wed, 8 Jul 2026 12:03:57 -0400 Subject: [PATCH 4/5] Harden metalman smoke dependency setup --- .github/workflows/smoke-metalman.yaml | 40 +++++++++++++++++++++------ 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/smoke-metalman.yaml b/.github/workflows/smoke-metalman.yaml index 7fbcf3715..a46e24f9d 100644 --- a/.github/workflows/smoke-metalman.yaml +++ b/.github/workflows/smoke-metalman.yaml @@ -90,13 +90,35 @@ jobs: - name: Install system dependencies run: | - sudo apt-get update - # Install sushy-tools via pip in the background while apt-get - # installs system packages. pip3 is already available on the - # runner image so this does not depend on apt-get. - sudo pip3 install --break-system-packages --ignore-installed sushy-tools & - PIP_PID=$! - sudo apt-get install -y --no-install-recommends \ + retry_root() { + local timeout_seconds=$1 + shift + + local attempt status + for attempt in 1 2 3; do + if sudo timeout --kill-after=30s "${timeout_seconds}s" "$@"; then + return 0 + fi + + status=$? + if [ "$attempt" -eq 3 ]; then + return "$status" + fi + + echo "command failed with status ${status} (attempt ${attempt}/3); retrying" + sleep $((attempt * 10)) + done + } + + APT_OPTS=( + -o Acquire::Retries=3 + -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30 + -o DPkg::Lock::Timeout=120 + ) + + retry_root 300 apt-get "${APT_OPTS[@]}" update + retry_root 600 env DEBIAN_FRONTEND=noninteractive apt-get "${APT_OPTS[@]}" install -y --no-install-recommends \ dnsmasq qemu-kvm libvirt-daemon-system libvirt-clients virtinst \ ovmf qemu-utils swtpm swtpm-tools \ python3-pip python3-yaml cpio iptables @@ -108,10 +130,10 @@ jobs: EXTRA_PKGS+=(zstd) fi if [ "${#EXTRA_PKGS[@]}" -gt 0 ]; then - sudo apt-get install -y --no-install-recommends "${EXTRA_PKGS[@]}" + retry_root 600 env DEBIAN_FRONTEND=noninteractive apt-get "${APT_OPTS[@]}" install -y --no-install-recommends "${EXTRA_PKGS[@]}" fi + retry_root 300 pip3 install --break-system-packages --ignore-installed sushy-tools sudo systemctl start libvirtd - wait "$PIP_PID" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 From 0a0d0b039ceb1b8ac66114deab867d65c2d6d07b Mon Sep 17 00:00:00 2001 From: imiller31 <39311954+imiller31@users.noreply.github.com> Date: Wed, 8 Jul 2026 12:19:22 -0400 Subject: [PATCH 5/5] Free space during metalman RAID smoke --- hack/smoke-metalman.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/hack/smoke-metalman.py b/hack/smoke-metalman.py index 43e667664..521c1ecef 100755 --- a/hack/smoke-metalman.py +++ b/hack/smoke-metalman.py @@ -1148,6 +1148,7 @@ def prepare_http_boot_helper_disk(raw_image: str, dest: Path) -> None: log(" Converting helper disk to qcow2") run(["qemu-img", "convert", "-f", "raw", "-O", "qcow2", str(disk_raw), str(dest)]) run(["qemu-img", "resize", str(dest), "20G"]) + shutil.rmtree(workdir, ignore_errors=True) def virt_filesystems(image: Path) -> list[tuple[str, str, str]]: @@ -1386,6 +1387,7 @@ def prepare_raid_machine_image(raw_image: str, raid_image: str) -> None: log(f" Building {raid_image}") run(["docker", "build", "-t", raid_image, "-f", str(workdir / "Containerfile"), str(workdir)]) + shutil.rmtree(workdir, ignore_errors=True) def assert_raid1_install() -> None: @@ -1709,12 +1711,14 @@ def main() -> None: run(["docker", "push", NETBOOT_IMAGE_NAME]) run(["docker", "push", AGENT_IMAGE_NAME]) - # Reclaim disk space consumed by Docker build cache. The host-ubuntu2404 - # build downloads a ~2 GB Ubuntu cloud image and converts it to raw; the - # intermediate layers are no longer needed once the images are pushed. - # Only prune the build cache (not running container images) to avoid - # disturbing the registry container. - log("Pruning Docker build cache to free disk space") + # Reclaim disk space after pushing to the local registry. Metalman pulls + # from the registry, so the duplicate loaded image tags and BuildKit cache + # are no longer needed. Keep the registry container itself running. + pushed_images = [IMAGE_NAME, NETBOOT_IMAGE_NAME, AGENT_IMAGE_NAME] + if IMAGE_NAME != RAW_IMAGE_NAME: + pushed_images.append(RAW_IMAGE_NAME) + log("Removing duplicate loaded OCI images and BuildKit cache") + run_quiet(["docker", "image", "rm", "-f", *dict.fromkeys(pushed_images)], check=False) run_quiet(["docker", "builder", "prune", "-af"], check=False) server_url = apiserver_url()