From 2b0b298f5322923cb8bdc0585a64a8f934beba69 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:04:43 +0000 Subject: [PATCH 01/43] metalman: separate netboot configuration axes --- api/machina/v1alpha3/machine_types.go | 72 ++++++++++---- .../v1alpha3/machine_validation_test.go | 27 ++++++ .../crd/unbounded-cloud.io_machines.yaml | 97 +++++++++++++++---- internal/metalman/dhcp/dhcp.go | 2 +- internal/metalman/dhcp/dhcp_test.go | 4 +- internal/metalman/machineops/controller.go | 4 +- .../metalman/machineops/controller_test.go | 10 +- internal/metalman/netboot/http.go | 6 +- internal/metalman/netboot/netboot_test.go | 12 +-- 9 files changed, 177 insertions(+), 57 deletions(-) diff --git a/api/machina/v1alpha3/machine_types.go b/api/machina/v1alpha3/machine_types.go index e590f5481..f4735005c 100644 --- a/api/machina/v1alpha3/machine_types.go +++ b/api/machina/v1alpha3/machine_types.go @@ -354,7 +354,11 @@ type RedfishSpec struct { PasswordRef SecretKeySelector `json:"passwordRef"` } -// PXESpec defines PXE boot configuration for a Machine. +// +kubebuilder:validation:XValidation:rule="!(self.transport == 'TFTP' && self.configurationSource == 'Redfish')",message="TFTP transport requires DHCP configuration" +// +kubebuilder:validation:XValidation:rule="self.networkMode != 'Static' || self.configurationSource == 'Redfish'",message="static networking requires Redfish configuration" +// +kubebuilder:validation:XValidation:rule="self.configurationSource != 'Redfish' || has(self.redfish)",message="Redfish configuration requires redfish connection details" + +// PXESpec defines network boot configuration for a Machine. type PXESpec struct { // Image is an OCI image reference containing the machine disk image. // The image must contain /disk/disk.img.gz. @@ -389,13 +393,24 @@ type PXESpec struct { // +optional NetbootPullSecretRef *NamespacedSecretReference `json:"netbootPullSecretRef,omitempty"` - // BootProtocol selects how metalman should trigger network boot for - // repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - // HTTP boot with a URL derived from the netboot image metadata. - // +kubebuilder:validation:Enum=PXE;HTTP - // +kubebuilder:default=PXE + // Transport selects the firmware boot artifact transport. + // +kubebuilder:default=TFTP + // +optional + Transport NetbootTransport `json:"transport,omitempty"` + + // ConfigurationSource selects how firmware receives its boot target. + // +kubebuilder:default=DHCP + // +optional + ConfigurationSource NetbootConfigurationSource `json:"configurationSource,omitempty"` + + // NetworkMode selects how firmware configures the provisioning network. + // +kubebuilder:default=DHCP // +optional - BootProtocol string `json:"bootProtocol,omitempty"` + NetworkMode NetbootNetworkMode `json:"networkMode,omitempty"` + + // EndpointRef names the NetbootEndpoint that serves this Machine. + // +kubebuilder:validation:MinLength=1 + EndpointRef string `json:"endpointRef"` // DHCPLeases defines static IPv4 provisioning network settings. PXE boot // uses them as DHCP leases. HTTP boot uses the first entry to configure the @@ -420,18 +435,39 @@ type PXESpec struct { } const ( - // PXEBootProtocolPXE uses DHCP/TFTP PXE boot. - PXEBootProtocolPXE = "PXE" - // PXEBootProtocolHTTP uses Redfish UEFI HTTP boot. - PXEBootProtocolHTTP = "HTTP" // PXEArchitectureAMD64 is the x86_64 target architecture for PXE boot. PXEArchitectureAMD64 = "amd64" // PXEArchitectureARM64 is the aarch64 target architecture for PXE boot. PXEArchitectureARM64 = "arm64" // DefaultPXEArchitecture is used when host.netboot.architecture is omitted. DefaultPXEArchitecture = PXEArchitectureAMD64 - // DefaultPXEBootProtocol is used when host.netboot.bootProtocol is omitted. - DefaultPXEBootProtocol = PXEBootProtocolPXE +) + +// NetbootTransport is the protocol firmware uses to fetch boot artifacts. +// +kubebuilder:validation:Enum=TFTP;HTTP +type NetbootTransport string + +const ( + NetbootTransportTFTP NetbootTransport = "TFTP" + NetbootTransportHTTP NetbootTransport = "HTTP" +) + +// NetbootConfigurationSource supplies the firmware boot target. +// +kubebuilder:validation:Enum=DHCP;Redfish +type NetbootConfigurationSource string + +const ( + NetbootConfigurationSourceDHCP NetbootConfigurationSource = "DHCP" + NetbootConfigurationSourceRedfish NetbootConfigurationSource = "Redfish" +) + +// NetbootNetworkMode configures the firmware provisioning interface. +// +kubebuilder:validation:Enum=DHCP;Static +type NetbootNetworkMode string + +const ( + NetbootNetworkModeDHCP NetbootNetworkMode = "DHCP" + NetbootNetworkModeStatic NetbootNetworkMode = "Static" ) // TargetArchitecture returns the effective PXE target architecture. @@ -443,13 +479,13 @@ func (p *PXESpec) TargetArchitecture() string { return p.Architecture } -// TargetBootProtocol returns the effective network boot protocol. -func (p *PXESpec) TargetBootProtocol() string { - if p == nil || p.BootProtocol == "" { - return DefaultPXEBootProtocol +// TargetTransport returns the effective firmware boot transport. +func (p *PXESpec) TargetTransport() NetbootTransport { + if p == nil || p.Transport == "" { + return NetbootTransportTFTP } - return p.BootProtocol + return p.Transport } // CloudInitSpec defines cloud-init customization for PXE-booted machines. diff --git a/api/machina/v1alpha3/machine_validation_test.go b/api/machina/v1alpha3/machine_validation_test.go index 113e924f0..567617a67 100644 --- a/api/machina/v1alpha3/machine_validation_test.go +++ b/api/machina/v1alpha3/machine_validation_test.go @@ -51,6 +51,33 @@ func TestMachineProviderOwnershipSchema(t *testing.T) { }) } +func TestMachineNetbootSchema(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("../../../deploy/machina/crd/unbounded-cloud.io_machines.yaml") + if err != nil { + t.Fatalf("read Machine CRD: %v", err) + } + + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("parse Machine CRD: %v", err) + } + + netbootSchema := crd.Spec.Versions[0].Schema.OpenAPIV3Schema. + Properties["spec"].Properties["host"].Properties["netboot"] + + for _, field := range []string{"transport", "configurationSource", "networkMode", "endpointRef"} { + if _, ok := netbootSchema.Properties[field]; !ok { + t.Errorf("netboot schema is missing %q", field) + } + } + + if _, ok := netbootSchema.Properties["bootProtocol"]; ok { + t.Error("netboot schema still exposes bootProtocol") + } +} + func assertSchemaValidations(t *testing.T, schema apiextensionsv1.JSONSchemaProps, want map[string]string) { t.Helper() diff --git a/deploy/machina/crd/unbounded-cloud.io_machines.yaml b/deploy/machina/crd/unbounded-cloud.io_machines.yaml index cad45fedf..70d3fc816 100644 --- a/deploy/machina/crd/unbounded-cloud.io_machines.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_machines.yaml @@ -331,16 +331,6 @@ spec: - amd64 - arm64 type: string - bootProtocol: - default: PXE - description: |- - BootProtocol selects how metalman should trigger network boot for - repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - HTTP boot with a URL derived from the netboot image metadata. - enum: - - PXE - - HTTP - type: string cloudInit: description: |- CloudInit contains optional cloud-init customization for PXE-booted @@ -368,6 +358,14 @@ spec: - namespace type: object type: object + configurationSource: + default: DHCP + description: ConfigurationSource selects how firmware receives + its boot target. + enum: + - DHCP + - Redfish + type: string dhcpLeases: description: |- DHCPLeases defines static IPv4 provisioning network settings. PXE boot @@ -401,6 +399,11 @@ spec: - subnetMask type: object type: array + endpointRef: + description: EndpointRef names the NetbootEndpoint that serves + this Machine. + minLength: 1 + type: string image: description: |- Image is an OCI image reference containing the machine disk image. @@ -433,6 +436,14 @@ spec: - name - namespace type: object + networkMode: + default: DHCP + description: NetworkMode selects how firmware configures the + provisioning network. + enum: + - DHCP + - Static + type: string pullSecretRef: description: |- PullSecretRef references a Docker registry credential Secret used to pull @@ -494,9 +505,27 @@ spec: Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... When omitted, the installer chooses a target disk automatically. type: string + transport: + default: TFTP + description: Transport selects the firmware boot artifact + transport. + enum: + - TFTP + - HTTP + type: string required: + - endpointRef - image type: object + x-kubernetes-validations: + - message: TFTP transport requires DHCP configuration + rule: '!(self.transport == ''TFTP'' && self.configurationSource + == ''Redfish'')' + - message: static networking requires Redfish configuration + rule: self.networkMode != 'Static' || self.configurationSource + == 'Redfish' + - message: Redfish configuration requires redfish connection details + rule: self.configurationSource != 'Redfish' || has(self.redfish) type: object x-kubernetes-validations: - message: at most one of netboot, azure, or external may be set @@ -585,16 +614,6 @@ spec: - amd64 - arm64 type: string - bootProtocol: - default: PXE - description: |- - BootProtocol selects how metalman should trigger network boot for - repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - HTTP boot with a URL derived from the netboot image metadata. - enum: - - PXE - - HTTP - type: string cloudInit: description: |- CloudInit contains optional cloud-init customization for PXE-booted @@ -622,6 +641,14 @@ spec: - namespace type: object type: object + configurationSource: + default: DHCP + description: ConfigurationSource selects how firmware receives + its boot target. + enum: + - DHCP + - Redfish + type: string dhcpLeases: description: |- DHCPLeases defines static IPv4 provisioning network settings. PXE boot @@ -655,6 +682,11 @@ spec: - subnetMask type: object type: array + endpointRef: + description: EndpointRef names the NetbootEndpoint that serves + this Machine. + minLength: 1 + type: string image: description: |- Image is an OCI image reference containing the machine disk image. @@ -687,6 +719,14 @@ spec: - name - namespace type: object + networkMode: + default: DHCP + description: NetworkMode selects how firmware configures the provisioning + network. + enum: + - DHCP + - Static + type: string pullSecretRef: description: |- PullSecretRef references a Docker registry credential Secret used to pull @@ -748,9 +788,26 @@ spec: Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... When omitted, the installer chooses a target disk automatically. type: string + transport: + default: TFTP + description: Transport selects the firmware boot artifact transport. + enum: + - TFTP + - HTTP + type: string required: + - endpointRef - image type: object + x-kubernetes-validations: + - message: TFTP transport requires DHCP configuration + rule: '!(self.transport == ''TFTP'' && self.configurationSource + == ''Redfish'')' + - message: static networking requires Redfish configuration + rule: self.networkMode != 'Static' || self.configurationSource == + 'Redfish' + - message: Redfish configuration requires redfish connection details + rule: self.configurationSource != 'Redfish' || has(self.redfish) ssh: description: |- SSH contains the SSH connection and credential details for the diff --git a/internal/metalman/dhcp/dhcp.go b/internal/metalman/dhcp/dhcp.go index 43ad373ad..5f73d81e1 100644 --- a/internal/metalman/dhcp/dhcp.go +++ b/internal/metalman/dhcp/dhcp.go @@ -200,7 +200,7 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { meta, err := s.OCICache.MetadataForRefArchitecture(netbootImage, architecture) if err != nil { log.Warn("OCI image metadata not available", "image", netbootImage, "architecture", architecture, "err", err) - } else if node.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP { + } else if node.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP { if isHTTPClientRequest(m) { bootURL, err := netboot.JoinServeURLPath(s.ServeURL, netboot.HTTPBootPathFromMetadata(meta)) if err != nil { diff --git a/internal/metalman/dhcp/dhcp_test.go b/internal/metalman/dhcp/dhcp_test.go index 3a1bc164f..87d074eb9 100644 --- a/internal/metalman/dhcp/dhcp_test.go +++ b/internal/metalman/dhcp/dhcp_test.go @@ -266,7 +266,7 @@ func TestDHCPHandlerHTTPBootSuppressesPXEBootOptions(t *testing.T) { Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ NetbootImage: netbootImageRef, - BootProtocol: v1alpha3.PXEBootProtocolHTTP, + Transport: v1alpha3.NetbootTransportHTTP, DHCPLeases: []v1alpha3.DHCPLease{{ MAC: "aa:bb:cc:dd:ee:f3", IPv4: "10.0.1.13", @@ -341,7 +341,7 @@ func TestDHCPHandlerHTTPBootClientGetsHTTPBootURL(t *testing.T) { Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ NetbootImage: netbootImageRef, - BootProtocol: v1alpha3.PXEBootProtocolHTTP, + Transport: v1alpha3.NetbootTransportHTTP, DHCPLeases: []v1alpha3.DHCPLease{{ MAC: "aa:bb:cc:dd:ee:f4", IPv4: "10.0.1.14", diff --git a/internal/metalman/machineops/controller.go b/internal/metalman/machineops/controller.go index 636c3a28a..f05698028 100644 --- a/internal/metalman/machineops/controller.go +++ b/internal/metalman/machineops/controller.go @@ -590,7 +590,7 @@ func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOpe } func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, machine *v1alpha3.Machine) error { - if machine.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP { + if machine.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP { bootURL, staticConfig, err := r.httpBootConfig(machine) if err != nil { return err @@ -715,7 +715,7 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma target.LastAttemptAt = nil } - if machine.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP { + if machine.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP { if _, _, err := r.httpBootConfig(machine); err != nil { if errors.Is(err, netboot.ErrNotYetDownloaded) { target.Message = "waiting for OCI image to become available" diff --git a/internal/metalman/machineops/controller_test.go b/internal/metalman/machineops/controller_test.go index ad662d3d7..d1fa4c112 100644 --- a/internal/metalman/machineops/controller_test.go +++ b/internal/metalman/machineops/controller_test.go @@ -471,7 +471,7 @@ func TestReconcilerFallsBackToBIOSHTTPBootURIForHostReplace(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -512,7 +512,7 @@ func TestReconcilerUsesBIOSHTTPBootURIWhenStandardURIAbsent(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-bios", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -553,7 +553,7 @@ func TestReconcilerFallsBackToBIOSWhenStaticInterfaceIsReadOnly(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-read-only-nic", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -631,7 +631,7 @@ func TestReconcilerRetriesHTTPHostReplaceWithoutStaticLease(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP op := testOperation("op-replace-http-no-lease", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -664,7 +664,7 @@ func TestReconcilerWaitsForHTTPBootImage(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-wait-image", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name diff --git a/internal/metalman/netboot/http.go b/internal/metalman/netboot/http.go index 6c3e9ee2f..b737681a2 100644 --- a/internal/metalman/netboot/http.go +++ b/internal/metalman/netboot/http.go @@ -159,7 +159,7 @@ func (h *HTTPServer) handleFile(w http.ResponseWriter, r *http.Request) { return } - if node.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP && + if node.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP && h.isHTTPBootLoaderDownload(imageRef, node.Spec.Netboot().TargetArchitecture(), path) { installRequested, err := h.installRequested(r.Context(), node) if err != nil { @@ -187,7 +187,7 @@ func (h *HTTPServer) handleFile(w http.ResponseWriter, r *http.Request) { return } - if node.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP && isOptionalShimRevocationsFile(path) { + if node.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP && isOptionalShimRevocationsFile(path) { serveMissingShimRevocationsFile(w, log, node, path) return @@ -422,7 +422,7 @@ func (h *HTTPServer) recordBootImageWritten(ctx context.Context, log *slog.Logge } func (h *HTTPServer) recordHTTPBootLoaderDownloaded(ctx context.Context, log *slog.Logger, node *v1alpha3.Machine, imageRef, path string) { - if h.StatusRecorder == nil || node == nil || node.Spec.Netboot() == nil || node.Spec.Netboot().TargetBootProtocol() != v1alpha3.PXEBootProtocolHTTP { + if h.StatusRecorder == nil || node == nil || node.Spec.Netboot() == nil || node.Spec.Netboot().TargetTransport() != v1alpha3.NetbootTransportHTTP { return } diff --git a/internal/metalman/netboot/netboot_test.go b/internal/metalman/netboot/netboot_test.go index 62c1fa9cb..e0c53b72b 100644 --- a/internal/metalman/netboot/netboot_test.go +++ b/internal/metalman/netboot/netboot_test.go @@ -237,9 +237,9 @@ func TestHTTPServer_HTTPBootLoaderRequiresActiveInstallOperation(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node-http"}, Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ - Image: "ghcr.io/test/image:v1", - BootProtocol: v1alpha3.PXEBootProtocolHTTP, - DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:10", IPv4: "10.0.1.60", SubnetMask: "255.255.255.0"}}, + Image: "ghcr.io/test/image:v1", + Transport: v1alpha3.NetbootTransportHTTP, + DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:10", IPv4: "10.0.1.60", SubnetMask: "255.255.255.0"}}, }, }, } @@ -314,9 +314,9 @@ func TestHTTPServer_MissingOptionalShimRevocationsFilesHTTPBoot(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node-revocations"}, Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ - Image: "ghcr.io/test/image:v1", - BootProtocol: v1alpha3.PXEBootProtocolHTTP, - DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:11", IPv4: "10.0.1.61", SubnetMask: "255.255.255.0"}}, + Image: "ghcr.io/test/image:v1", + Transport: v1alpha3.NetbootTransportHTTP, + DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:11", IPv4: "10.0.1.61", SubnetMask: "255.255.255.0"}}, }, }, } From f1e2996e6bcf4292bc228b41d5c6b0d901947222 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:10:48 +0000 Subject: [PATCH 02/43] metalman: add durable netboot session APIs --- .../v1alpha3/machineoperation_types.go | 5 + .../v1alpha3/netboot_validation_test.go | 96 ++++ api/machina/v1alpha3/netbootendpoint_types.go | 170 +++++++ api/machina/v1alpha3/netbootsession_types.go | 218 +++++++++ api/machina/v1alpha3/zz_generated.deepcopy.go | 417 ++++++++++++++++++ .../unbounded-cloud.io_machineoperations.yaml | 17 + .../unbounded-cloud.io_netbootendpoints.yaml | 303 +++++++++++++ .../unbounded-cloud.io_netbootsessions.yaml | 414 +++++++++++++++++ 8 files changed, 1640 insertions(+) create mode 100644 api/machina/v1alpha3/netboot_validation_test.go create mode 100644 api/machina/v1alpha3/netbootendpoint_types.go create mode 100644 api/machina/v1alpha3/netbootsession_types.go create mode 100644 deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml create mode 100644 deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml diff --git a/api/machina/v1alpha3/machineoperation_types.go b/api/machina/v1alpha3/machineoperation_types.go index 0412ab7a9..f854a22b1 100644 --- a/api/machina/v1alpha3/machineoperation_types.go +++ b/api/machina/v1alpha3/machineoperation_types.go @@ -164,6 +164,11 @@ type ProviderOperationStatus struct { // one MachineOperation target. It is persisted before provider execution so // retries do not silently adopt later desired-state changes. type MachineOperationTargetInput struct { + // NetbootSessionRef identifies the immutable provisioning session assigned + // to this HostReplace target. + // +optional + NetbootSessionRef *NetbootSessionReference `json:"netbootSessionRef,omitempty"` + // ProviderRef identifies the exact provider-owned resource referenced by // host.external.machineRef when the operation target was initialized. // +optional diff --git a/api/machina/v1alpha3/netboot_validation_test.go b/api/machina/v1alpha3/netboot_validation_test.go new file mode 100644 index 000000000..b19d26805 --- /dev/null +++ b/api/machina/v1alpha3/netboot_validation_test.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha3 + +import ( + "os" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func TestNetbootEndpointSchema(t *testing.T) { + t.Parallel() + + crd := readCRD(t, "../../../deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml") + if crd.Spec.Scope != apiextensionsv1.ClusterScoped { + t.Errorf("scope = %q, want %q", crd.Spec.Scope, apiextensionsv1.ClusterScoped) + } + + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + for _, field := range []string{"siteRef", "type", "externalURL", "tls", "managedL2", "http"} { + if _, ok := spec.Properties[field]; !ok { + t.Errorf("endpoint spec is missing %q", field) + } + } + assertSchemaValidations(t, spec, map[string]string{ + "self.tls.trust != 'Public' || (self.externalURL.startsWith('https://') && self.tls.mode != 'Disabled')": "public endpoints require HTTPS", + "self.type == 'ManagedL2' ? has(self.managedL2) : !has(self.managedL2)": "managedL2 configuration must be set only for ManagedL2 endpoints", + "self.type == 'HTTP' ? has(self.http) : !has(self.http)": "http configuration must be set only for HTTP endpoints", + }) + assertSchemaValidations(t, spec.Properties["tls"], map[string]string{ + "self.mode == 'Secret' ? has(self.secretRef) : !has(self.secretRef)": "secretRef must be set only when TLS mode is Secret", + }) + + status := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] + for _, field := range []string{"observedGeneration", "claim", "conditions"} { + if _, ok := status.Properties[field]; !ok { + t.Errorf("endpoint status is missing %q", field) + } + } +} + +func TestNetbootSessionSchema(t *testing.T) { + t.Parallel() + + crd := readCRD(t, "../../../deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml") + if crd.Spec.Scope != apiextensionsv1.ClusterScoped { + t.Errorf("scope = %q, want %q", crd.Spec.Scope, apiextensionsv1.ClusterScoped) + } + + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + for _, field := range []string{"machine", "operation", "endpoint", "boot", "artifacts", "expiresAt"} { + if _, ok := spec.Properties[field]; !ok { + t.Errorf("session spec is missing %q", field) + } + } + assertSchemaValidations(t, spec, map[string]string{ + "self == oldSelf": "netboot session spec is immutable", + }) + + status := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] + for _, field := range []string{"phase", "conditions"} { + if _, ok := status.Properties[field]; !ok { + t.Errorf("session status is missing %q", field) + } + } +} + +func TestMachineOperationTargetInputHasNetbootSessionRef(t *testing.T) { + t.Parallel() + + crd := readCRD(t, "../../../deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml") + input := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"]. + Properties["targets"].Items.Schema.Properties["input"] + if _, ok := input.Properties["netbootSessionRef"]; !ok { + t.Error("operation target input is missing netbootSessionRef") + } +} + +func readCRD(t *testing.T, path string) apiextensionsv1.CustomResourceDefinition { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read CRD: %v", err) + } + + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("parse CRD: %v", err) + } + + return crd +} diff --git a/api/machina/v1alpha3/netbootendpoint_types.go b/api/machina/v1alpha3/netbootendpoint_types.go new file mode 100644 index 000000000..f0f766d73 --- /dev/null +++ b/api/machina/v1alpha3/netbootendpoint_types.go @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &NetbootEndpoint{}, &NetbootEndpointList{}) + metav1.AddToGroupVersion(s, GroupVersion) + + return nil + }) +} + +// NetbootEndpointType identifies where an endpoint edge runs. +// +kubebuilder:validation:Enum=ManagedL2;ExternalL2;HTTP +type NetbootEndpointType string + +const ( + NetbootEndpointTypeManagedL2 NetbootEndpointType = "ManagedL2" + NetbootEndpointTypeExternalL2 NetbootEndpointType = "ExternalL2" + NetbootEndpointTypeHTTP NetbootEndpointType = "HTTP" +) + +// NetbootEndpointTrust identifies the network trust boundary of an endpoint. +// +kubebuilder:validation:Enum=TrustedLAN;Public +type NetbootEndpointTrust string + +const ( + NetbootEndpointTrustTrustedLAN NetbootEndpointTrust = "TrustedLAN" + NetbootEndpointTrustPublic NetbootEndpointTrust = "Public" +) + +// NetbootEndpointTLSMode identifies where TLS is terminated. +// +kubebuilder:validation:Enum=Disabled;Secret;External +type NetbootEndpointTLSMode string + +const ( + NetbootEndpointTLSDisabled NetbootEndpointTLSMode = "Disabled" + NetbootEndpointTLSSecret NetbootEndpointTLSMode = "Secret" + NetbootEndpointTLSExternal NetbootEndpointTLSMode = "External" +) + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=nbe +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Site",type="string",JSONPath=".spec.siteRef" +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.type" +// +kubebuilder:printcolumn:name="URL",type="string",JSONPath=".spec.externalURL" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// NetbootEndpoint declares a stable client-facing netboot endpoint and its edge +// placement. The endpoint URL is snapshotted into each NetbootSession. +type NetbootEndpoint struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetbootEndpointSpec `json:"spec,omitempty"` + Status NetbootEndpointStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetbootEndpointList contains a list of NetbootEndpoint resources. +type NetbootEndpointList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetbootEndpoint `json:"items"` +} + +// +kubebuilder:validation:XValidation:rule="self.tls.trust != 'Public' || (self.externalURL.startsWith('https://') && self.tls.mode != 'Disabled')",message="public endpoints require HTTPS" +// +kubebuilder:validation:XValidation:rule="self.type == 'ManagedL2' ? has(self.managedL2) : !has(self.managedL2)",message="managedL2 configuration must be set only for ManagedL2 endpoints" +// +kubebuilder:validation:XValidation:rule="self.type == 'HTTP' ? has(self.http) : !has(self.http)",message="http configuration must be set only for HTTP endpoints" + +// NetbootEndpointSpec defines a stable edge endpoint. +type NetbootEndpointSpec struct { + // SiteRef names the Site whose Machines may use this endpoint. + // +kubebuilder:validation:MinLength=1 + SiteRef string `json:"siteRef"` + + // Type identifies how the endpoint edge is operated. + Type NetbootEndpointType `json:"type"` + + // ExternalURL is the stable base URL advertised to firmware and installers. + // +kubebuilder:validation:Pattern=`^https?://` + ExternalURL string `json:"externalURL"` + + // TLS defines the endpoint trust boundary and TLS termination mode. + TLS NetbootEndpointTLS `json:"tls"` + + // ManagedL2 configures an operator-managed edge on a provisioning LAN. + // +optional + ManagedL2 *NetbootManagedL2Spec `json:"managedL2,omitempty"` + + // HTTP configures an operator-managed HTTP edge Service. + // +optional + HTTP *NetbootHTTPEndpointSpec `json:"http,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="self.mode == 'Secret' ? has(self.secretRef) : !has(self.secretRef)",message="secretRef must be set only when TLS mode is Secret" + +// NetbootEndpointTLS defines transport security for an endpoint. +type NetbootEndpointTLS struct { + // Trust identifies whether the endpoint is confined to a trusted LAN or is + // reachable across an untrusted network. + Trust NetbootEndpointTrust `json:"trust"` + + // Mode identifies where TLS is terminated. + Mode NetbootEndpointTLSMode `json:"mode"` + + // SecretRef references the serving certificate when mode is Secret. + // +optional + SecretRef *NamespacedSecretReference `json:"secretRef,omitempty"` +} + +// NetbootManagedL2Spec places a managed edge on a provisioning network. +type NetbootManagedL2Spec struct { + // NodeSelector selects nodes attached to the provisioning network. + NodeSelector metav1.LabelSelector `json:"nodeSelector"` + + // Interface is the host interface used for DHCP and TFTP. + // +kubebuilder:validation:MinLength=1 + Interface string `json:"interface"` + + // Address is the stable address advertised by DHCP and used by the edge. + // +kubebuilder:validation:MinLength=1 + Address string `json:"address"` +} + +// NetbootHTTPEndpointSpec configures the Service for an HTTP-only edge. +type NetbootHTTPEndpointSpec struct { + // ServiceType controls how the edge Service is exposed. + // +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer + // +kubebuilder:default=ClusterIP + // +optional + ServiceType corev1.ServiceType `json:"serviceType,omitempty"` +} + +// NetbootEndpointStatus reports edge ownership and readiness. +type NetbootEndpointStatus struct { + // ObservedGeneration is the latest spec generation processed by the edge. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Claim identifies the edge currently responsible for this endpoint. + // +optional + Claim *NetbootEndpointClaim `json:"claim,omitempty"` + + // Conditions report endpoint readiness and degradation. + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// NetbootEndpointClaim records the current endpoint edge claimant. +type NetbootEndpointClaim struct { + // HolderIdentity uniquely identifies the claiming edge process. + HolderIdentity string `json:"holderIdentity"` + + // RenewedAt records the latest successful claim heartbeat. + RenewedAt metav1.Time `json:"renewedAt"` +} diff --git a/api/machina/v1alpha3/netbootsession_types.go b/api/machina/v1alpha3/netbootsession_types.go new file mode 100644 index 000000000..dce9b8c53 --- /dev/null +++ b/api/machina/v1alpha3/netbootsession_types.go @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha3 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &NetbootSession{}, &NetbootSessionList{}) + metav1.AddToGroupVersion(s, GroupVersion) + + return nil + }) +} + +// NetbootSessionPhase is the durable lifecycle phase of a provisioning session. +// +kubebuilder:validation:Enum=Pending;Preparing;Ready;Active;Complete;Failed;Expired +type NetbootSessionPhase string + +const ( + NetbootSessionPhasePending NetbootSessionPhase = "Pending" + NetbootSessionPhasePreparing NetbootSessionPhase = "Preparing" + NetbootSessionPhaseReady NetbootSessionPhase = "Ready" + NetbootSessionPhaseActive NetbootSessionPhase = "Active" + NetbootSessionPhaseComplete NetbootSessionPhase = "Complete" + NetbootSessionPhaseFailed NetbootSessionPhase = "Failed" + NetbootSessionPhaseExpired NetbootSessionPhase = "Expired" +) + +// Condition types for NetbootSession. +const ( + NetbootSessionConditionPrepared = "Prepared" + NetbootSessionConditionEndpointReady = "EndpointReady" + NetbootSessionConditionBootLoaderDownloaded = "BootLoaderDownloaded" + NetbootSessionConditionBootImageWritten = "BootImageWritten" + NetbootSessionConditionCloudInitDone = "CloudInitDone" + NetbootSessionConditionAttested = "Attested" +) + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=nbs +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".spec.machine.name" +// +kubebuilder:printcolumn:name="Endpoint",type="string",JSONPath=".spec.endpoint.name" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Expires",type="date",JSONPath=".spec.expiresAt" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// NetbootSession is an immutable provisioning contract for one +// MachineOperation target. Runtime progress is recorded only in status. +type NetbootSession struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetbootSessionSpec `json:"spec,omitempty"` + Status NetbootSessionStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetbootSessionList contains a list of NetbootSession resources. +type NetbootSessionList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetbootSession `json:"items"` +} + +// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="netboot session spec is immutable" + +// NetbootSessionSpec snapshots all input needed to provision one target. +type NetbootSessionSpec struct { + // Machine identifies the exact Machine revision being provisioned. + Machine NetbootSessionObjectSnapshot `json:"machine"` + + // Operation identifies the owning MachineOperation. + Operation NetbootSessionObjectSnapshot `json:"operation"` + + // Endpoint snapshots the selected endpoint and advertised URL. + Endpoint NetbootSessionEndpointSnapshot `json:"endpoint"` + + // Boot snapshots firmware and provisioning network configuration. + Boot NetbootSessionBoot `json:"boot"` + + // Artifacts identifies immutable OCI sources and files for this session. + Artifacts NetbootSessionArtifacts `json:"artifacts"` + + // ExpiresAt is the last time new requests may use this session. + ExpiresAt metav1.Time `json:"expiresAt"` +} + +// NetbootSessionObjectSnapshot identifies an exact Kubernetes object revision. +type NetbootSessionObjectSnapshot struct { + // Name is the cluster-scoped object name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // UID detects deletion and recreation of the object. + UID types.UID `json:"uid"` + + // Generation records the observed desired-state generation. + // +kubebuilder:validation:Minimum=1 + Generation int64 `json:"generation"` +} + +// NetbootSessionEndpointSnapshot identifies the endpoint selected for a session. +type NetbootSessionEndpointSnapshot struct { + // Name is the NetbootEndpoint name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // UID detects deletion and recreation of the endpoint. + UID types.UID `json:"uid"` + + // ExternalURL is the immutable base URL advertised for this session. + // +kubebuilder:validation:Pattern=`^https?://` + ExternalURL string `json:"externalURL"` +} + +// NetbootSessionBoot snapshots firmware and provisioning network settings. +type NetbootSessionBoot struct { + Transport NetbootTransport `json:"transport"` + ConfigurationSource NetbootConfigurationSource `json:"configurationSource"` + NetworkMode NetbootNetworkMode `json:"networkMode"` + + // Architecture selects the boot artifact platform. + // +kubebuilder:validation:Enum=amd64;arm64 + Architecture string `json:"architecture"` + + // DHCPLeases snapshots the target's provisioning network settings. + // +optional + DHCPLeases []DHCPLease `json:"dhcpLeases,omitempty"` + + // TargetDisk is the block device written by the installer. + // +optional + TargetDisk string `json:"targetDisk,omitempty"` +} + +// NetbootSessionArtifacts snapshots immutable OCI sources and artifact paths. +type NetbootSessionArtifacts struct { + MachineImage NetbootSessionImage `json:"machineImage"` + NetbootImage NetbootSessionImage `json:"netbootImage"` + + // Files lists the named files an edge may request for this session. + // +listType=map + // +listMapKey=name + Files []NetbootSessionArtifact `json:"files"` +} + +// NetbootSessionImage identifies an OCI image resolved to an immutable digest. +type NetbootSessionImage struct { + // Reference is the source repository reference used to resolve the image. + // +kubebuilder:validation:MinLength=1 + Reference string `json:"reference"` + + // Digest is the immutable OCI manifest digest. + // +kubebuilder:validation:Pattern=`^sha256:[a-f0-9]{64}$` + Digest string `json:"digest"` + + // PullSecretRef references registry credentials without copying them into + // the session. + // +optional + PullSecretRef *NamespacedSecretReference `json:"pullSecretRef,omitempty"` +} + +// NetbootSessionArtifact maps a public artifact name to an immutable image path. +type NetbootSessionArtifact struct { + // Name is the stable route name used by edges and rendered templates. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Source selects the OCI image containing this file. + // +kubebuilder:validation:Enum=MachineImage;NetbootImage + Source string `json:"source"` + + // Path is the absolute path within the unpacked OCI image. + // +kubebuilder:validation:Pattern=`^/` + Path string `json:"path"` + + // Size is the expected file size when known. + // +optional + // +kubebuilder:validation:Minimum=0 + Size *int64 `json:"size,omitempty"` +} + +// NetbootSessionStatus reports preparation and target-scoped milestones. +type NetbootSessionStatus struct { + // Phase is the current durable lifecycle phase. + // +optional + Phase NetbootSessionPhase `json:"phase,omitempty"` + + // CapabilityID identifies the signing key and capability generation without + // persisting a bearer capability. + // +optional + CapabilityID string `json:"capabilityID,omitempty"` + + // Conditions contain preparation, endpoint readiness, and target-scoped + // provisioning milestones. + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// NetbootSessionReference identifies the immutable session assigned to an +// operation target. +type NetbootSessionReference struct { + // Name is the NetbootSession name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // UID detects deletion and recreation of the session. + UID types.UID `json:"uid"` +} diff --git a/api/machina/v1alpha3/zz_generated.deepcopy.go b/api/machina/v1alpha3/zz_generated.deepcopy.go index 137751bce..d9738b74b 100644 --- a/api/machina/v1alpha3/zz_generated.deepcopy.go +++ b/api/machina/v1alpha3/zz_generated.deepcopy.go @@ -961,6 +961,11 @@ func (in *MachineOperationStatus) DeepCopy() *MachineOperationStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineOperationTargetInput) DeepCopyInto(out *MachineOperationTargetInput) { *out = *in + if in.NetbootSessionRef != nil { + in, out := &in.NetbootSessionRef, &out.NetbootSessionRef + *out = new(NetbootSessionReference) + **out = **in + } if in.ProviderRef != nil { in, out := &in.ProviderRef, &out.ProviderRef *out = new(ProviderMachineSnapshot) @@ -1155,6 +1160,418 @@ func (in *NamespacedSecretReference) DeepCopy() *NamespacedSecretReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpoint) DeepCopyInto(out *NetbootEndpoint) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpoint. +func (in *NetbootEndpoint) DeepCopy() *NetbootEndpoint { + if in == nil { + return nil + } + out := new(NetbootEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootEndpoint) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointClaim) DeepCopyInto(out *NetbootEndpointClaim) { + *out = *in + in.RenewedAt.DeepCopyInto(&out.RenewedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointClaim. +func (in *NetbootEndpointClaim) DeepCopy() *NetbootEndpointClaim { + if in == nil { + return nil + } + out := new(NetbootEndpointClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointList) DeepCopyInto(out *NetbootEndpointList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetbootEndpoint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointList. +func (in *NetbootEndpointList) DeepCopy() *NetbootEndpointList { + if in == nil { + return nil + } + out := new(NetbootEndpointList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootEndpointList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointSpec) DeepCopyInto(out *NetbootEndpointSpec) { + *out = *in + in.TLS.DeepCopyInto(&out.TLS) + if in.ManagedL2 != nil { + in, out := &in.ManagedL2, &out.ManagedL2 + *out = new(NetbootManagedL2Spec) + (*in).DeepCopyInto(*out) + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(NetbootHTTPEndpointSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointSpec. +func (in *NetbootEndpointSpec) DeepCopy() *NetbootEndpointSpec { + if in == nil { + return nil + } + out := new(NetbootEndpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointStatus) DeepCopyInto(out *NetbootEndpointStatus) { + *out = *in + if in.Claim != nil { + in, out := &in.Claim, &out.Claim + *out = new(NetbootEndpointClaim) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointStatus. +func (in *NetbootEndpointStatus) DeepCopy() *NetbootEndpointStatus { + if in == nil { + return nil + } + out := new(NetbootEndpointStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointTLS) DeepCopyInto(out *NetbootEndpointTLS) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(NamespacedSecretReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointTLS. +func (in *NetbootEndpointTLS) DeepCopy() *NetbootEndpointTLS { + if in == nil { + return nil + } + out := new(NetbootEndpointTLS) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootHTTPEndpointSpec) DeepCopyInto(out *NetbootHTTPEndpointSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootHTTPEndpointSpec. +func (in *NetbootHTTPEndpointSpec) DeepCopy() *NetbootHTTPEndpointSpec { + if in == nil { + return nil + } + out := new(NetbootHTTPEndpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootManagedL2Spec) DeepCopyInto(out *NetbootManagedL2Spec) { + *out = *in + in.NodeSelector.DeepCopyInto(&out.NodeSelector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootManagedL2Spec. +func (in *NetbootManagedL2Spec) DeepCopy() *NetbootManagedL2Spec { + if in == nil { + return nil + } + out := new(NetbootManagedL2Spec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSession) DeepCopyInto(out *NetbootSession) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSession. +func (in *NetbootSession) DeepCopy() *NetbootSession { + if in == nil { + return nil + } + out := new(NetbootSession) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootSession) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionArtifact) DeepCopyInto(out *NetbootSessionArtifact) { + *out = *in + if in.Size != nil { + in, out := &in.Size, &out.Size + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionArtifact. +func (in *NetbootSessionArtifact) DeepCopy() *NetbootSessionArtifact { + if in == nil { + return nil + } + out := new(NetbootSessionArtifact) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionArtifacts) DeepCopyInto(out *NetbootSessionArtifacts) { + *out = *in + in.MachineImage.DeepCopyInto(&out.MachineImage) + in.NetbootImage.DeepCopyInto(&out.NetbootImage) + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = make([]NetbootSessionArtifact, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionArtifacts. +func (in *NetbootSessionArtifacts) DeepCopy() *NetbootSessionArtifacts { + if in == nil { + return nil + } + out := new(NetbootSessionArtifacts) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionBoot) DeepCopyInto(out *NetbootSessionBoot) { + *out = *in + if in.DHCPLeases != nil { + in, out := &in.DHCPLeases, &out.DHCPLeases + *out = make([]DHCPLease, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionBoot. +func (in *NetbootSessionBoot) DeepCopy() *NetbootSessionBoot { + if in == nil { + return nil + } + out := new(NetbootSessionBoot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionEndpointSnapshot) DeepCopyInto(out *NetbootSessionEndpointSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionEndpointSnapshot. +func (in *NetbootSessionEndpointSnapshot) DeepCopy() *NetbootSessionEndpointSnapshot { + if in == nil { + return nil + } + out := new(NetbootSessionEndpointSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionImage) DeepCopyInto(out *NetbootSessionImage) { + *out = *in + if in.PullSecretRef != nil { + in, out := &in.PullSecretRef, &out.PullSecretRef + *out = new(NamespacedSecretReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionImage. +func (in *NetbootSessionImage) DeepCopy() *NetbootSessionImage { + if in == nil { + return nil + } + out := new(NetbootSessionImage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionList) DeepCopyInto(out *NetbootSessionList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetbootSession, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionList. +func (in *NetbootSessionList) DeepCopy() *NetbootSessionList { + if in == nil { + return nil + } + out := new(NetbootSessionList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootSessionList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionObjectSnapshot) DeepCopyInto(out *NetbootSessionObjectSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionObjectSnapshot. +func (in *NetbootSessionObjectSnapshot) DeepCopy() *NetbootSessionObjectSnapshot { + if in == nil { + return nil + } + out := new(NetbootSessionObjectSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionReference) DeepCopyInto(out *NetbootSessionReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionReference. +func (in *NetbootSessionReference) DeepCopy() *NetbootSessionReference { + if in == nil { + return nil + } + out := new(NetbootSessionReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionSpec) DeepCopyInto(out *NetbootSessionSpec) { + *out = *in + out.Machine = in.Machine + out.Operation = in.Operation + out.Endpoint = in.Endpoint + in.Boot.DeepCopyInto(&out.Boot) + in.Artifacts.DeepCopyInto(&out.Artifacts) + in.ExpiresAt.DeepCopyInto(&out.ExpiresAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionSpec. +func (in *NetbootSessionSpec) DeepCopy() *NetbootSessionSpec { + if in == nil { + return nil + } + out := new(NetbootSessionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionStatus) DeepCopyInto(out *NetbootSessionStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionStatus. +func (in *NetbootSessionStatus) DeepCopy() *NetbootSessionStatus { + if in == nil { + return nil + } + out := new(NetbootSessionStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PXESpec) DeepCopyInto(out *PXESpec) { *out = *in diff --git a/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml b/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml index a47f313a3..1d5c30b39 100644 --- a/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml @@ -331,6 +331,23 @@ spec: HostImage is the resolved provider-interpreted host image for HostReplace. An empty value instructs the provider to preserve the current image. type: string + netbootSessionRef: + description: |- + NetbootSessionRef identifies the immutable provisioning session assigned + to this HostReplace target. + properties: + name: + description: Name is the NetbootSession name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of + the session. + type: string + required: + - name + - uid + type: object providerRef: description: |- ProviderRef identifies the exact provider-owned resource referenced by diff --git a/deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml b/deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml new file mode 100644 index 000000000..70df872a7 --- /dev/null +++ b/deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml @@ -0,0 +1,303 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: netbootendpoints.unbounded-cloud.io +spec: + group: unbounded-cloud.io + names: + kind: NetbootEndpoint + listKind: NetbootEndpointList + plural: netbootendpoints + shortNames: + - nbe + singular: netbootendpoint + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.siteRef + name: Site + type: string + - jsonPath: .spec.type + name: Type + type: string + - jsonPath: .spec.externalURL + name: URL + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + NetbootEndpoint declares a stable client-facing netboot endpoint and its edge + placement. The endpoint URL is snapshotted into each NetbootSession. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NetbootEndpointSpec defines a stable edge endpoint. + properties: + externalURL: + description: ExternalURL is the stable base URL advertised to firmware + and installers. + pattern: ^https?:// + type: string + http: + description: HTTP configures an operator-managed HTTP edge Service. + properties: + serviceType: + default: ClusterIP + description: ServiceType controls how the edge Service is exposed. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + managedL2: + description: ManagedL2 configures an operator-managed edge on a provisioning + LAN. + properties: + address: + description: Address is the stable address advertised by DHCP + and used by the edge. + minLength: 1 + type: string + interface: + description: Interface is the host interface used for DHCP and + TFTP. + minLength: 1 + type: string + nodeSelector: + description: NodeSelector selects nodes attached to the provisioning + network. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - address + - interface + - nodeSelector + type: object + siteRef: + description: SiteRef names the Site whose Machines may use this endpoint. + minLength: 1 + type: string + tls: + description: TLS defines the endpoint trust boundary and TLS termination + mode. + properties: + mode: + description: Mode identifies where TLS is terminated. + enum: + - Disabled + - Secret + - External + type: string + secretRef: + description: SecretRef references the serving certificate when + mode is Secret. + properties: + name: + description: Name of the secret. + minLength: 1 + type: string + namespace: + description: Namespace of the secret. + minLength: 1 + type: string + required: + - name + - namespace + type: object + trust: + description: |- + Trust identifies whether the endpoint is confined to a trusted LAN or is + reachable across an untrusted network. + enum: + - TrustedLAN + - Public + type: string + required: + - mode + - trust + type: object + x-kubernetes-validations: + - message: secretRef must be set only when TLS mode is Secret + rule: 'self.mode == ''Secret'' ? has(self.secretRef) : !has(self.secretRef)' + type: + description: Type identifies how the endpoint edge is operated. + enum: + - ManagedL2 + - ExternalL2 + - HTTP + type: string + required: + - externalURL + - siteRef + - tls + - type + type: object + x-kubernetes-validations: + - message: public endpoints require HTTPS + rule: self.tls.trust != 'Public' || (self.externalURL.startsWith('https://') + && self.tls.mode != 'Disabled') + - message: managedL2 configuration must be set only for ManagedL2 endpoints + rule: 'self.type == ''ManagedL2'' ? has(self.managedL2) : !has(self.managedL2)' + - message: http configuration must be set only for HTTP endpoints + rule: 'self.type == ''HTTP'' ? has(self.http) : !has(self.http)' + status: + description: NetbootEndpointStatus reports edge ownership and readiness. + properties: + claim: + description: Claim identifies the edge currently responsible for this + endpoint. + properties: + holderIdentity: + description: HolderIdentity uniquely identifies the claiming edge + process. + type: string + renewedAt: + description: RenewedAt records the latest successful claim heartbeat. + format: date-time + type: string + required: + - holderIdentity + - renewedAt + type: object + conditions: + description: Conditions report endpoint readiness and degradation. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the latest spec generation processed + by the edge. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml new file mode 100644 index 000000000..df26ce67d --- /dev/null +++ b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml @@ -0,0 +1,414 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: netbootsessions.unbounded-cloud.io +spec: + group: unbounded-cloud.io + names: + kind: NetbootSession + listKind: NetbootSessionList + plural: netbootsessions + shortNames: + - nbs + singular: netbootsession + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.machine.name + name: Machine + type: string + - jsonPath: .spec.endpoint.name + name: Endpoint + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.expiresAt + name: Expires + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + NetbootSession is an immutable provisioning contract for one + MachineOperation target. Runtime progress is recorded only in status. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NetbootSessionSpec snapshots all input needed to provision + one target. + properties: + artifacts: + description: Artifacts identifies immutable OCI sources and files + for this session. + properties: + files: + description: Files lists the named files an edge may request for + this session. + items: + description: NetbootSessionArtifact maps a public artifact name + to an immutable image path. + properties: + name: + description: Name is the stable route name used by edges + and rendered templates. + minLength: 1 + type: string + path: + description: Path is the absolute path within the unpacked + OCI image. + pattern: ^/ + type: string + size: + description: Size is the expected file size when known. + format: int64 + minimum: 0 + type: integer + source: + description: Source selects the OCI image containing this + file. + enum: + - MachineImage + - NetbootImage + type: string + required: + - name + - path + - source + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + machineImage: + description: NetbootSessionImage identifies an OCI image resolved + to an immutable digest. + properties: + digest: + description: Digest is the immutable OCI manifest digest. + pattern: ^sha256:[a-f0-9]{64}$ + type: string + pullSecretRef: + description: |- + PullSecretRef references registry credentials without copying them into + the session. + properties: + name: + description: Name of the secret. + minLength: 1 + type: string + namespace: + description: Namespace of the secret. + minLength: 1 + type: string + required: + - name + - namespace + type: object + reference: + description: Reference is the source repository reference + used to resolve the image. + minLength: 1 + type: string + required: + - digest + - reference + type: object + netbootImage: + description: NetbootSessionImage identifies an OCI image resolved + to an immutable digest. + properties: + digest: + description: Digest is the immutable OCI manifest digest. + pattern: ^sha256:[a-f0-9]{64}$ + type: string + pullSecretRef: + description: |- + PullSecretRef references registry credentials without copying them into + the session. + properties: + name: + description: Name of the secret. + minLength: 1 + type: string + namespace: + description: Namespace of the secret. + minLength: 1 + type: string + required: + - name + - namespace + type: object + reference: + description: Reference is the source repository reference + used to resolve the image. + minLength: 1 + type: string + required: + - digest + - reference + type: object + required: + - files + - machineImage + - netbootImage + type: object + boot: + description: Boot snapshots firmware and provisioning network configuration. + properties: + architecture: + description: Architecture selects the boot artifact platform. + enum: + - amd64 + - arm64 + type: string + configurationSource: + description: NetbootConfigurationSource supplies the firmware + boot target. + enum: + - DHCP + - Redfish + type: string + dhcpLeases: + description: DHCPLeases snapshots the target's provisioning network + settings. + items: + description: DHCPLease defines static IPv4 provisioning network + settings. + properties: + dns: + description: DNS is a list of DNS server addresses. + items: + type: string + type: array + gateway: + description: Gateway is the default gateway. + type: string + ipv4: + description: IPv4 is the IP address to assign. + type: string + mac: + description: MAC is the MAC address of the network interface. + type: string + subnetMask: + description: SubnetMask is the subnet mask for the lease. + type: string + required: + - gateway + - ipv4 + - mac + - subnetMask + type: object + type: array + networkMode: + description: NetbootNetworkMode configures the firmware provisioning + interface. + enum: + - DHCP + - Static + type: string + targetDisk: + description: TargetDisk is the block device written by the installer. + type: string + transport: + description: NetbootTransport is the protocol firmware uses to + fetch boot artifacts. + enum: + - TFTP + - HTTP + type: string + required: + - architecture + - configurationSource + - networkMode + - transport + type: object + endpoint: + description: Endpoint snapshots the selected endpoint and advertised + URL. + properties: + externalURL: + description: ExternalURL is the immutable base URL advertised + for this session. + pattern: ^https?:// + type: string + name: + description: Name is the NetbootEndpoint name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of the endpoint. + type: string + required: + - externalURL + - name + - uid + type: object + expiresAt: + description: ExpiresAt is the last time new requests may use this + session. + format: date-time + type: string + machine: + description: Machine identifies the exact Machine revision being provisioned. + properties: + generation: + description: Generation records the observed desired-state generation. + format: int64 + minimum: 1 + type: integer + name: + description: Name is the cluster-scoped object name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of the object. + type: string + required: + - generation + - name + - uid + type: object + operation: + description: Operation identifies the owning MachineOperation. + properties: + generation: + description: Generation records the observed desired-state generation. + format: int64 + minimum: 1 + type: integer + name: + description: Name is the cluster-scoped object name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of the object. + type: string + required: + - generation + - name + - uid + type: object + required: + - artifacts + - boot + - endpoint + - expiresAt + - machine + - operation + type: object + x-kubernetes-validations: + - message: netboot session spec is immutable + rule: self == oldSelf + status: + description: NetbootSessionStatus reports preparation and target-scoped + milestones. + properties: + capabilityID: + description: |- + CapabilityID identifies the signing key and capability generation without + persisting a bearer capability. + type: string + conditions: + description: |- + Conditions contain preparation, endpoint readiness, and target-scoped + provisioning milestones. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + phase: + description: Phase is the current durable lifecycle phase. + enum: + - Pending + - Preparing + - Ready + - Active + - Complete + - Failed + - Expired + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} From 07dd22b510422a2364e5e497431ad3bd07f90999 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:20:17 +0000 Subject: [PATCH 03/43] metalman: persist immutable netboot sessions --- deploy/machina/06-metalman-rbac.yaml.tmpl | 10 + internal/metalman/commands/serve_pxe.go | 8 + internal/metalman/machineops/controller.go | 44 ++++ .../metalman/machineops/controller_test.go | 44 ++++ .../metalman/machineops/session_manager.go | 245 ++++++++++++++++++ .../machineops/session_manager_test.go | 150 +++++++++++ 6 files changed, 501 insertions(+) create mode 100644 internal/metalman/machineops/session_manager.go create mode 100644 internal/metalman/machineops/session_manager_test.go diff --git a/deploy/machina/06-metalman-rbac.yaml.tmpl b/deploy/machina/06-metalman-rbac.yaml.tmpl index 42dd9adf0..a590e7afd 100644 --- a/deploy/machina/06-metalman-rbac.yaml.tmpl +++ b/deploy/machina/06-metalman-rbac.yaml.tmpl @@ -123,6 +123,16 @@ rules: - apiGroups: ["unbounded-cloud.io"] resources: ["machineoperations/status"] verbs: ["get", "update", "patch"] + # Netboot session controller + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootsessions"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootsessions/status"] + verbs: ["get", "update", "patch"] # Secrets (Redfish passwords) - apiGroups: [""] resources: ["secrets"] diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index 75142a6e7..45756a57f 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -215,6 +215,13 @@ func ServePXECmd() *cobra.Command { redfishPool := redfish.NewPool() defer redfishPool.Close() + sessionManager := &metalmachineops.KubernetesSessionManager{ + Client: mgr.GetClient(), + Cache: ociCache, + DefaultNetbootRef: defaultNetbootImage, + DefaultNetbootPullSecret: defaultNetbootPullSecretRef, + } + statusQueue := &metalmachineops.StatusQueue{Client: mgr.GetClient()} resolver := netboot.FileResolver{ @@ -237,6 +244,7 @@ func ServePXECmd() *cobra.Command { APIReader: mgr.GetAPIReader(), Site: site, PowerClients: &metalmachineops.RedfishPowerClientFactory{Reader: mgr.GetClient(), Pool: redfishPool}, + Sessions: sessionManager, HTTPBootURL: resolver.HTTPBootURL, MaxConcurrentMachines: operationMaxConcurrentMachines, MaxAttempts: operationMaxAttempts, diff --git a/internal/metalman/machineops/controller.go b/internal/metalman/machineops/controller.go index f05698028..bd6b9d70e 100644 --- a/internal/metalman/machineops/controller.go +++ b/internal/metalman/machineops/controller.go @@ -64,6 +64,12 @@ type PowerClientFactory interface { ForMachine(ctx context.Context, machine *v1alpha3.Machine) (PowerClient, error) } +// SessionManager creates or retrieves the immutable session for one +// HostReplace target. +type SessionManager interface { + Ensure(ctx context.Context, operation *v1alpha3.MachineOperation, machine *v1alpha3.Machine) (*v1alpha3.NetbootSession, error) +} + // Reconciler reconciles metalman-owned host MachineOperations. type Reconciler struct { client.Client @@ -71,6 +77,7 @@ type Reconciler struct { Site string PowerClients PowerClientFactory + Sessions SessionManager HTTPBootURL func(*v1alpha3.Machine) (string, error) MaxConcurrentMachines int MaxAttempts int32 @@ -83,6 +90,9 @@ type Reconciler struct { // +kubebuilder:rbac:groups=unbounded-cloud.io,resources=machineoperations/status,verbs=get;update;patch // +kubebuilder:rbac:groups=unbounded-cloud.io,resources=machines,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=unbounded-cloud.io,resources=machines/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=unbounded-cloud.io,resources=netbootendpoints,verbs=get;list;watch +// +kubebuilder:rbac:groups=unbounded-cloud.io,resources=netbootsessions,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=unbounded-cloud.io,resources=netbootsessions/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -387,6 +397,40 @@ func (r *Reconciler) advanceTarget(ctx context.Context, op *v1alpha3.MachineOper case v1alpha3.OperationHostReboot: return r.advanceReboot(ctx, op, &machine, target, now) case v1alpha3.OperationHostReplace: + if r.Sessions != nil { + session, err := r.Sessions.Ensure(ctx, op, &machine) + if err != nil { + if errors.Is(err, netboot.ErrNotYetDownloaded) { + target.Message = "waiting for OCI images to resolve to immutable digests" + + return targetChange{target: target} + } + + return retryTarget(target, fmt.Errorf("ensure netboot session: %w", err), now, r.maxAttempts()) + } + + if target.Input == nil { + target.Input = &v1alpha3.MachineOperationTargetInput{} + } + + if target.Input.NetbootSessionRef == nil { + target.Input.NetbootSessionRef = &v1alpha3.NetbootSessionReference{Name: session.Name, UID: session.UID} + target.Message = fmt.Sprintf("persisted netboot session %s", session.Name) + + return targetChange{target: target} + } + + if target.Input.NetbootSessionRef.Name != session.Name || target.Input.NetbootSessionRef.UID != session.UID { + return failTarget(target, reasonExecutionFailed, "persisted netboot session identity changed", now) + } + + if session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive { + target.Message = fmt.Sprintf("waiting for netboot session %s to become ready", session.Name) + + return targetChange{target: target} + } + } + return r.advanceReplace(ctx, op, &machine, target, now) default: return failTarget(target, reasonUnsupportedTarget, fmt.Sprintf("%s is not handled by metalman", op.Spec.OperationKind), now) diff --git a/internal/metalman/machineops/controller_test.go b/internal/metalman/machineops/controller_test.go index d1fa4c112..6a7f239d6 100644 --- a/internal/metalman/machineops/controller_test.go +++ b/internal/metalman/machineops/controller_test.go @@ -380,6 +380,42 @@ func TestReconcilerRequestsHostReplaceOnceAndCompletesAfterRepave(t *testing.T) require.Equal(t, v1alpha3.OperationPhaseComplete, completed.Status.Phase) } +func TestReconcilerPersistsReadySessionBeforeHostReplaceSideEffects(t *testing.T) { + t.Parallel() + + s := testScheme(t) + machine := testBareMetalMachine("machine-session", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 3 + op := testOperation("op-session", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 2 + op.Spec.MachineRef = machine.Name + + session := &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: "netboot-session", UID: "session-uid"}, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } + sessions := &recordingSessionManager{session: session} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() + power := &recordingPowerClient{states: map[string]redfish.PowerState{machine.Name: redfish.PowerOff}} + reconciler := testReconciler(c, power, "rack-a") + reconciler.Sessions = sessions + + _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + require.Empty(t, power.calls) + + var preparing v1alpha3.MachineOperation + require.NoError(t, c.Get(t.Context(), client.ObjectKey{Name: op.Name}, &preparing)) + require.Equal(t, &v1alpha3.NetbootSessionReference{Name: session.Name, UID: session.UID}, preparing.Status.Targets[0].Input.NetbootSessionRef) + require.Equal(t, "persisted netboot session netboot-session", preparing.Status.Targets[0].Message) + + _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + require.Equal(t, []string{"machine-session:SetBootOverride:Pxe:Continuous", "machine-session:On"}, power.calls) +} + func TestReconcilerPowersOnHostReplaceTargetWhenOff(t *testing.T) { t.Parallel() @@ -1169,6 +1205,14 @@ type recordingPowerClient struct { calls []string } +type recordingSessionManager struct { + session *v1alpha3.NetbootSession +} + +func (m *recordingSessionManager) Ensure(_ context.Context, _ *v1alpha3.MachineOperation, _ *v1alpha3.Machine) (*v1alpha3.NetbootSession, error) { + return m.session.DeepCopy(), nil +} + func (r *recordingPowerClient) ForMachine(_ context.Context, machine *v1alpha3.Machine) (PowerClient, error) { r.mu.Lock() defer r.mu.Unlock() diff --git a/internal/metalman/machineops/session_manager.go b/internal/metalman/machineops/session_manager.go new file mode 100644 index 000000000..7db37399a --- /dev/null +++ b/internal/metalman/machineops/session_manager.go @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/metalman/netboot" +) + +const ( + sessionOperationUIDLabel = "metalman.unbounded-cloud.io/operation-uid" + sessionMachineUIDLabel = "metalman.unbounded-cloud.io/machine-uid" + defaultSessionTTL = 24 * time.Hour +) + +// KubernetesSessionManager persists immutable sessions in the Kubernetes API. +type KubernetesSessionManager struct { + Client client.Client + Cache *netboot.OCICache + DefaultNetbootRef string + DefaultNetbootPullSecret *v1alpha3.NamespacedSecretReference + Now func() metav1.Time +} + +// Ensure returns the durable session for one operation target, creating it +// only after both OCI references have resolved to immutable digests. +func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alpha3.MachineOperation, machine *v1alpha3.Machine) (*v1alpha3.NetbootSession, error) { + if operation.UID == "" || machine.UID == "" { + return nil, errors.New("operation and Machine UIDs are required for netboot session identity") + } + + name := sessionName(operation.UID, machine.UID) + + var existing v1alpha3.NetbootSession + if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, &existing); err == nil { + if existing.Spec.Operation.UID != operation.UID || existing.Spec.Machine.UID != machine.UID { + return nil, fmt.Errorf("netboot session %s belongs to different objects", name) + } + + if existing.Status.Phase == v1alpha3.NetbootSessionPhasePreparing { + return m.refreshEndpointReadiness(ctx, &existing) + } + + return &existing, nil + } else if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("get netboot session %s: %w", name, err) + } + + netbootSpec := machine.Spec.Netboot() + if netbootSpec == nil { + return nil, errors.New("Machine has no netboot configuration") + } + + var endpoint v1alpha3.NetbootEndpoint + if err := m.Client.Get(ctx, client.ObjectKey{Name: netbootSpec.EndpointRef}, &endpoint); err != nil { + return nil, fmt.Errorf("get NetbootEndpoint %s: %w", netbootSpec.EndpointRef, err) + } + + architecture := netbootSpec.TargetArchitecture() + machineDigest := m.Cache.DigestForArchitecture(netbootSpec.Image, architecture) + if machineDigest == "" { + return nil, fmt.Errorf("%w: machine image %q for architecture %q", netboot.ErrNotYetDownloaded, netbootSpec.Image, architecture) + } + + netbootRef := netbootSpec.NetbootImage + netbootPullSecret := netbootSpec.NetbootPullSecretRef + if netbootRef == "" { + netbootRef = m.DefaultNetbootRef + netbootPullSecret = m.DefaultNetbootPullSecret + } + + netbootDigest := m.Cache.DigestForArchitecture(netbootRef, architecture) + if netbootDigest == "" { + return nil, fmt.Errorf("%w: netboot image %q for architecture %q", netboot.ErrNotYetDownloaded, netbootRef, architecture) + } + + now := m.now() + session := &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + sessionOperationUIDLabel: string(operation.UID), + sessionMachineUIDLabel: string(machine.UID), + }, + }, + Spec: v1alpha3.NetbootSessionSpec{ + Machine: objectSnapshot(machine.Name, machine.UID, machine.Generation), + Operation: objectSnapshot(operation.Name, operation.UID, operation.Generation), + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{ + Name: endpoint.Name, + UID: endpoint.UID, + ExternalURL: endpoint.Spec.ExternalURL, + }, + Boot: v1alpha3.NetbootSessionBoot{ + Transport: netbootSpec.TargetTransport(), + ConfigurationSource: targetConfigurationSource(netbootSpec), + NetworkMode: targetNetworkMode(netbootSpec), + Architecture: architecture, + DHCPLeases: append([]v1alpha3.DHCPLease(nil), netbootSpec.DHCPLeases...), + TargetDisk: netbootSpec.TargetDisk, + }, + Artifacts: v1alpha3.NetbootSessionArtifacts{ + MachineImage: v1alpha3.NetbootSessionImage{ + Reference: netbootSpec.Image, + Digest: machineDigest, + PullSecretRef: netbootSpec.PullSecretRef.DeepCopy(), + }, + NetbootImage: v1alpha3.NetbootSessionImage{ + Reference: netbootRef, + Digest: netbootDigest, + PullSecretRef: netbootPullSecret.DeepCopy(), + }, + Files: sessionArtifacts(), + }, + ExpiresAt: metav1.NewTime(now.Add(defaultSessionTTL)), + }, + } + + if err := m.Client.Create(ctx, session); err != nil { + if apierrors.IsAlreadyExists(err) { + if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, session); err != nil { + return nil, fmt.Errorf("get concurrently created netboot session %s: %w", name, err) + } + + return session, nil + } + + return nil, fmt.Errorf("create netboot session %s: %w", name, err) + } + + return m.refreshEndpointReadinessWithEndpoint(ctx, session, &endpoint) +} + +func (m *KubernetesSessionManager) refreshEndpointReadiness(ctx context.Context, session *v1alpha3.NetbootSession) (*v1alpha3.NetbootSession, error) { + var endpoint v1alpha3.NetbootEndpoint + if err := m.Client.Get(ctx, client.ObjectKey{Name: session.Spec.Endpoint.Name}, &endpoint); err != nil { + return nil, fmt.Errorf("get NetbootEndpoint %s: %w", session.Spec.Endpoint.Name, err) + } + + if endpoint.UID != session.Spec.Endpoint.UID { + return nil, fmt.Errorf("NetbootEndpoint %s identity changed", endpoint.Name) + } + + return m.refreshEndpointReadinessWithEndpoint(ctx, session, &endpoint) +} + +func (m *KubernetesSessionManager) refreshEndpointReadinessWithEndpoint(ctx context.Context, session *v1alpha3.NetbootSession, endpoint *v1alpha3.NetbootEndpoint) (*v1alpha3.NetbootSession, error) { + now := m.now() + + ready := endpoint.Status.ObservedGeneration == endpoint.Generation && apimeta.IsStatusConditionTrue(endpoint.Status.Conditions, "Ready") + session.Status.Phase = v1alpha3.NetbootSessionPhasePreparing + if ready { + session.Status.Phase = v1alpha3.NetbootSessionPhaseReady + } + + apimeta.SetStatusCondition(&session.Status.Conditions, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionPrepared, + Status: metav1.ConditionTrue, + Reason: "ArtifactsResolved", + Message: "OCI references resolved to immutable digests", + ObservedGeneration: session.Generation, + LastTransitionTime: now, + }) + apimeta.SetStatusCondition(&session.Status.Conditions, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionEndpointReady, + Status: conditionStatus(ready), + Reason: endpointReadyReason(ready), + Message: fmt.Sprintf("NetbootEndpoint %s readiness", endpoint.Name), + ObservedGeneration: session.Generation, + LastTransitionTime: now, + }) + + if err := m.Client.Status().Update(ctx, session); err != nil { + return nil, fmt.Errorf("update netboot session %s status: %w", session.Name, err) + } + + return session, nil +} + +func (m *KubernetesSessionManager) now() metav1.Time { + if m.Now != nil { + return m.Now() + } + + return metav1.Now() +} + +func sessionName(operationUID, machineUID types.UID) string { + return fmt.Sprintf("netboot-%s-%s", operationUID, machineUID) +} + +func objectSnapshot(name string, uid types.UID, generation int64) v1alpha3.NetbootSessionObjectSnapshot { + return v1alpha3.NetbootSessionObjectSnapshot{Name: name, UID: uid, Generation: generation} +} + +func targetConfigurationSource(spec *v1alpha3.PXESpec) v1alpha3.NetbootConfigurationSource { + if spec.ConfigurationSource == "" { + return v1alpha3.NetbootConfigurationSourceDHCP + } + + return spec.ConfigurationSource +} + +func targetNetworkMode(spec *v1alpha3.PXESpec) v1alpha3.NetbootNetworkMode { + if spec.NetworkMode == "" { + return v1alpha3.NetbootNetworkModeDHCP + } + + return spec.NetworkMode +} + +func sessionArtifacts() []v1alpha3.NetbootSessionArtifact { + return []v1alpha3.NetbootSessionArtifact{ + {Name: "disk.img.gz", Source: "MachineImage", Path: "/disk/disk.img.gz"}, + {Name: "metadata.yaml", Source: "NetbootImage", Path: "/disk/metadata.yaml"}, + } +} + +func conditionStatus(ready bool) metav1.ConditionStatus { + if ready { + return metav1.ConditionTrue + } + + return metav1.ConditionFalse +} + +func endpointReadyReason(ready bool) string { + if ready { + return "Ready" + } + + return "NotReady" +} diff --git a/internal/metalman/machineops/session_manager_test.go b/internal/metalman/machineops/session_manager_test.go new file mode 100644 index 000000000..96d007575 --- /dev/null +++ b/internal/metalman/machineops/session_manager_test.go @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/metalman/netboot" +) + +func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { + t.Parallel() + + machine := testBareMetalMachine("machine-session", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 4 + machine.Spec.PXE.EndpointRef = "rack-a-edge" + machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" + op := testOperation("replace-session", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 2 + endpoint := readyTestEndpoint() + + cache := netboot.NewOCICache(t.TempDir()) + cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('a', 64)) + cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('b', 64)) + + s := testScheme(t) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(endpoint).WithStatusSubresource(&v1alpha3.NetbootSession{}).Build() + manager := &KubernetesSessionManager{ + Client: c, + Cache: cache, + Now: func() metav1.Time { return fixedNow() }, + } + + session, err := manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, v1alpha3.NetbootSessionPhaseReady, session.Status.Phase) + require.Equal(t, "sha256:"+stringOf('a', 64), session.Spec.Artifacts.MachineImage.Digest) + require.Equal(t, "sha256:"+stringOf('b', 64), session.Spec.Artifacts.NetbootImage.Digest) + require.Equal(t, machine.UID, session.Spec.Machine.UID) + require.Equal(t, op.UID, session.Spec.Operation.UID) + require.Equal(t, endpoint.Spec.ExternalURL, session.Spec.Endpoint.ExternalURL) + require.True(t, session.Spec.ExpiresAt.Time.Equal(fixedNow().Add(24*time.Hour))) + + machine.Spec.PXE.Image = "ghcr.io/test/changed:v2" + reused, err := manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, session.Name, reused.Name) + require.Equal(t, "ghcr.io/test/host:v1", reused.Spec.Artifacts.MachineImage.Reference) + + var sessions v1alpha3.NetbootSessionList + require.NoError(t, c.List(t.Context(), &sessions, client.MatchingLabels{sessionOperationUIDLabel: string(op.UID)})) + require.Len(t, sessions.Items, 1) +} + +func TestSessionManagerWaitsForImmutableDigests(t *testing.T) { + t.Parallel() + + machine := testBareMetalMachine("machine-pending", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 1 + machine.Spec.PXE.EndpointRef = "rack-a-edge" + machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" + op := testOperation("replace-pending", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 1 + + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(readyTestEndpoint()).WithStatusSubresource(&v1alpha3.NetbootSession{}).Build() + manager := &KubernetesSessionManager{Client: c, Cache: netboot.NewOCICache(t.TempDir())} + + _, err := manager.Ensure(t.Context(), op, machine) + require.ErrorIs(t, err, netboot.ErrNotYetDownloaded) + + var sessions v1alpha3.NetbootSessionList + require.NoError(t, c.List(t.Context(), &sessions)) + require.Empty(t, sessions.Items) +} + +func TestSessionManagerPromotesExistingSessionWhenEndpointBecomesReady(t *testing.T) { + t.Parallel() + + machine := testBareMetalMachine("machine-endpoint", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 1 + machine.Spec.PXE.EndpointRef = "rack-a-edge" + machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" + op := testOperation("replace-endpoint", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 1 + endpoint := readyTestEndpoint() + endpoint.Status.Conditions[0].Status = metav1.ConditionFalse + + cache := netboot.NewOCICache(t.TempDir()) + cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('a', 64)) + cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('b', 64)) + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(endpoint).WithStatusSubresource(endpoint, &v1alpha3.NetbootSession{}).Build() + manager := &KubernetesSessionManager{Client: c, Cache: cache} + + session, err := manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, v1alpha3.NetbootSessionPhasePreparing, session.Status.Phase) + + endpoint.Status.Conditions[0].Status = metav1.ConditionTrue + require.NoError(t, c.Status().Update(t.Context(), endpoint)) + + session, err = manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, v1alpha3.NetbootSessionPhaseReady, session.Status.Phase) +} + +func readyTestEndpoint() *v1alpha3.NetbootEndpoint { + return &v1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a-edge", UID: "endpoint-uid", Generation: 3}, + Spec: v1alpha3.NetbootEndpointSpec{ + SiteRef: "rack-a", + Type: v1alpha3.NetbootEndpointTypeExternalL2, + ExternalURL: "http://192.0.2.10:8880", + TLS: v1alpha3.NetbootEndpointTLS{ + Trust: v1alpha3.NetbootEndpointTrustTrustedLAN, + Mode: v1alpha3.NetbootEndpointTLSDisabled, + }, + }, + Status: v1alpha3.NetbootEndpointStatus{ + ObservedGeneration: 3, + Conditions: []metav1.Condition{{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "Available", + }}, + }, + } +} + +func stringOf(value byte, count int) string { + result := make([]byte, count) + for i := range result { + result[i] = value + } + + return string(result) +} From 86b5b341c0b49dc44d6f2a3fa1475eb1594455f0 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:24:58 +0000 Subject: [PATCH 04/43] metalman: split runtime role commands --- cmd/metalman/main.go | 4 +- internal/metalman/commands/controller.go | 11 ++ internal/metalman/commands/edge.go | 11 ++ internal/metalman/commands/roles.go | 67 +++++++++ internal/metalman/commands/roles_test.go | 84 +++++++++++ internal/metalman/commands/serve_pxe.go | 179 ++++++++++++++--------- internal/metalman/commands/server.go | 11 ++ 7 files changed, 293 insertions(+), 74 deletions(-) create mode 100644 internal/metalman/commands/controller.go create mode 100644 internal/metalman/commands/edge.go create mode 100644 internal/metalman/commands/roles.go create mode 100644 internal/metalman/commands/roles_test.go create mode 100644 internal/metalman/commands/server.go diff --git a/cmd/metalman/main.go b/cmd/metalman/main.go index 3a75e90c0..d7b4cc3d3 100644 --- a/cmd/metalman/main.go +++ b/cmd/metalman/main.go @@ -22,7 +22,9 @@ func main() { Use: "metalman", Short: "Bare metal provisioning for Kubernetes", } - root.AddCommand(commands.ServePXECmd()) + root.AddCommand(commands.ControllerCmd()) + root.AddCommand(commands.ServerCmd()) + root.AddCommand(commands.EdgeCmd()) root.AddCommand(version.Command()) root.CompletionOptions.DisableDefaultCmd = true diff --git a/internal/metalman/commands/controller.go b/internal/metalman/commands/controller.go new file mode 100644 index 000000000..1f3085f02 --- /dev/null +++ b/internal/metalman/commands/controller.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import "github.com/spf13/cobra" + +// ControllerCmd runs Metalman's leader-elected Kubernetes control loops. +func ControllerCmd() *cobra.Command { + return newMetalmanRoleCmd(metalmanRoleController, "Run the Metalman control loops") +} diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go new file mode 100644 index 000000000..296e55cc0 --- /dev/null +++ b/internal/metalman/commands/edge.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import "github.com/spf13/cobra" + +// EdgeCmd runs Metalman's provisioning-network protocol edge. +func EdgeCmd() *cobra.Command { + return newMetalmanRoleCmd(metalmanRoleEdge, "Run the Metalman provisioning protocol edge") +} diff --git a/internal/metalman/commands/roles.go b/internal/metalman/commands/roles.go new file mode 100644 index 000000000..eb71b08ad --- /dev/null +++ b/internal/metalman/commands/roles.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +type metalmanRole string + +const ( + metalmanRoleController metalmanRole = "controller" + metalmanRoleServer metalmanRole = "server" + metalmanRoleEdge metalmanRole = "edge" + metalmanRoleLegacy metalmanRole = "serve-pxe" +) + +type roleComponents struct { + leaderElection bool + ociReconciler bool + redfish bool + machineOps bool + dhcp bool + tftp bool + http bool + attestation bool + statusUpdates bool +} + +func componentsForRole(role metalmanRole) roleComponents { + switch role { + case metalmanRoleController: + return roleComponents{ + leaderElection: true, + ociReconciler: true, + redfish: true, + machineOps: true, + } + case metalmanRoleServer: + return roleComponents{ + ociReconciler: true, + http: true, + attestation: true, + statusUpdates: true, + } + case metalmanRoleEdge: + return roleComponents{ + ociReconciler: true, + dhcp: true, + tftp: true, + http: true, + attestation: true, + statusUpdates: true, + } + case metalmanRoleLegacy: + return roleComponents{ + leaderElection: true, + ociReconciler: true, + redfish: true, + machineOps: true, + dhcp: true, + tftp: true, + http: true, + attestation: true, + statusUpdates: true, + } + default: + return roleComponents{} + } +} diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go new file mode 100644 index 000000000..2a382f627 --- /dev/null +++ b/internal/metalman/commands/roles_test.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import ( + "testing" + + "github.com/spf13/cobra" +) + +func TestMetalmanRoleComponentsAreIsolated(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + role metalmanRole + want roleComponents + }{ + { + name: "controller", + role: metalmanRoleController, + want: roleComponents{ + leaderElection: true, + ociReconciler: true, + redfish: true, + machineOps: true, + }, + }, + { + name: "server", + role: metalmanRoleServer, + want: roleComponents{ + ociReconciler: true, + http: true, + attestation: true, + statusUpdates: true, + }, + }, + { + name: "edge", + role: metalmanRoleEdge, + want: roleComponents{ + ociReconciler: true, + dhcp: true, + tftp: true, + http: true, + attestation: true, + statusUpdates: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := componentsForRole(tt.role); got != tt.want { + t.Fatalf("componentsForRole(%q) = %#v, want %#v", tt.role, got, tt.want) + } + }) + } +} + +func TestMetalmanRoleCommands(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + cmd func() *cobra.Command + }{ + {name: "controller", cmd: ControllerCmd}, + {name: "server", cmd: ServerCmd}, + {name: "edge", cmd: EdgeCmd}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := tt.cmd().Name(); got != tt.name { + t.Fatalf("command name = %q, want %q", got, tt.name) + } + }) + } +} diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index 45756a57f..3effd441b 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -38,8 +38,13 @@ import ( // omits spec.host.netboot.netbootImage. It is set at build time via -ldflags. var DefaultNetbootImage = "netboot:latest" -// ServePXECmd returns a cobra.Command that runs PXE servers and the BMC control loop. +// ServePXECmd returns the legacy monolithic command. Production wiring uses +// ControllerCmd, ServerCmd, and EdgeCmd instead. func ServePXECmd() *cobra.Command { + return newMetalmanRoleCmd(metalmanRoleLegacy, "Run PXE servers and BMC control loop") +} + +func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { var ( site string cacheDir string @@ -59,10 +64,11 @@ func ServePXECmd() *cobra.Command { defaultNetbootImage string defaultNetbootPullSecret string ) + components := componentsForRole(role) cmd := &cobra.Command{ - Use: "serve-pxe", - Short: "Run PXE servers and BMC control loop", + Use: string(role), + Short: short, RunE: func(cmd *cobra.Command, _ []string) error { ctx := ctrl.SetupSignalHandler() cfg := ctrl.GetConfigOrDie() @@ -85,7 +91,7 @@ func ServePXECmd() *cobra.Command { mgr, err := ctrl.NewManager(cfg, manager.Options{ Scheme: scheme, - LeaderElection: true, + LeaderElection: components.leaderElection, LeaderElectionID: leID, LeaderElectionNamespace: leaderElectionNamespace, LeaseDuration: &leaseDuration, @@ -203,13 +209,15 @@ func ServePXECmd() *cobra.Command { ociCache := netboot.NewOCICache(cacheDir) - if err := (&netboot.OCIReconciler{ - Client: mgr.GetClient(), - Cache: ociCache, - DefaultNetbootRef: defaultNetbootImage, - DefaultNetbootPullSecretRef: defaultNetbootPullSecretRef, - }).SetupWithManager(mgr); err != nil { - return fmt.Errorf("setting up OCI reconciler: %w", err) + if components.ociReconciler { + if err := (&netboot.OCIReconciler{ + Client: mgr.GetClient(), + Cache: ociCache, + DefaultNetbootRef: defaultNetbootImage, + DefaultNetbootPullSecretRef: defaultNetbootPullSecretRef, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setting up OCI reconciler: %w", err) + } } redfishPool := redfish.NewPool() @@ -222,7 +230,10 @@ func ServePXECmd() *cobra.Command { DefaultNetbootPullSecret: defaultNetbootPullSecretRef, } - statusQueue := &metalmachineops.StatusQueue{Client: mgr.GetClient()} + var statusQueue *metalmachineops.StatusQueue + if components.statusUpdates { + statusQueue = &metalmachineops.StatusQueue{Client: mgr.GetClient()} + } resolver := netboot.FileResolver{ Cache: ociCache, @@ -235,22 +246,26 @@ func ServePXECmd() *cobra.Command { ProviderLabels: providerLabels, } - if err := (&redfish.Reconciler{Client: mgr.GetClient(), Pool: redfishPool, FileResolver: &resolver}).SetupWithManager(mgr); err != nil { - return fmt.Errorf("setting up Redfish reconciler: %w", err) + if components.redfish { + if err := (&redfish.Reconciler{Client: mgr.GetClient(), Pool: redfishPool, FileResolver: &resolver}).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setting up Redfish reconciler: %w", err) + } } - if err := (&metalmachineops.Reconciler{ - Client: mgr.GetClient(), - APIReader: mgr.GetAPIReader(), - Site: site, - PowerClients: &metalmachineops.RedfishPowerClientFactory{Reader: mgr.GetClient(), Pool: redfishPool}, - Sessions: sessionManager, - HTTPBootURL: resolver.HTTPBootURL, - MaxConcurrentMachines: operationMaxConcurrentMachines, - MaxAttempts: operationMaxAttempts, - PollInterval: operationPollInterval, - }).SetupWithManager(mgr); err != nil { - return fmt.Errorf("setting up MachineOperation reconciler: %w", err) + if components.machineOps { + if err := (&metalmachineops.Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Site: site, + PowerClients: &metalmachineops.RedfishPowerClientFactory{Reader: mgr.GetClient(), Pool: redfishPool}, + Sessions: sessionManager, + HTTPBootURL: resolver.HTTPBootURL, + MaxConcurrentMachines: operationMaxConcurrentMachines, + MaxAttempts: operationMaxAttempts, + PollInterval: operationPollInterval, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setting up MachineOperation reconciler: %w", err) + } } if dhcpInterface != "" && dhcpAutoInterface { @@ -277,52 +292,61 @@ func ServePXECmd() *cobra.Command { dhcpServerIP = ifIP } - dhcpServer := &dhcp.Server{ - Interface: dhcpInterface, - Port: dhcpPort, - Reader: mgr.GetClient(), - ServerIP: dhcpServerIP, - OCICache: ociCache, - ServeURL: serveURL, - DefaultNetbootRef: defaultNetbootImage, - } - if err := mgr.Add(dhcpServer); err != nil { - return fmt.Errorf("adding DHCP server: %w", err) - } - - if err := mgr.Add(statusQueue); err != nil { - return fmt.Errorf("adding status queue: %w", err) + if components.dhcp { + dhcpServer := &dhcp.Server{ + Interface: dhcpInterface, + Port: dhcpPort, + Reader: mgr.GetClient(), + ServerIP: dhcpServerIP, + OCICache: ociCache, + ServeURL: serveURL, + DefaultNetbootRef: defaultNetbootImage, + } + if err := mgr.Add(dhcpServer); err != nil { + return fmt.Errorf("adding DHCP server: %w", err) + } } - tftpServer := &netboot.TFTPServer{ - BindAddr: bindAddress, - FileResolver: resolver, - StatusRecorder: statusQueue, - } - if err := mgr.Add(tftpServer); err != nil { - return fmt.Errorf("adding TFTP server: %w", err) + if statusQueue != nil { + if err := mgr.Add(statusQueue); err != nil { + return fmt.Errorf("adding status queue: %w", err) + } } - attestHandler := &attestation.Handler{ - Clientset: clientset, - ClusterCA: clusterCA, - LookupNodeByIP: resolver.LookupNodeByIP, - StatusUpdater: &StatusUpdater{Client: mgr.GetClient()}, + if components.tftp { + tftpServer := &netboot.TFTPServer{ + BindAddr: bindAddress, + FileResolver: resolver, + StatusRecorder: statusQueue, + } + if err := mgr.Add(tftpServer); err != nil { + return fmt.Errorf("adding TFTP server: %w", err) + } } - httpMux := http.NewServeMux() - httpMux.HandleFunc("POST /attest", attestHandler.Attest) + if components.http { + httpMux := http.NewServeMux() + if components.attestation { + attestHandler := &attestation.Handler{ + Clientset: clientset, + ClusterCA: clusterCA, + LookupNodeByIP: resolver.LookupNodeByIP, + StatusUpdater: &StatusUpdater{Client: mgr.GetClient()}, + } + httpMux.HandleFunc("POST /attest", attestHandler.Attest) + } - httpServer := &netboot.HTTPServer{ - BindAddr: bindAddress, - Port: httpPort, - Client: mgr.GetClient(), - Mux: httpMux, - FileResolver: resolver, - StatusRecorder: statusQueue, - } - if err := mgr.Add(httpServer); err != nil { - return fmt.Errorf("adding HTTP server: %w", err) + httpServer := &netboot.HTTPServer{ + BindAddr: bindAddress, + Port: httpPort, + Client: mgr.GetClient(), + Mux: httpMux, + FileResolver: resolver, + StatusRecorder: statusQueue, + } + if err := mgr.Add(httpServer); err != nil { + return fmt.Errorf("adding HTTP server: %w", err) + } } siteDisplay := site @@ -331,7 +355,8 @@ func ServePXECmd() *cobra.Command { } PrintConfig("site", siteDisplay) - PrintConfig("leader-election", leID) + PrintConfig("role", string(role)) + PrintConfig("leader-election", fmt.Sprintf("%t", components.leaderElection)) PrintConfig("serve-url", serveURL) PrintConfig("default-netboot-image", defaultNetbootImage) PrintConfig("cache-dir", cacheDir) @@ -339,15 +364,23 @@ func ServePXECmd() *cobra.Command { PrintConfig("dhcp-port", fmt.Sprintf("%d", dhcpPort)) fmt.Println() - if dhcpInterface != "" { - PrintService("DHCP", fmt.Sprintf("%s:%d", dhcpInterface, dhcpPort)) - } else { - PrintService("DHCP", fmt.Sprintf("0.0.0.0:%d (relay)", dhcpPort)) + if components.dhcp { + if dhcpInterface != "" { + PrintService("DHCP", fmt.Sprintf("%s:%d", dhcpInterface, dhcpPort)) + } else { + PrintService("DHCP", fmt.Sprintf("0.0.0.0:%d (relay)", dhcpPort)) + } } - PrintService("TFTP", fmt.Sprintf("%s:69", bindAddress)) - PrintService("HTTP", fmt.Sprintf("%s:%d", bindAddress, httpPort)) - PrintService("Redfish", "reconciler") + if components.tftp { + PrintService("TFTP", fmt.Sprintf("%s:69", bindAddress)) + } + if components.http { + PrintService("HTTP", fmt.Sprintf("%s:%d", bindAddress, httpPort)) + } + if components.redfish { + PrintService("Redfish", "reconciler") + } PrintReady() return mgr.Start(ctx) diff --git a/internal/metalman/commands/server.go b/internal/metalman/commands/server.go new file mode 100644 index 000000000..54924978a --- /dev/null +++ b/internal/metalman/commands/server.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import "github.com/spf13/cobra" + +// ServerCmd runs Metalman's replicated artifact and callback server. +func ServerCmd() *cobra.Command { + return newMetalmanRoleCmd(metalmanRoleServer, "Run the Metalman artifact and control server") +} From ce535033d30f973057d64b28b302470ee7e8aac0 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:27:06 +0000 Subject: [PATCH 05/43] metalman: decouple edge from cluster runtime --- internal/metalman/commands/edge.go | 93 +++++++++++++++++++++++- internal/metalman/commands/roles.go | 7 +- internal/metalman/commands/roles_test.go | 70 ++++++++++++++++-- 3 files changed, 156 insertions(+), 14 deletions(-) diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go index 296e55cc0..342e60594 100644 --- a/internal/metalman/commands/edge.go +++ b/internal/metalman/commands/edge.go @@ -3,9 +3,98 @@ package commands -import "github.com/spf13/cobra" +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httputil" + "net/url" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" +) + +const defaultHTTPReadHeaderTimeout = 10 * time.Second // EdgeCmd runs Metalman's provisioning-network protocol edge. func EdgeCmd() *cobra.Command { - return newMetalmanRoleCmd(metalmanRoleEdge, "Run the Metalman provisioning protocol edge") + var ( + backendURL string + bindAddress string + httpPort int + ) + + cmd := &cobra.Command{ + Use: string(metalmanRoleEdge), + Short: "Run the Metalman provisioning protocol edge", + RunE: func(cmd *cobra.Command, _ []string) error { + backend, err := parseEdgeBackendURL(backendURL) + if err != nil { + return err + } + + ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + addr := fmt.Sprintf("%s:%d", bindAddress, httpPort) + server := &http.Server{ + Addr: addr, + Handler: newEdgeProxy(backend), + ReadHeaderTimeout: defaultHTTPReadHeaderTimeout, + } + + go func() { + <-ctx.Done() + _ = server.Shutdown(context.Background()) + }() + + PrintConfig("role", string(metalmanRoleEdge)) + PrintConfig("backend-url", backend.String()) + PrintService("HTTP", addr) + PrintReady() + slog.InfoContext(ctx, "starting Metalman edge", "addr", addr, "backend", backend.String()) + + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("serving edge HTTP: %w", err) + } + + return nil + }, + } + + cmd.Flags().StringVar(&backendURL, "backend-url", "", "Metalman server base URL") + cmd.Flags().StringVar(&bindAddress, "bind-address", "0.0.0.0", "IP address to bind the edge HTTP listener") + cmd.Flags().IntVar(&httpPort, "http-port", 8880, "Port for the edge HTTP listener") + _ = cmd.MarkFlagRequired("backend-url") + + return cmd +} + +func parseEdgeBackendURL(value string) (*url.URL, error) { + backend, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("parsing --backend-url: %w", err) + } + if backend.Scheme != "http" && backend.Scheme != "https" { + return nil, errors.New("--backend-url must use http or https") + } + if backend.Host == "" { + return nil, errors.New("--backend-url must include a host") + } + + return backend, nil +} + +func newEdgeProxy(backend *url.URL) http.Handler { + proxy := httputil.NewSingleHostReverseProxy(backend) + proxy.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) { + slog.Warn("Metalman edge backend request failed", "err", err) + http.Error(w, "Metalman backend unavailable", http.StatusBadGateway) + } + + return proxy } diff --git a/internal/metalman/commands/roles.go b/internal/metalman/commands/roles.go index eb71b08ad..909a64cb9 100644 --- a/internal/metalman/commands/roles.go +++ b/internal/metalman/commands/roles.go @@ -42,12 +42,7 @@ func componentsForRole(role metalmanRole) roleComponents { } case metalmanRoleEdge: return roleComponents{ - ociReconciler: true, - dhcp: true, - tftp: true, - http: true, - attestation: true, - statusUpdates: true, + http: true, } case metalmanRoleLegacy: return roleComponents{ diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index 2a382f627..d043ad0d3 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -4,6 +4,10 @@ package commands import ( + "io" + "net/http" + "net/http/httptest" + "net/url" "testing" "github.com/spf13/cobra" @@ -41,12 +45,7 @@ func TestMetalmanRoleComponentsAreIsolated(t *testing.T) { name: "edge", role: metalmanRoleEdge, want: roleComponents{ - ociReconciler: true, - dhcp: true, - tftp: true, - http: true, - attestation: true, - statusUpdates: true, + http: true, }, }, } @@ -82,3 +81,62 @@ func TestMetalmanRoleCommands(t *testing.T) { }) } } + +func TestEdgeCommandRequiresOnlyBackendConnection(t *testing.T) { + t.Parallel() + + cmd := EdgeCmd() + for _, name := range []string{"backend-url", "bind-address", "http-port"} { + if cmd.Flags().Lookup(name) == nil { + t.Errorf("edge command has no --%s flag", name) + } + } + + for _, name := range []string{"site", "cache-dir", "leader-elect-lease-duration"} { + if cmd.Flags().Lookup(name) != nil { + t.Errorf("edge command unexpectedly has controller flag --%s", name) + } + } +} + +func TestEdgeProxyPreservesSessionPathAndRange(t *testing.T) { + t.Parallel() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/netboot/capability/artifact/disk.img.gz" { + t.Errorf("backend path = %q", r.URL.Path) + } + if got := r.Header.Get("Range"); got != "bytes=4096-8191" { + t.Errorf("backend Range = %q", got) + } + w.Header().Set("Content-Range", "bytes 4096-8191/16384") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("range")) + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/capability/artifact/disk.img.gz", nil) + request.Header.Set("Range", "bytes=4096-8191") + response := httptest.NewRecorder() + + newEdgeProxy(backendURL).ServeHTTP(response, request) + + if response.Code != http.StatusPartialContent { + t.Fatalf("status = %d, want %d", response.Code, http.StatusPartialContent) + } + if got := response.Header().Get("Content-Range"); got != "bytes 4096-8191/16384" { + t.Errorf("Content-Range = %q", got) + } + body, err := io.ReadAll(response.Result().Body) + if err != nil { + t.Fatal(err) + } + if got := string(body); got != "range" { + t.Errorf("body = %q", got) + } +} From 8f44c2952dd9d592b52949300616f0d86f1edfcd Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:32:38 +0000 Subject: [PATCH 06/43] metalman: serve immutable session artifacts --- internal/metalman/netboot/capability.go | 99 ++++++++++++++++ internal/metalman/netboot/oci_cache.go | 15 ++- internal/metalman/netboot/session_http.go | 109 +++++++++++++++++ .../metalman/netboot/session_http_test.go | 110 ++++++++++++++++++ 4 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 internal/metalman/netboot/capability.go create mode 100644 internal/metalman/netboot/session_http.go create mode 100644 internal/metalman/netboot/session_http_test.go diff --git a/internal/metalman/netboot/capability.go b/internal/metalman/netboot/capability.go new file mode 100644 index 000000000..512212022 --- /dev/null +++ b/internal/metalman/netboot/capability.go @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +const minimumCapabilityKeySize = 32 + +type capabilityClaims struct { + KeyID string `json:"kid"` + Session string `json:"session"` + SessionUID string `json:"sessionUID"` + Expires int64 `json:"expires"` +} + +// CapabilitySigner issues operation-scoped bearer capabilities without +// persisting the bearer value in Kubernetes. +type CapabilitySigner struct { + key []byte + keyID string + now func() time.Time +} + +func NewCapabilitySigner(key []byte, keyID string, now func() time.Time) (*CapabilitySigner, error) { + if len(key) < minimumCapabilityKeySize { + return nil, fmt.Errorf("capability HMAC key must be at least %d bytes", minimumCapabilityKeySize) + } + if strings.TrimSpace(keyID) == "" { + return nil, errors.New("capability key ID is required") + } + if now == nil { + now = time.Now + } + + return &CapabilitySigner{key: append([]byte(nil), key...), keyID: keyID, now: now}, nil +} + +func (s *CapabilitySigner) Sign(session *v1alpha3.NetbootSession) (string, error) { + if session == nil || session.Name == "" || session.UID == "" { + return "", errors.New("session name and UID are required") + } + + claims := capabilityClaims{ + KeyID: s.keyID, + Session: session.Name, + SessionUID: string(session.UID), + Expires: session.Spec.ExpiresAt.Unix(), + } + payload, err := json.Marshal(claims) + if err != nil { + return "", fmt.Errorf("marshal capability claims: %w", err) + } + + encoded := base64.RawURLEncoding.EncodeToString(payload) + return encoded + "." + s.signature(encoded), nil +} + +func (s *CapabilitySigner) Verify(session *v1alpha3.NetbootSession, capability string) error { + encoded, signature, ok := strings.Cut(capability, ".") + if !ok || encoded == "" || signature == "" || !hmac.Equal([]byte(signature), []byte(s.signature(encoded))) { + return errors.New("invalid capability") + } + + payload, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return errors.New("invalid capability") + } + var claims capabilityClaims + if err := json.Unmarshal(payload, &claims); err != nil { + return errors.New("invalid capability") + } + if session == nil || claims.KeyID != s.keyID || claims.Session != session.Name || claims.SessionUID != string(session.UID) || claims.Expires != session.Spec.ExpiresAt.Unix() { + return errors.New("invalid capability") + } + if !s.now().Before(time.Unix(claims.Expires, 0)) { + return errors.New("expired capability") + } + + return nil +} + +func (s *CapabilitySigner) signature(payload string) string { + mac := hmac.New(sha256.New, s.key) + _, _ = mac.Write([]byte(payload)) + + return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} diff --git a/internal/metalman/netboot/oci_cache.go b/internal/metalman/netboot/oci_cache.go index 5c9fe4d69..cdd83a5a0 100644 --- a/internal/metalman/netboot/oci_cache.go +++ b/internal/metalman/netboot/oci_cache.go @@ -193,6 +193,19 @@ func (c *OCICache) ResolvePathForArchitecture(imageRef, architecture, reqPath st return "", false, fmt.Errorf("image %q for architecture %q not yet pulled", imageRef, normalizeArchitecture(architecture)) } + return c.ResolveDigestPathForArchitecture(digest, architecture, reqPath) +} + +// ResolveDigestPathForArchitecture resolves a file from an immutable cached +// digest without consulting the mutable image-reference index. +func (c *OCICache) ResolveDigestPathForArchitecture(digest, architecture, reqPath string) (diskPath string, isTemplate bool, err error) { + if digest == "" { + return "", false, fmt.Errorf("image digest is required") + } + if !c.IsCachedForArchitecture(digest, architecture) { + return "", false, fmt.Errorf("%w: image digest %q for architecture %q is not cached", ErrNotYetDownloaded, digest, normalizeArchitecture(architecture)) + } + // Reject absolute paths and Windows-style volume names. if filepath.IsAbs(reqPath) || filepath.VolumeName(reqPath) != "" { return "", false, fmt.Errorf("invalid request path %q: must be relative", reqPath) @@ -223,7 +236,7 @@ func (c *OCICache) ResolvePathForArchitecture(imageRef, architecture, reqPath st return cleanedBase, false, nil } - return "", false, fmt.Errorf("file not found in image %q: %s", imageRef, reqPath) + return "", false, fmt.Errorf("file not found in image digest %q: %s", digest, reqPath) } // InvalidateRef removes the digest mapping for an image reference, diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go new file mode 100644 index 000000000..cc6b9d2a5 --- /dev/null +++ b/internal/metalman/netboot/session_http.go @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "errors" + "net/http" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +// SessionHTTPServer serves only immutable artifacts authorized by a +// session-scoped capability. +type SessionHTTPServer struct { + Client client.Reader + Cache *OCICache + Capabilities *CapabilitySigner +} + +func (s *SessionHTTPServer) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /v1/netboot/sessions/{session}/{capability}/artifacts/{artifact...}", s.handleArtifact) + + return mux +} + +func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Request) { + if s.Client == nil || s.Cache == nil || s.Capabilities == nil { + http.Error(w, "session server unavailable", http.StatusServiceUnavailable) + return + } + + var session v1alpha3.NetbootSession + if err := s.Client.Get(r.Context(), client.ObjectKey{Name: r.PathValue("session")}, &session); err != nil { + if apierrors.IsNotFound(err) { + http.NotFound(w, r) + return + } + http.Error(w, "loading session", http.StatusServiceUnavailable) + return + } + if err := s.Capabilities.Verify(&session, r.PathValue("capability")); err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive { + http.Error(w, "session unavailable", http.StatusServiceUnavailable) + return + } + + artifact, ok := sessionArtifact(&session, r.PathValue("artifact")) + if !ok { + http.NotFound(w, r) + return + } + image, ok := sessionArtifactImage(&session, artifact.Source) + if !ok { + http.NotFound(w, r) + return + } + + reqPath := strings.TrimPrefix(artifact.Path, "/disk/") + if reqPath == artifact.Path { + http.NotFound(w, r) + return + } + diskPath, isTemplate, err := s.Cache.ResolveDigestPathForArchitecture(image.Digest, session.Spec.Boot.Architecture, reqPath) + if err != nil { + if errors.Is(err, ErrNotYetDownloaded) { + w.Header().Set("Retry-After", "5") + http.Error(w, "artifact unavailable", http.StatusServiceUnavailable) + return + } + http.NotFound(w, r) + return + } + if isTemplate { + http.Error(w, "rendered artifact unavailable", http.StatusServiceUnavailable) + return + } + + http.ServeFile(w, r, diskPath) +} + +func sessionArtifact(session *v1alpha3.NetbootSession, name string) (v1alpha3.NetbootSessionArtifact, bool) { + for _, artifact := range session.Spec.Artifacts.Files { + if artifact.Name == name { + return artifact, true + } + } + + return v1alpha3.NetbootSessionArtifact{}, false +} + +func sessionArtifactImage(session *v1alpha3.NetbootSession, source string) (v1alpha3.NetbootSessionImage, bool) { + switch source { + case "MachineImage": + return session.Spec.Artifacts.MachineImage, true + case "NetbootImage": + return session.Spec.Artifacts.NetbootImage, true + default: + return v1alpha3.NetbootSessionImage{}, false + } +} diff --git a/internal/metalman/netboot/session_http_test.go b/internal/metalman/netboot/session_http_test.go new file mode 100644 index 000000000..a8939d136 --- /dev/null +++ b/internal/metalman/netboot/session_http_test.go @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +func TestSessionHTTPServesImmutableArtifactWithCapabilityAndRange(t *testing.T) { + t.Parallel() + + const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + cache := setupOCICache(t, "unused.example/image:latest", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", map[string][]byte{ + "disk.img.gz": []byte("0123456789"), + }) + require.NoError(t, populateOCICache(cache.CacheDir, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", map[string][]byte{ + "disk.img.gz": []byte("mutable-tag-content"), + })) + cache.SetDigest("unused.example/image:latest", "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") + session := testNetbootSession("session-a", digest) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-a/"+capability+"/artifacts/disk.img.gz", nil) + request.Header.Set("Range", "bytes=2-5") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusPartialContent, response.Code) + require.Equal(t, "bytes 2-5/10", response.Header().Get("Content-Range")) + require.Equal(t, "2345", response.Body.String()) +} + +func TestSessionHTTPRejectsInvalidExpiredAndUnlistedCapabilities(t *testing.T) { + t.Parallel() + + const digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + cache := setupOCICache(t, "unused.example/image:latest", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", map[string][]byte{ + "disk.img.gz": []byte("disk"), + "secret": []byte("secret"), + }) + session := testNetbootSession("session-b", digest) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + now := time.Unix(1_700_000_000, 0) + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { return now }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() + + for name, test := range map[string]struct { + path string + wantStatus int + }{ + "invalid capability": {path: "/v1/netboot/sessions/session-b/not-a-capability/artifacts/disk.img.gz", wantStatus: http.StatusUnauthorized}, + "tampered capability": {path: "/v1/netboot/sessions/session-b/" + capability + "x/artifacts/disk.img.gz", wantStatus: http.StatusUnauthorized}, + "unlisted artifact": {path: "/v1/netboot/sessions/session-b/" + capability + "/artifacts/secret", wantStatus: http.StatusNotFound}, + } { + t.Run(name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, test.path, nil)) + require.Equal(t, test.wantStatus, response.Code) + }) + } + + now = session.Spec.ExpiresAt.Add(time.Second) + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-b/"+capability+"/artifacts/disk.img.gz", nil)) + require.Equal(t, http.StatusUnauthorized, response.Code) + body, err := io.ReadAll(response.Result().Body) + require.NoError(t, err) + require.NotContains(t, string(body), capability) +} + +func testNetbootSession(name, digest string) *v1alpha3.NetbootSession { + return &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid")}, + Spec: v1alpha3.NetbootSessionSpec{ + Machine: v1alpha3.NetbootSessionObjectSnapshot{Name: "machine-a", UID: "machine-uid", Generation: 1}, + Operation: v1alpha3.NetbootSessionObjectSnapshot{Name: "operation-a", UID: "operation-uid", Generation: 1}, + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{Name: "endpoint-a", UID: "endpoint-uid", ExternalURL: "https://boot.example.com"}, + Boot: v1alpha3.NetbootSessionBoot{Architecture: v1alpha3.PXEArchitectureAMD64}, + Artifacts: v1alpha3.NetbootSessionArtifacts{ + MachineImage: v1alpha3.NetbootSessionImage{Reference: "unused.example/image:latest", Digest: digest}, + NetbootImage: v1alpha3.NetbootSessionImage{Reference: "unused.example/netboot:latest", Digest: digest}, + Files: []v1alpha3.NetbootSessionArtifact{{Name: "disk.img.gz", Source: "MachineImage", Path: "/disk/disk.img.gz"}}, + }, + ExpiresAt: metav1.NewTime(time.Unix(1_700_003_600, 0)), + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } +} From 5b210485b7f577ce04e353288e7672c9ceab658e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:37:30 +0000 Subject: [PATCH 07/43] metalman: scope callbacks to netboot sessions --- internal/metalman/commands/roles.go | 2 + internal/metalman/commands/roles_test.go | 1 + internal/metalman/commands/serve_pxe.go | 22 ++++ internal/metalman/machineops/controller.go | 8 +- .../metalman/machineops/controller_test.go | 52 ++++---- .../metalman/machineops/session_status.go | 103 ++++++++++++++++ .../machineops/session_status_test.go | 81 +++++++++++++ internal/metalman/netboot/session_http.go | 112 ++++++++++++++---- .../metalman/netboot/session_http_test.go | 39 ++++++ 9 files changed, 373 insertions(+), 47 deletions(-) create mode 100644 internal/metalman/machineops/session_status.go create mode 100644 internal/metalman/machineops/session_status_test.go diff --git a/internal/metalman/commands/roles.go b/internal/metalman/commands/roles.go index 909a64cb9..8c0b293a1 100644 --- a/internal/metalman/commands/roles.go +++ b/internal/metalman/commands/roles.go @@ -22,6 +22,7 @@ type roleComponents struct { http bool attestation bool statusUpdates bool + sessionHTTP bool } func componentsForRole(role metalmanRole) roleComponents { @@ -39,6 +40,7 @@ func componentsForRole(role metalmanRole) roleComponents { http: true, attestation: true, statusUpdates: true, + sessionHTTP: true, } case metalmanRoleEdge: return roleComponents{ diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index d043ad0d3..50e1562d0 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -39,6 +39,7 @@ func TestMetalmanRoleComponentsAreIsolated(t *testing.T) { http: true, attestation: true, statusUpdates: true, + sessionHTTP: true, }, }, { diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index 3effd441b..dfe7d9896 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -63,6 +63,8 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { operationPollInterval time.Duration defaultNetbootImage string defaultNetbootPullSecret string + capabilityKeyFile string + capabilityKeyID string ) components := componentsForRole(role) @@ -326,6 +328,22 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { if components.http { httpMux := http.NewServeMux() + if components.sessionHTTP { + capabilityKey, err := os.ReadFile(capabilityKeyFile) + if err != nil { + return fmt.Errorf("reading capability key: %w", err) + } + capabilities, err := netboot.NewCapabilitySigner(capabilityKey, capabilityKeyID, nil) + if err != nil { + return fmt.Errorf("creating capability signer: %w", err) + } + (&netboot.SessionHTTPServer{ + Client: mgr.GetClient(), + Cache: ociCache, + Capabilities: capabilities, + StatusRecorder: &metalmachineops.SessionStatusRecorder{Client: mgr.GetClient()}, + }).RegisterHandlers(httpMux) + } if components.attestation { attestHandler := &attestation.Handler{ Clientset: clientset, @@ -404,6 +422,10 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { cmd.Flags().DurationVar(&operationPollInterval, "operation-poll-interval", 5*time.Second, "Poll interval for in-progress MachineOperations") cmd.Flags().StringVar(&defaultNetbootImage, "default-netboot-image", DefaultNetbootImage, "Default OCI image containing PXE netboot artifacts") cmd.Flags().StringVar(&defaultNetbootPullSecret, "default-netboot-pull-secret", "", "Namespaced Secret reference (namespace/name) for pulling the default netboot OCI image") + if components.sessionHTTP { + cmd.Flags().StringVar(&capabilityKeyFile, "capability-key-file", "/var/run/secrets/metalman/capability.key", "File containing the shared capability HMAC key") + cmd.Flags().StringVar(&capabilityKeyID, "capability-key-id", "v1", "Identifier for the active capability HMAC key") + } return cmd } diff --git a/internal/metalman/machineops/controller.go b/internal/metalman/machineops/controller.go index bd6b9d70e..c51e56057 100644 --- a/internal/metalman/machineops/controller.go +++ b/internal/metalman/machineops/controller.go @@ -608,11 +608,11 @@ func (r *Reconciler) waitForPowerAction(target v1alpha3.MachineOperationTargetSt } func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOperation, machine *v1alpha3.Machine, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { - if !apimeta.IsStatusConditionTrue(op.Status.Conditions, v1alpha3.MachineOperationConditionBootImageWritten) { + if !apimeta.IsStatusConditionTrue(target.Conditions, v1alpha3.MachineOperationConditionBootImageWritten) { return r.waitForRepaveBoot(ctx, machine, target, now) } - if change, done := cloudInitReplaceStatus(op, target, now); done { + if change, done := cloudInitReplaceStatus(target, now); done { return change } @@ -817,8 +817,8 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma return targetChange{target: target} } -func cloudInitReplaceStatus(op *v1alpha3.MachineOperation, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) (targetChange, bool) { - cond := apimeta.FindStatusCondition(op.Status.Conditions, v1alpha3.MachineOperationConditionCloudInitDone) +func cloudInitReplaceStatus(target v1alpha3.MachineOperationTargetStatus, now metav1.Time) (targetChange, bool) { + cond := apimeta.FindStatusCondition(target.Conditions, v1alpha3.MachineOperationConditionCloudInitDone) if cond != nil { switch cond.Status { case metav1.ConditionTrue: diff --git a/internal/metalman/machineops/controller_test.go b/internal/metalman/machineops/controller_test.go index 6a7f239d6..0061f3ee1 100644 --- a/internal/metalman/machineops/controller_test.go +++ b/internal/metalman/machineops/controller_test.go @@ -368,8 +368,8 @@ func TestReconcilerRequestsHostReplaceOnceAndCompletesAfterRepave(t *testing.T) require.Equal(t, metav1.ConditionUnknown, cloudInitCond.Status) require.Equal(t, "Pending", cloudInitCond.Reason) - markOperationCondition(t, c, op.Name, v1alpha3.MachineOperationConditionBootImageWritten, metav1.ConditionTrue, "Succeeded", "boot image written") - markOperationCondition(t, c, op.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") + markTargetCondition(t, c, op.Name, machine.Name, v1alpha3.MachineOperationConditionBootImageWritten, metav1.ConditionTrue, "Succeeded", "boot image written") + markTargetCondition(t, c, op.Name, machine.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") require.NoError(t, c.Create(context.Background(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}})) _, err = reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) @@ -800,12 +800,12 @@ func TestReconcilerKeepsHostReplaceInProgressUntilNodeExists(t *testing.T) { op.Spec.MachineRef = machine.Name op.Spec.TTLSecondsAfterFinished = &ttl op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() @@ -832,12 +832,12 @@ func TestReconcilerKeepsHostReplaceInProgressUntilCloudInitCompletes(t *testing. op := testOperation("op-replace-wait-cloudinit", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}}).WithStatusSubresource(op, machine).Build() @@ -855,7 +855,7 @@ func TestReconcilerKeepsHostReplaceInProgressUntilCloudInitCompletes(t *testing. require.Equal(t, v1alpha3.OperationStageWaitingCloudInit, waiting.Status.Targets[0].Stage) require.Equal(t, "waiting for first-boot cloud-init to complete", waiting.Status.Targets[0].Message) - markOperationCondition(t, c, op.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") + markTargetCondition(t, c, op.Name, machine.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") _, err = reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) require.NoError(t, err) @@ -874,12 +874,12 @@ func TestReconcilerFailsHostReplaceWhenCloudInitFails(t *testing.T) { op := testOperation("op-replace-cloudinit-failed", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionFalse) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionFalse), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}}).WithStatusSubresource(op, machine).Build() @@ -903,8 +903,8 @@ func TestReconcilerTimesOutHostReplaceCloudInitCondition(t *testing.T) { op := testOperation("op-replace-cloudinit-timeout", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown) - apimeta.SetStatusCondition(&op.Status.Conditions, metav1.Condition{ + conditions := hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown) + apimeta.SetStatusCondition(&conditions, metav1.Condition{ Type: v1alpha3.MachineOperationConditionCloudInitDone, Status: metav1.ConditionFalse, Reason: "Running", @@ -916,6 +916,7 @@ func TestReconcilerTimesOutHostReplaceCloudInitCondition(t *testing.T) { Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingCloudInit, ObservedGeneration: machine.Generation, + Conditions: conditions, }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}}).WithStatusSubresource(op, machine).Build() @@ -940,12 +941,12 @@ func TestReconcilerUsesKubernetesNodeRefForHostReplaceCompletion(t *testing.T) { op := testOperation("op-replace-custom-node", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() @@ -977,21 +978,18 @@ func TestReconcilerCompletesAllHostReplaceTargetsWhenOperationMilestonesComplete op := testOperation("op-replace-multi", v1alpha3.OperationHostReplace) op.Spec.MachineSelector = &metav1.LabelSelector{MatchLabels: map[string]string{siteLabel: "rack-a"}} op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = []metav1.Condition{ - {Type: v1alpha3.MachineOperationConditionBootLoaderDownloaded, Status: metav1.ConditionUnknown, Reason: "Pending"}, - {Type: v1alpha3.MachineOperationConditionBootImageWritten, Status: metav1.ConditionTrue, Reason: "Succeeded"}, - {Type: v1alpha3.MachineOperationConditionCloudInitDone, Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "machine-a completed cloud-init"}, - } op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{ { MachineRef: machineA.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }, { MachineRef: machineB.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingCloudInit, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }, } @@ -1400,19 +1398,27 @@ func testRedfishSecret() *corev1.Secret { } } -func markOperationCondition(t *testing.T, c client.Client, opName, conditionType string, status metav1.ConditionStatus, reason, message string) { +func markTargetCondition(t *testing.T, c client.Client, opName, machineName, conditionType string, status metav1.ConditionStatus, reason, message string) { t.Helper() var op v1alpha3.MachineOperation require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: opName}, &op)) - apimeta.SetStatusCondition(&op.Status.Conditions, metav1.Condition{ - Type: conditionType, - Status: status, - Reason: reason, - Message: message, - ObservedGeneration: op.Generation, - }) - require.NoError(t, c.Status().Update(context.Background(), &op)) + for i := range op.Status.Targets { + if op.Status.Targets[i].MachineRef != machineName { + continue + } + apimeta.SetStatusCondition(&op.Status.Targets[i].Conditions, metav1.Condition{ + Type: conditionType, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: op.Status.Targets[i].ObservedGeneration, + }) + require.NoError(t, c.Status().Update(context.Background(), &op)) + + return + } + t.Fatalf("operation %s has no target %s", opName, machineName) } func hostReplaceConditions(bootImage, cloudInit metav1.ConditionStatus) []metav1.Condition { diff --git a/internal/metalman/machineops/session_status.go b/internal/metalman/machineops/session_status.go new file mode 100644 index 000000000..4b205f831 --- /dev/null +++ b/internal/metalman/machineops/session_status.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +// SessionStatusRecorder durably records a server-observed milestone on the +// exact immutable session and its exact MachineOperation target. +type SessionStatusRecorder struct { + Client client.Client + Now func() metav1.Time +} + +func (r *SessionStatusRecorder) RecordCondition(ctx context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error { + if r == nil || r.Client == nil || sessionName == "" || sessionUID == "" { + return fmt.Errorf("session status recorder is not configured") + } + + var session v1alpha3.NetbootSession + if err := r.Client.Get(ctx, client.ObjectKey{Name: sessionName}, &session); err != nil { + return fmt.Errorf("get NetbootSession %s: %w", sessionName, err) + } + if session.UID != sessionUID { + return fmt.Errorf("NetbootSession %s identity changed", sessionName) + } + + condition.ObservedGeneration = session.Generation + condition.LastTransitionTime = r.now() + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var latest v1alpha3.NetbootSession + if err := r.Client.Get(ctx, client.ObjectKey{Name: sessionName}, &latest); err != nil { + return err + } + if latest.UID != sessionUID { + return fmt.Errorf("NetbootSession %s identity changed", sessionName) + } + if apimeta.IsStatusConditionTrue(latest.Status.Conditions, condition.Type) { + return nil + } + apimeta.SetStatusCondition(&latest.Status.Conditions, condition) + if latest.Status.Phase == v1alpha3.NetbootSessionPhaseReady { + latest.Status.Phase = v1alpha3.NetbootSessionPhaseActive + } + + return r.Client.Status().Update(ctx, &latest) + }); err != nil { + return fmt.Errorf("update NetbootSession %s status: %w", sessionName, err) + } + + return r.recordTargetCondition(ctx, &session, condition) +} + +func (r *SessionStatusRecorder) recordTargetCondition(ctx context.Context, session *v1alpha3.NetbootSession, condition metav1.Condition) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + var operation v1alpha3.MachineOperation + if err := r.Client.Get(ctx, client.ObjectKey{Name: session.Spec.Operation.Name}, &operation); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if operation.UID != session.Spec.Operation.UID { + return fmt.Errorf("MachineOperation %s identity changed", operation.Name) + } + + for i := range operation.Status.Targets { + target := &operation.Status.Targets[i] + if target.Input == nil || target.Input.NetbootSessionRef == nil || target.Input.NetbootSessionRef.Name != session.Name || target.Input.NetbootSessionRef.UID != session.UID { + continue + } + if apimeta.IsStatusConditionTrue(target.Conditions, condition.Type) { + return nil + } + condition.ObservedGeneration = target.ObservedGeneration + apimeta.SetStatusCondition(&target.Conditions, condition) + + return r.Client.Status().Update(ctx, &operation) + } + + return fmt.Errorf("MachineOperation %s has no target for NetbootSession %s", operation.Name, session.Name) + }) +} + +func (r *SessionStatusRecorder) now() metav1.Time { + if r.Now != nil { + return r.Now() + } + + return metav1.Now() +} diff --git a/internal/metalman/machineops/session_status_test.go b/internal/metalman/machineops/session_status_test.go new file mode 100644 index 000000000..e6d7fd6cf --- /dev/null +++ b/internal/metalman/machineops/session_status_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +func TestSessionStatusRecorderUpdatesOnlyExactSessionAndTarget(t *testing.T) { + t.Parallel() + + sessionA := statusTestSession("session-a", "session-a-uid", "operation-uid", "machine-a") + sessionB := statusTestSession("session-b", "session-b-uid", "operation-uid", "machine-b") + op := testOperation("operation-a", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Status.Phase = v1alpha3.OperationPhaseInProgress + op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{ + {MachineRef: "machine-a", Phase: v1alpha3.OperationPhaseInProgress, Input: &v1alpha3.MachineOperationTargetInput{NetbootSessionRef: &v1alpha3.NetbootSessionReference{Name: sessionA.Name, UID: sessionA.UID}}}, + {MachineRef: "machine-b", Phase: v1alpha3.OperationPhaseInProgress, Input: &v1alpha3.MachineOperationTargetInput{NetbootSessionRef: &v1alpha3.NetbootSessionReference{Name: sessionB.Name, UID: sessionB.UID}}}, + } + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(sessionA, sessionB, op).WithStatusSubresource(sessionA, sessionB, op).Build() + recorder := &SessionStatusRecorder{Client: c, Now: fixedNow} + + require.NoError(t, recorder.RecordCondition(context.Background(), sessionA.Name, sessionA.UID, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionBootImageWritten, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "image written", + })) + + var updatedA v1alpha3.NetbootSession + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: sessionA.Name}, &updatedA)) + require.True(t, apimeta.IsStatusConditionTrue(updatedA.Status.Conditions, v1alpha3.NetbootSessionConditionBootImageWritten)) + + var updatedB v1alpha3.NetbootSession + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: sessionB.Name}, &updatedB)) + require.Nil(t, apimeta.FindStatusCondition(updatedB.Status.Conditions, v1alpha3.NetbootSessionConditionBootImageWritten)) + + var updatedOp v1alpha3.MachineOperation + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: op.Name}, &updatedOp)) + require.True(t, apimeta.IsStatusConditionTrue(updatedOp.Status.Targets[0].Conditions, v1alpha3.MachineOperationConditionBootImageWritten)) + require.Nil(t, apimeta.FindStatusCondition(updatedOp.Status.Targets[1].Conditions, v1alpha3.MachineOperationConditionBootImageWritten)) + require.Nil(t, apimeta.FindStatusCondition(updatedOp.Status.Conditions, v1alpha3.MachineOperationConditionBootImageWritten)) +} + +func TestSessionStatusRecorderRejectsStaleSessionUID(t *testing.T) { + t.Parallel() + + session := statusTestSession("session-a", "current-uid", "operation-uid", "machine-a") + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(session).WithStatusSubresource(session).Build() + recorder := &SessionStatusRecorder{Client: c} + + err := recorder.RecordCondition(context.Background(), session.Name, types.UID("stale-uid"), metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionCloudInitDone, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + }) + require.ErrorContains(t, err, "identity changed") +} + +func statusTestSession(name string, uid, operationUID types.UID, machineName string) *v1alpha3.NetbootSession { + return &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: uid}, + Spec: v1alpha3.NetbootSessionSpec{ + Machine: v1alpha3.NetbootSessionObjectSnapshot{Name: machineName, UID: types.UID(machineName + "-uid"), Generation: 1}, + Operation: v1alpha3.NetbootSessionObjectSnapshot{Name: "operation-a", UID: operationUID, Generation: 1}, + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseActive}, + } +} diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go index cc6b9d2a5..0077d5da4 100644 --- a/internal/metalman/netboot/session_http.go +++ b/internal/metalman/netboot/session_http.go @@ -4,11 +4,14 @@ package netboot import ( + "context" "errors" "net/http" "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" @@ -17,48 +20,75 @@ import ( // SessionHTTPServer serves only immutable artifacts authorized by a // session-scoped capability. type SessionHTTPServer struct { - Client client.Reader - Cache *OCICache - Capabilities *CapabilitySigner + Client client.Reader + Cache *OCICache + Capabilities *CapabilitySigner + StatusRecorder SessionConditionRecorder +} + +// SessionConditionRecorder persists a milestone for an exact session identity. +type SessionConditionRecorder interface { + RecordCondition(ctx context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error } func (s *SessionHTTPServer) Handler() http.Handler { mux := http.NewServeMux() - mux.HandleFunc("GET /v1/netboot/sessions/{session}/{capability}/artifacts/{artifact...}", s.handleArtifact) + s.RegisterHandlers(mux) return mux } -func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Request) { - if s.Client == nil || s.Cache == nil || s.Capabilities == nil { - http.Error(w, "session server unavailable", http.StatusServiceUnavailable) +// RegisterHandlers adds authenticated session artifact and callback routes. +func (s *SessionHTTPServer) RegisterHandlers(mux *http.ServeMux) { + mux.HandleFunc("GET /v1/netboot/sessions/{session}/{capability}/artifacts/{artifact...}", s.handleArtifact) + mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/callbacks/{milestone}", s.handleCallback) +} + +func (s *SessionHTTPServer) handleCallback(w http.ResponseWriter, r *http.Request) { + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + if s.StatusRecorder == nil { + http.Error(w, "session status unavailable", http.StatusServiceUnavailable) return } - var session v1alpha3.NetbootSession - if err := s.Client.Get(r.Context(), client.ObjectKey{Name: r.PathValue("session")}, &session); err != nil { - if apierrors.IsNotFound(err) { - http.NotFound(w, r) - return - } - http.Error(w, "loading session", http.StatusServiceUnavailable) + conditionType, ok := sessionMilestoneCondition(r.PathValue("milestone")) + if !ok { + http.NotFound(w, r) return } - if err := s.Capabilities.Verify(&session, r.PathValue("capability")); err != nil { - http.Error(w, "unauthorized", http.StatusUnauthorized) + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "provisioning client reported milestone", + }); err != nil { + http.Error(w, "recording session status", http.StatusServiceUnavailable) return } - if session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive { - http.Error(w, "session unavailable", http.StatusServiceUnavailable) + + w.WriteHeader(http.StatusNoContent) +} + +func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Request) { + if s.Cache == nil { + http.Error(w, "session server unavailable", http.StatusServiceUnavailable) return } - artifact, ok := sessionArtifact(&session, r.PathValue("artifact")) + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + + artifact, ok := sessionArtifact(session, r.PathValue("artifact")) if !ok { http.NotFound(w, r) return } - image, ok := sessionArtifactImage(&session, artifact.Source) + image, ok := sessionArtifactImage(session, artifact.Source) if !ok { http.NotFound(w, r) return @@ -87,6 +117,48 @@ func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Reques http.ServeFile(w, r, diskPath) } +func (s *SessionHTTPServer) authorizeSession(w http.ResponseWriter, r *http.Request) (*v1alpha3.NetbootSession, bool) { + if s.Client == nil || s.Capabilities == nil { + http.Error(w, "session server unavailable", http.StatusServiceUnavailable) + return nil, false + } + + var session v1alpha3.NetbootSession + if err := s.Client.Get(r.Context(), client.ObjectKey{Name: r.PathValue("session")}, &session); err != nil { + if apierrors.IsNotFound(err) { + http.NotFound(w, r) + return nil, false + } + http.Error(w, "loading session", http.StatusServiceUnavailable) + return nil, false + } + if err := s.Capabilities.Verify(&session, r.PathValue("capability")); err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return nil, false + } + if session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive { + http.Error(w, "session unavailable", http.StatusServiceUnavailable) + return nil, false + } + + return &session, true +} + +func sessionMilestoneCondition(milestone string) (string, bool) { + switch milestone { + case "boot-loader-downloaded": + return v1alpha3.NetbootSessionConditionBootLoaderDownloaded, true + case "boot-image-written": + return v1alpha3.NetbootSessionConditionBootImageWritten, true + case "cloud-init-done": + return v1alpha3.NetbootSessionConditionCloudInitDone, true + case "attested": + return v1alpha3.NetbootSessionConditionAttested, true + default: + return "", false + } +} + func sessionArtifact(session *v1alpha3.NetbootSession, name string) (v1alpha3.NetbootSessionArtifact, bool) { for _, artifact := range session.Spec.Artifacts.Files { if artifact.Name == name { diff --git a/internal/metalman/netboot/session_http_test.go b/internal/metalman/netboot/session_http_test.go index a8939d136..cac391367 100644 --- a/internal/metalman/netboot/session_http_test.go +++ b/internal/metalman/netboot/session_http_test.go @@ -4,6 +4,7 @@ package netboot import ( + "context" "io" "net/http" "net/http/httptest" @@ -90,6 +91,44 @@ func TestSessionHTTPRejectsInvalidExpiredAndUnlistedCapabilities(t *testing.T) { require.NotContains(t, string(body), capability) } +func TestSessionHTTPRecordsAuthenticatedSessionCallback(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-c", "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + recorder := &recordingSessionConditionRecorder{} + handler := (&SessionHTTPServer{Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, StatusRecorder: recorder}).Handler() + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/v1/netboot/sessions/session-c/"+capability+"/callbacks/boot-image-written", nil)) + + require.Equal(t, http.StatusNoContent, response.Code) + require.Equal(t, session.Name, recorder.sessionName) + require.Equal(t, session.UID, recorder.sessionUID) + require.Equal(t, v1alpha3.NetbootSessionConditionBootImageWritten, recorder.condition.Type) + require.Equal(t, metav1.ConditionTrue, recorder.condition.Status) +} + +type recordingSessionConditionRecorder struct { + sessionName string + sessionUID types.UID + condition metav1.Condition +} + +func (r *recordingSessionConditionRecorder) RecordCondition(_ context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error { + r.sessionName = sessionName + r.sessionUID = sessionUID + r.condition = condition + + return nil +} + func testNetbootSession(name, digest string) *v1alpha3.NetbootSession { return &v1alpha3.NetbootSession{ ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid")}, From 95b7b1f1841403c16f9e35685428e6a5d8e92672 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:47:19 +0000 Subject: [PATCH 08/43] metalman: advertise session capability boot URLs --- .../v1alpha3/netboot_validation_test.go | 8 +++ api/machina/v1alpha3/netbootsession_types.go | 4 ++ .../unbounded-cloud.io_netbootsessions.yaml | 6 ++ internal/metalman/commands/roles.go | 2 + internal/metalman/commands/roles_test.go | 1 + internal/metalman/commands/serve_pxe.go | 17 ++++- internal/metalman/machineops/controller.go | 67 +++++++++++++------ .../metalman/machineops/controller_test.go | 45 +++++++++++++ .../metalman/machineops/session_manager.go | 26 ++++++- .../machineops/session_manager_test.go | 26 ++++++- internal/metalman/netboot/session_http.go | 23 +++++++ .../metalman/netboot/session_http_test.go | 21 ++++++ 12 files changed, 221 insertions(+), 25 deletions(-) diff --git a/api/machina/v1alpha3/netboot_validation_test.go b/api/machina/v1alpha3/netboot_validation_test.go index b19d26805..cef7975c1 100644 --- a/api/machina/v1alpha3/netboot_validation_test.go +++ b/api/machina/v1alpha3/netboot_validation_test.go @@ -59,6 +59,14 @@ func TestNetbootSessionSchema(t *testing.T) { assertSchemaValidations(t, spec, map[string]string{ "self == oldSelf": "netboot session spec is immutable", }) + boot := spec.Properties["boot"] + firmware, ok := boot.Properties["firmwareArtifact"] + if !ok { + t.Fatal("session boot snapshot is missing firmwareArtifact") + } + if firmware.MinLength == nil || *firmware.MinLength != 1 { + t.Error("session boot firmwareArtifact must be non-empty") + } status := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] for _, field := range []string{"phase", "conditions"} { diff --git a/api/machina/v1alpha3/netbootsession_types.go b/api/machina/v1alpha3/netbootsession_types.go index dce9b8c53..6bdd00e3c 100644 --- a/api/machina/v1alpha3/netbootsession_types.go +++ b/api/machina/v1alpha3/netbootsession_types.go @@ -127,6 +127,10 @@ type NetbootSessionBoot struct { ConfigurationSource NetbootConfigurationSource `json:"configurationSource"` NetworkMode NetbootNetworkMode `json:"networkMode"` + // FirmwareArtifact is the named immutable artifact advertised to firmware. + // +kubebuilder:validation:MinLength=1 + FirmwareArtifact string `json:"firmwareArtifact"` + // Architecture selects the boot artifact platform. // +kubebuilder:validation:Enum=amd64;arm64 Architecture string `json:"architecture"` diff --git a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml index df26ce67d..4f0c1404b 100644 --- a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml @@ -224,6 +224,11 @@ spec: - subnetMask type: object type: array + firmwareArtifact: + description: FirmwareArtifact is the named immutable artifact + advertised to firmware. + minLength: 1 + type: string networkMode: description: NetbootNetworkMode configures the firmware provisioning interface. @@ -244,6 +249,7 @@ spec: required: - architecture - configurationSource + - firmwareArtifact - networkMode - transport type: object diff --git a/internal/metalman/commands/roles.go b/internal/metalman/commands/roles.go index 8c0b293a1..aa9eccc9a 100644 --- a/internal/metalman/commands/roles.go +++ b/internal/metalman/commands/roles.go @@ -23,6 +23,7 @@ type roleComponents struct { attestation bool statusUpdates bool sessionHTTP bool + sessionManager bool } func componentsForRole(role metalmanRole) roleComponents { @@ -33,6 +34,7 @@ func componentsForRole(role metalmanRole) roleComponents { ociReconciler: true, redfish: true, machineOps: true, + sessionManager: true, } case metalmanRoleServer: return roleComponents{ diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index 50e1562d0..2baf0639c 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -29,6 +29,7 @@ func TestMetalmanRoleComponentsAreIsolated(t *testing.T) { ociReconciler: true, redfish: true, machineOps: true, + sessionManager: true, }, }, { diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index dfe7d9896..f38b897c9 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -255,6 +255,20 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { } if components.machineOps { + var sessionHTTPBootURL func(*v1alpha3.NetbootSession) (string, error) + if components.sessionManager { + capabilityKey, err := os.ReadFile(capabilityKeyFile) + if err != nil { + return fmt.Errorf("reading capability key: %w", err) + } + capabilities, err := netboot.NewCapabilitySigner(capabilityKey, capabilityKeyID, nil) + if err != nil { + return fmt.Errorf("creating capability signer: %w", err) + } + sessionHTTPBootURL = func(session *v1alpha3.NetbootSession) (string, error) { + return netboot.SessionArtifactURL(capabilities, session, session.Spec.Boot.FirmwareArtifact) + } + } if err := (&metalmachineops.Reconciler{ Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), @@ -262,6 +276,7 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { PowerClients: &metalmachineops.RedfishPowerClientFactory{Reader: mgr.GetClient(), Pool: redfishPool}, Sessions: sessionManager, HTTPBootURL: resolver.HTTPBootURL, + SessionHTTPBootURL: sessionHTTPBootURL, MaxConcurrentMachines: operationMaxConcurrentMachines, MaxAttempts: operationMaxAttempts, PollInterval: operationPollInterval, @@ -422,7 +437,7 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { cmd.Flags().DurationVar(&operationPollInterval, "operation-poll-interval", 5*time.Second, "Poll interval for in-progress MachineOperations") cmd.Flags().StringVar(&defaultNetbootImage, "default-netboot-image", DefaultNetbootImage, "Default OCI image containing PXE netboot artifacts") cmd.Flags().StringVar(&defaultNetbootPullSecret, "default-netboot-pull-secret", "", "Namespaced Secret reference (namespace/name) for pulling the default netboot OCI image") - if components.sessionHTTP { + if components.sessionHTTP || components.sessionManager { cmd.Flags().StringVar(&capabilityKeyFile, "capability-key-file", "/var/run/secrets/metalman/capability.key", "File containing the shared capability HMAC key") cmd.Flags().StringVar(&capabilityKeyID, "capability-key-id", "v1", "Identifier for the active capability HMAC key") } diff --git a/internal/metalman/machineops/controller.go b/internal/metalman/machineops/controller.go index c51e56057..dbf51d98a 100644 --- a/internal/metalman/machineops/controller.go +++ b/internal/metalman/machineops/controller.go @@ -79,6 +79,7 @@ type Reconciler struct { PowerClients PowerClientFactory Sessions SessionManager HTTPBootURL func(*v1alpha3.Machine) (string, error) + SessionHTTPBootURL func(*v1alpha3.NetbootSession) (string, error) MaxConcurrentMachines int MaxAttempts int32 PollInterval time.Duration @@ -397,8 +398,10 @@ func (r *Reconciler) advanceTarget(ctx context.Context, op *v1alpha3.MachineOper case v1alpha3.OperationHostReboot: return r.advanceReboot(ctx, op, &machine, target, now) case v1alpha3.OperationHostReplace: + var session *v1alpha3.NetbootSession if r.Sessions != nil { - session, err := r.Sessions.Ensure(ctx, op, &machine) + var err error + session, err = r.Sessions.Ensure(ctx, op, &machine) if err != nil { if errors.Is(err, netboot.ErrNotYetDownloaded) { target.Message = "waiting for OCI images to resolve to immutable digests" @@ -431,7 +434,7 @@ func (r *Reconciler) advanceTarget(ctx context.Context, op *v1alpha3.MachineOper } } - return r.advanceReplace(ctx, op, &machine, target, now) + return r.advanceReplace(ctx, op, &machine, session, target, now) default: return failTarget(target, reasonUnsupportedTarget, fmt.Sprintf("%s is not handled by metalman", op.Spec.OperationKind), now) } @@ -607,9 +610,9 @@ func (r *Reconciler) waitForPowerAction(target v1alpha3.MachineOperationTargetSt return targetChange{}, false } -func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOperation, machine *v1alpha3.Machine, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { +func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOperation, machine *v1alpha3.Machine, session *v1alpha3.NetbootSession, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { if !apimeta.IsStatusConditionTrue(target.Conditions, v1alpha3.MachineOperationConditionBootImageWritten) { - return r.waitForRepaveBoot(ctx, machine, target, now) + return r.waitForRepaveBoot(ctx, machine, session, target, now) } if change, done := cloudInitReplaceStatus(target, now); done { @@ -633,9 +636,13 @@ func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOpe return completeTarget(target, "HostReplace completed", now) } -func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, machine *v1alpha3.Machine) error { - if machine.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP { - bootURL, staticConfig, err := r.httpBootConfig(machine) +func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, machine *v1alpha3.Machine, session *v1alpha3.NetbootSession) error { + transport := machine.Spec.Netboot().TargetTransport() + if session != nil { + transport = session.Spec.Boot.Transport + } + if transport == v1alpha3.NetbootTransportHTTP { + bootURL, staticConfig, err := r.httpBootConfig(machine, session) if err != nil { return err } @@ -650,17 +657,27 @@ func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, ma return nil } -func (r *Reconciler) httpBootConfig(machine *v1alpha3.Machine) (string, redfish.StaticIPv4Config, error) { - if r.HTTPBootURL == nil { - return "", redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot URL resolver is not configured") +func (r *Reconciler) httpBootConfig(machine *v1alpha3.Machine, session *v1alpha3.NetbootSession) (string, redfish.StaticIPv4Config, error) { + var ( + bootURL string + err error + ) + if session != nil { + if r.SessionHTTPBootURL == nil { + return "", redfish.StaticIPv4Config{}, fmt.Errorf("session HTTP boot URL resolver is not configured") + } + bootURL, err = r.SessionHTTPBootURL(session) + } else { + if r.HTTPBootURL == nil { + return "", redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot URL resolver is not configured") + } + bootURL, err = r.HTTPBootURL(machine) } - - bootURL, err := r.HTTPBootURL(machine) if err != nil { return "", redfish.StaticIPv4Config{}, err } - staticConfig, err := httpBootStaticNetworkConfig(machine) + staticConfig, err := httpBootStaticNetworkConfig(machine, session) if err != nil { return "", redfish.StaticIPv4Config{}, err } @@ -668,12 +685,18 @@ func (r *Reconciler) httpBootConfig(machine *v1alpha3.Machine) (string, redfish. return bootURL, staticConfig, nil } -func httpBootStaticNetworkConfig(machine *v1alpha3.Machine) (redfish.StaticIPv4Config, error) { - if machine.Spec.Netboot() == nil || len(machine.Spec.Netboot().DHCPLeases) == 0 { +func httpBootStaticNetworkConfig(machine *v1alpha3.Machine, session *v1alpha3.NetbootSession) (redfish.StaticIPv4Config, error) { + var leases []v1alpha3.DHCPLease + if session != nil { + leases = session.Spec.Boot.DHCPLeases + } else if machine.Spec.Netboot() != nil { + leases = machine.Spec.Netboot().DHCPLeases + } + if len(leases) == 0 { return redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot requires at least one static lease in spec.host.netboot.dhcpLeases") } - lease := machine.Spec.Netboot().DHCPLeases[0] + lease := leases[0] config := redfish.StaticIPv4Config{ MAC: lease.MAC, Address: lease.IPv4, @@ -743,7 +766,7 @@ func setBIOSHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string return pc.SetBootOverride(ctx, redfish.BootTargetUefiHTTP, redfish.BootOnce) } -func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Machine, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { +func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Machine, session *v1alpha3.NetbootSession, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { if target.Stage == v1alpha3.OperationStageWaitingRepave && target.LastAttemptAt != nil { if now.Sub(target.LastAttemptAt.Time) < r.powerActionTimeout() { target.Message = "waiting for PXE installer to write the boot image" @@ -759,8 +782,12 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma target.LastAttemptAt = nil } - if machine.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP { - if _, _, err := r.httpBootConfig(machine); err != nil { + transport := machine.Spec.Netboot().TargetTransport() + if session != nil { + transport = session.Spec.Boot.Transport + } + if transport == v1alpha3.NetbootTransportHTTP { + if _, _, err := r.httpBootConfig(machine, session); err != nil { if errors.Is(err, netboot.ErrNotYetDownloaded) { target.Message = "waiting for OCI image to become available" @@ -801,7 +828,7 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma return targetChange{target: target} } - if err := r.configureRepaveBoot(ctx, pc, machine); err != nil { + if err := r.configureRepaveBoot(ctx, pc, machine, session); err != nil { return retryTarget(target, err, now, r.maxAttempts()) } diff --git a/internal/metalman/machineops/controller_test.go b/internal/metalman/machineops/controller_test.go index 0061f3ee1..9207faf45 100644 --- a/internal/metalman/machineops/controller_test.go +++ b/internal/metalman/machineops/controller_test.go @@ -416,6 +416,51 @@ func TestReconcilerPersistsReadySessionBeforeHostReplaceSideEffects(t *testing.T require.Equal(t, []string{"machine-session:SetBootOverride:Pxe:Continuous", "machine-session:On"}, power.calls) } +func TestReconcilerUsesSessionCapabilityURLForHTTPBoot(t *testing.T) { + t.Parallel() + + s := testScheme(t) + machine := testBareMetalMachine("machine-session-http", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 3 + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportTFTP + machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:ff", IPv4: "192.0.2.99"}} + op := testOperation("op-session-http", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 2 + op.Spec.MachineRef = machine.Name + session := &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: "netboot-session", UID: "session-uid"}, + Spec: v1alpha3.NetbootSessionSpec{ + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{ExternalURL: "https://boot.example.com"}, + Boot: v1alpha3.NetbootSessionBoot{ + Transport: v1alpha3.NetbootTransportHTTP, + FirmwareArtifact: "bootx64.efi", + DHCPLeases: []v1alpha3.DHCPLease{httpBootLease()}, + }, + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } + c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() + power := &recordingPowerClient{states: map[string]redfish.PowerState{machine.Name: redfish.PowerOff}} + reconciler := testReconciler(c, power, "rack-a") + reconciler.Sessions = &recordingSessionManager{session: session} + reconciler.SessionHTTPBootURL = func(got *v1alpha3.NetbootSession) (string, error) { + require.Equal(t, session, got) + + return "https://boot.example.com/v1/netboot/sessions/netboot-session/capability/artifacts/bootx64.efi", nil + } + + _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + + require.Contains(t, power.calls, "machine-session-http:SetHTTPBootOverride:https://boot.example.com/v1/netboot/sessions/netboot-session/capability/artifacts/bootx64.efi") +} + func TestReconcilerPowersOnHostReplaceTargetWhenOff(t *testing.T) { t.Parallel() diff --git a/internal/metalman/machineops/session_manager.go b/internal/metalman/machineops/session_manager.go index 7db37399a..d6f4339af 100644 --- a/internal/metalman/machineops/session_manager.go +++ b/internal/metalman/machineops/session_manager.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -85,6 +86,14 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph if netbootDigest == "" { return nil, fmt.Errorf("%w: netboot image %q for architecture %q", netboot.ErrNotYetDownloaded, netbootRef, architecture) } + metadata, err := m.Cache.MetadataForArchitecture(netbootDigest, architecture) + if err != nil { + return nil, fmt.Errorf("read netboot image metadata: %w", err) + } + firmwareArtifact := firmwareArtifactForTransport(metadata, netbootSpec.TargetTransport()) + if firmwareArtifact == "" { + return nil, fmt.Errorf("netboot image %q has no firmware artifact for %s", netbootRef, netbootSpec.TargetTransport()) + } now := m.now() session := &v1alpha3.NetbootSession{ @@ -107,6 +116,7 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph Transport: netbootSpec.TargetTransport(), ConfigurationSource: targetConfigurationSource(netbootSpec), NetworkMode: targetNetworkMode(netbootSpec), + FirmwareArtifact: firmwareArtifact, Architecture: architecture, DHCPLeases: append([]v1alpha3.DHCPLease(nil), netbootSpec.DHCPLeases...), TargetDisk: netbootSpec.TargetDisk, @@ -122,7 +132,7 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph Digest: netbootDigest, PullSecretRef: netbootPullSecret.DeepCopy(), }, - Files: sessionArtifacts(), + Files: sessionArtifacts(firmwareArtifact), }, ExpiresAt: metav1.NewTime(now.Add(defaultSessionTTL)), }, @@ -221,13 +231,25 @@ func targetNetworkMode(spec *v1alpha3.PXESpec) v1alpha3.NetbootNetworkMode { return spec.NetworkMode } -func sessionArtifacts() []v1alpha3.NetbootSessionArtifact { +func sessionArtifacts(firmwareArtifact string) []v1alpha3.NetbootSessionArtifact { return []v1alpha3.NetbootSessionArtifact{ {Name: "disk.img.gz", Source: "MachineImage", Path: "/disk/disk.img.gz"}, {Name: "metadata.yaml", Source: "NetbootImage", Path: "/disk/metadata.yaml"}, + {Name: firmwareArtifact, Source: "NetbootImage", Path: "/disk/" + firmwareArtifact}, } } +func firmwareArtifactForTransport(metadata *netboot.ImageMetadata, transport v1alpha3.NetbootTransport) string { + if metadata == nil { + return "" + } + if transport == v1alpha3.NetbootTransportHTTP { + return netboot.HTTPBootPathFromMetadata(metadata) + } + + return strings.TrimPrefix(metadata.DHCPBootImageName, "/") +} + func conditionStatus(ready bool) metav1.ConditionStatus { if ready { return metav1.ConditionTrue diff --git a/internal/metalman/machineops/session_manager_test.go b/internal/metalman/machineops/session_manager_test.go index 96d007575..ae1b4d130 100644 --- a/internal/metalman/machineops/session_manager_test.go +++ b/internal/metalman/machineops/session_manager_test.go @@ -4,6 +4,8 @@ package machineops import ( + "os" + "path/filepath" "testing" "time" @@ -24,14 +26,18 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { machine.Generation = 4 machine.Spec.PXE.EndpointRef = "rack-a-edge" machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP op := testOperation("replace-session", v1alpha3.OperationHostReplace) op.UID = "operation-uid" op.Generation = 2 endpoint := readyTestEndpoint() cache := netboot.NewOCICache(t.TempDir()) - cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('a', 64)) - cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('b', 64)) + machineDigest := "sha256:" + stringOf('a', 64) + netbootDigest := "sha256:" + stringOf('b', 64) + cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, machineDigest) + cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, netbootDigest) + require.NoError(t, writeSessionMetadata(cache, netbootDigest, "http/bootx64.efi")) s := testScheme(t) c := fake.NewClientBuilder().WithScheme(s).WithObjects(endpoint).WithStatusSubresource(&v1alpha3.NetbootSession{}).Build() @@ -49,6 +55,12 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { require.Equal(t, machine.UID, session.Spec.Machine.UID) require.Equal(t, op.UID, session.Spec.Operation.UID) require.Equal(t, endpoint.Spec.ExternalURL, session.Spec.Endpoint.ExternalURL) + require.Equal(t, "http/bootx64.efi", session.Spec.Boot.FirmwareArtifact) + require.Contains(t, session.Spec.Artifacts.Files, v1alpha3.NetbootSessionArtifact{ + Name: "http/bootx64.efi", + Source: "NetbootImage", + Path: "/disk/http/bootx64.efi", + }) require.True(t, session.Spec.ExpiresAt.Time.Equal(fixedNow().Add(24*time.Hour))) machine.Spec.PXE.Image = "ghcr.io/test/changed:v2" @@ -102,6 +114,7 @@ func TestSessionManagerPromotesExistingSessionWhenEndpointBecomesReady(t *testin cache := netboot.NewOCICache(t.TempDir()) cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('a', 64)) cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('b', 64)) + require.NoError(t, writeSessionMetadata(cache, "sha256:"+stringOf('b', 64), "bootx64.efi")) c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(endpoint).WithStatusSubresource(endpoint, &v1alpha3.NetbootSession{}).Build() manager := &KubernetesSessionManager{Client: c, Cache: cache} @@ -117,6 +130,15 @@ func TestSessionManagerPromotesExistingSessionWhenEndpointBecomesReady(t *testin require.Equal(t, v1alpha3.NetbootSessionPhaseReady, session.Status.Phase) } +func writeSessionMetadata(cache *netboot.OCICache, digest, httpBootPath string) error { + diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.DefaultPXEArchitecture) + if err := os.MkdirAll(diskDir, 0o755); err != nil { + return err + } + + return os.WriteFile(filepath.Join(diskDir, "metadata.yaml"), []byte("dhcpBootImageName: "+httpBootPath+"\nhttpBootPath: "+httpBootPath+"\n"), 0o600) +} + func readyTestEndpoint() *v1alpha3.NetbootEndpoint { return &v1alpha3.NetbootEndpoint{ ObjectMeta: metav1.ObjectMeta{Name: "rack-a-edge", UID: "endpoint-uid", Generation: 3}, diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go index 0077d5da4..67fb82605 100644 --- a/internal/metalman/netboot/session_http.go +++ b/internal/metalman/netboot/session_http.go @@ -6,7 +6,9 @@ package netboot import ( "context" "errors" + "fmt" "net/http" + pathpkg "path" "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -31,6 +33,27 @@ type SessionConditionRecorder interface { RecordCondition(ctx context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error } +// SessionArtifactURL returns the externally advertised capability URL for one +// artifact listed by the immutable session. +func SessionArtifactURL(signer *CapabilitySigner, session *v1alpha3.NetbootSession, artifactName string) (string, error) { + if signer == nil || session == nil { + return "", errors.New("capability signer and session are required") + } + if _, ok := sessionArtifact(session, artifactName); !ok { + return "", fmt.Errorf("artifact %q is not listed by session %s", artifactName, session.Name) + } + cleanArtifact := strings.TrimPrefix(artifactName, "/") + if cleanArtifact == "" || pathpkg.Clean(cleanArtifact) != cleanArtifact || strings.HasPrefix(artifactName, "/") { + return "", fmt.Errorf("invalid artifact name %q", artifactName) + } + capability, err := signer.Sign(session) + if err != nil { + return "", err + } + + return JoinServeURLPath(session.Spec.Endpoint.ExternalURL, pathpkg.Join("v1/netboot/sessions", session.Name, capability, "artifacts", cleanArtifact)) +} + func (s *SessionHTTPServer) Handler() http.Handler { mux := http.NewServeMux() s.RegisterHandlers(mux) diff --git a/internal/metalman/netboot/session_http_test.go b/internal/metalman/netboot/session_http_test.go index cac391367..df2503bdc 100644 --- a/internal/metalman/netboot/session_http_test.go +++ b/internal/metalman/netboot/session_http_test.go @@ -115,6 +115,27 @@ func TestSessionHTTPRecordsAuthenticatedSessionCallback(t *testing.T) { require.Equal(t, metav1.ConditionTrue, recorder.condition.Status) } +func TestSessionArtifactURLUsesEndpointAndCapability(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-url", "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd") + session.Spec.Endpoint.ExternalURL = "https://boot.example.com/base/" + session.Spec.Boot.FirmwareArtifact = "http/bootx64.efi" + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, v1alpha3.NetbootSessionArtifact{ + Name: "http/bootx64.efi", Source: "NetbootImage", Path: "/disk/http/bootx64.efi", + }) + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + bootURL, err := SessionArtifactURL(signer, session, session.Spec.Boot.FirmwareArtifact) + require.NoError(t, err) + require.Equal(t, "https://boot.example.com/base/v1/netboot/sessions/session-url/"+capability+"/artifacts/http/bootx64.efi", bootURL) +} + type recordingSessionConditionRecorder struct { sessionName string sessionUID types.UID From 97b5520339f6fd48cae9424ea0a850c29a28dca1 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 16:54:08 +0000 Subject: [PATCH 09/43] metalman: resume edge artifact transfers --- internal/metalman/commands/edge.go | 128 ++++++++++++++++++++++- internal/metalman/commands/roles_test.go | 113 ++++++++++++++++++++ 2 files changed, 240 insertions(+), 1 deletion(-) diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go index 342e60594..e8d2fbc17 100644 --- a/internal/metalman/commands/edge.go +++ b/internal/metalman/commands/edge.go @@ -7,11 +7,14 @@ import ( "context" "errors" "fmt" + "io" "log/slog" "net/http" "net/http/httputil" "net/url" "os/signal" + "strconv" + "strings" "syscall" "time" @@ -20,6 +23,8 @@ import ( const defaultHTTPReadHeaderTimeout = 10 * time.Second +const maxArtifactBackendAttempts = 3 + // EdgeCmd runs Metalman's provisioning-network protocol edge. func EdgeCmd() *cobra.Command { var ( @@ -96,5 +101,126 @@ func newEdgeProxy(backend *url.URL) http.Handler { http.Error(w, "Metalman backend unavailable", http.StatusBadGateway) } - return proxy + return &edgeProxy{ + proxy: proxy, + transport: http.DefaultTransport, + } +} + +type edgeProxy struct { + proxy *httputil.ReverseProxy + transport http.RoundTripper +} + +func (e *edgeProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/v1/netboot/sessions/") && strings.Contains(r.URL.Path, "/artifacts/") { + e.serveArtifact(w, r) + return + } + + e.proxy.ServeHTTP(w, r) +} + +func (e *edgeProxy) serveArtifact(w http.ResponseWriter, r *http.Request) { + request := r.Clone(r.Context()) + e.proxy.Director(request) + + response, err := e.transport.RoundTrip(request) + if err != nil { + slog.Warn("Metalman edge artifact request failed", "err", err) + http.Error(w, "Metalman backend unavailable", http.StatusBadGateway) + return + } + + copyResponseHeaders(w.Header(), response.Header) + w.WriteHeader(response.StatusCode) + + remaining := response.ContentLength + if remaining <= 0 || (response.StatusCode != http.StatusOK && response.StatusCode != http.StatusPartialContent) { + _, _ = io.Copy(w, response.Body) + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + return + } + + start, end, ok := responseByteRange(response) + if !ok { + _, _ = io.Copy(w, response.Body) + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + return + } + + written, copyErr := io.Copy(w, response.Body) + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + start += written + remaining -= written + if copyErr == nil && remaining == 0 { + return + } + + for attempt := 2; attempt <= maxArtifactBackendAttempts && remaining > 0 && start <= end; attempt++ { + request = r.Clone(r.Context()) + e.proxy.Director(request) + request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + response, err = e.transport.RoundTrip(request) + if err != nil { + slog.Warn("Metalman edge artifact resume failed", "path", r.URL.Path, "err", err) + continue + } + if response.StatusCode != http.StatusPartialContent || response.ContentLength != remaining { + response.Body.Close() //nolint:errcheck // Invalid resume response. + slog.Warn("Metalman edge artifact resume returned an invalid range", "path", r.URL.Path, "status", response.StatusCode) + return + } + resumeStart, resumeEnd, valid := responseByteRange(response) + if !valid || resumeStart != start || resumeEnd != end { + response.Body.Close() //nolint:errcheck // Invalid resume response. + slog.Warn("Metalman edge artifact resume returned mismatched bytes", "path", r.URL.Path) + return + } + + written, copyErr = io.Copy(w, response.Body) + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + start += written + remaining -= written + if copyErr == nil && remaining == 0 { + return + } + } + + slog.Warn("Metalman edge artifact transfer failed", "path", r.URL.Path, "err", copyErr) +} + +func responseByteRange(response *http.Response) (int64, int64, bool) { + if response.StatusCode == http.StatusOK { + return 0, response.ContentLength - 1, true + } + + value := response.Header.Get("Content-Range") + if !strings.HasPrefix(value, "bytes ") { + return 0, 0, false + } + rangeAndSize := strings.SplitN(strings.TrimPrefix(value, "bytes "), "/", 2) + if len(rangeAndSize) != 2 { + return 0, 0, false + } + bounds := strings.SplitN(rangeAndSize[0], "-", 2) + if len(bounds) != 2 { + return 0, 0, false + } + start, err := strconv.ParseInt(bounds[0], 10, 64) + if err != nil { + return 0, 0, false + } + end, err := strconv.ParseInt(bounds[1], 10, 64) + if err != nil || start < 0 || end < start || end-start+1 != response.ContentLength { + return 0, 0, false + } + + return start, end, true +} + +func copyResponseHeaders(dst, src http.Header) { + for key, values := range src { + dst[key] = append([]string(nil), values...) + } } diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index 2baf0639c..5ef0d905e 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "sync/atomic" "testing" "github.com/spf13/cobra" @@ -142,3 +143,115 @@ func TestEdgeProxyPreservesSessionPathAndRange(t *testing.T) { t.Errorf("body = %q", got) } } + +func TestEdgeProxyResumesTruncatedArtifactFromBackendRange(t *testing.T) { + t.Parallel() + + const artifact = "immutable-artifact" + + var requests atomic.Int32 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + if got := r.Header.Get("Range"); got != "" { + t.Errorf("initial Range = %q, want empty", got) + } + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("resume Range = %q, want %q", got, "bytes=9-17") + } + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + t.Errorf("unexpected backend request %d", requests.Load()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + edge := httptest.NewServer(newEdgeProxy(backendURL)) + defer edge.Close() + + response, err := http.Get(edge.URL + "/v1/netboot/sessions/session/capability/artifacts/disk.img.gz") //nolint:noctx // Test request. + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() //nolint:errcheck // Test cleanup. + + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if got := string(body); got != artifact { + t.Errorf("body = %q, want %q", got, artifact) + } + if got := requests.Load(); got != 2 { + t.Errorf("backend requests = %d, want 2", got) + } +} + +func TestEdgeProxyRetriesFailedArtifactResumeRequest(t *testing.T) { + t.Parallel() + + const artifact = "immutable-artifact" + + var requests atomic.Int32 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijacking failed resume request: %v", err) + return + } + _ = conn.Close() + case 3: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("resume Range = %q, want %q", got, "bytes=9-17") + } + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + t.Errorf("unexpected backend request %d", requests.Load()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + edge := httptest.NewServer(newEdgeProxy(backendURL)) + defer edge.Close() + + response, err := http.Get(edge.URL + "/v1/netboot/sessions/session/capability/artifacts/disk.img.gz") //nolint:noctx // Test request. + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() //nolint:errcheck // Test cleanup. + + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if got := string(body); got != artifact { + t.Errorf("body = %q, want %q", got, artifact) + } + if got := requests.Load(); got != 3 { + t.Errorf("backend requests = %d, want 3", got) + } +} From dab49d57afaec18312203bb5c65b4426d17d0568 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 17:14:35 +0000 Subject: [PATCH 10/43] metalman: serve DHCP from immutable sessions --- deploy/machina/06-metalman-rbac.yaml.tmpl | 11 ++ internal/metalman/commands/edge.go | 60 +++++- internal/metalman/commands/roles_test.go | 2 +- internal/metalman/commands/serve_pxe.go | 8 + internal/metalman/dhcp/backend.go | 112 +++++++++++ internal/metalman/dhcp/backend_test.go | 96 +++++++++ internal/metalman/dhcp/dhcp.go | 99 +++++++--- internal/metalman/dhcp/dhcp_test.go | 68 +++++++ internal/metalman/netboot/capability.go | 4 + internal/metalman/netboot/edge_auth.go | 48 +++++ internal/metalman/netboot/edge_auth_test.go | 64 ++++++ .../metalman/netboot/session_dhcp_test.go | 183 ++++++++++++++++++ internal/metalman/netboot/session_http.go | 98 +++++++++- 13 files changed, 816 insertions(+), 37 deletions(-) create mode 100644 internal/metalman/dhcp/backend.go create mode 100644 internal/metalman/dhcp/backend_test.go create mode 100644 internal/metalman/netboot/edge_auth.go create mode 100644 internal/metalman/netboot/edge_auth_test.go create mode 100644 internal/metalman/netboot/session_dhcp_test.go diff --git a/deploy/machina/06-metalman-rbac.yaml.tmpl b/deploy/machina/06-metalman-rbac.yaml.tmpl index a590e7afd..e7b2eff83 100644 --- a/deploy/machina/06-metalman-rbac.yaml.tmpl +++ b/deploy/machina/06-metalman-rbac.yaml.tmpl @@ -12,6 +12,13 @@ metadata: name: metalman-bootstrap namespace: {{ default "unbounded-system" .Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metalman-edge + namespace: {{ default "unbounded-system" .Namespace }} + --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -145,6 +152,10 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + # Authenticate audience-bound edge ServiceAccount tokens. + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go index e8d2fbc17..6d478a205 100644 --- a/internal/metalman/commands/edge.go +++ b/internal/metalman/commands/edge.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "log/slog" + "net" "net/http" "net/http/httputil" "net/url" @@ -19,6 +20,8 @@ import ( "time" "github.com/spf13/cobra" + + "github.com/Azure/unbounded/internal/metalman/dhcp" ) const defaultHTTPReadHeaderTimeout = 10 * time.Second @@ -28,9 +31,15 @@ const maxArtifactBackendAttempts = 3 // EdgeCmd runs Metalman's provisioning-network protocol edge. func EdgeCmd() *cobra.Command { var ( - backendURL string - bindAddress string - httpPort int + backendURL string + bindAddress string + httpPort int + endpoint string + edgeTokenFile string + dhcpEnabled bool + dhcpInterface string + dhcpServerIP string + dhcpPort int ) cmd := &cobra.Command{ @@ -51,6 +60,23 @@ func EdgeCmd() *cobra.Command { Handler: newEdgeProxy(backend), ReadHeaderTimeout: defaultHTTPReadHeaderTimeout, } + if dhcpEnabled { + decisionProvider, err := dhcp.NewHTTPDecisionProviderFromTokenFile(backend.String(), endpoint, edgeTokenFile, nil) + if err != nil { + return fmt.Errorf("creating DHCP backend: %w", err) + } + serverIP, err := edgeDHCPServerIP(dhcpServerIP, dhcpInterface) + if err != nil { + return err + } + dhcpServer := &dhcp.Server{Interface: dhcpInterface, Port: dhcpPort, DecisionProvider: decisionProvider, ServerIP: serverIP} + go func() { + if err := dhcpServer.Start(ctx); err != nil && ctx.Err() == nil { + slog.ErrorContext(ctx, "Metalman edge DHCP server failed", "err", err) + stop() + } + }() + } go func() { <-ctx.Done() @@ -74,11 +100,39 @@ func EdgeCmd() *cobra.Command { cmd.Flags().StringVar(&backendURL, "backend-url", "", "Metalman server base URL") cmd.Flags().StringVar(&bindAddress, "bind-address", "0.0.0.0", "IP address to bind the edge HTTP listener") cmd.Flags().IntVar(&httpPort, "http-port", 8880, "Port for the edge HTTP listener") + cmd.Flags().StringVar(&endpoint, "endpoint", "", "NetbootEndpoint served by this edge") + cmd.Flags().StringVar(&edgeTokenFile, "edge-token-file", "/var/run/secrets/metalman/token", "Audience-bound ServiceAccount token file") + cmd.Flags().BoolVar(&dhcpEnabled, "dhcp-enabled", false, "Enable the DHCP protocol edge") + cmd.Flags().StringVar(&dhcpInterface, "dhcp-interface", "", "Provisioning interface for direct DHCP; empty enables relay-only mode") + cmd.Flags().StringVar(&dhcpServerIP, "dhcp-server-ip", "", "DHCP server IPv4 address; defaults to the interface or outbound address") + cmd.Flags().IntVar(&dhcpPort, "dhcp-port", 67, "DHCP listener port") _ = cmd.MarkFlagRequired("backend-url") + _ = cmd.MarkFlagRequired("endpoint") return cmd } +func edgeDHCPServerIP(configured, iface string) (net.IP, error) { + if configured != "" { + ip := net.ParseIP(configured).To4() + if ip == nil { + return nil, errors.New("--dhcp-server-ip must be an IPv4 address") + } + + return ip, nil + } + if iface != "" { + return InterfaceIPv4(iface) + } + + ip, err := OutboundIP() + if err != nil || ip.To4() == nil { + return nil, errors.New("detecting DHCP server IPv4 address; set --dhcp-server-ip") + } + + return ip.To4(), nil +} + func parseEdgeBackendURL(value string) (*url.URL, error) { backend, err := url.Parse(value) if err != nil { diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index 5ef0d905e..9174746b8 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -89,7 +89,7 @@ func TestEdgeCommandRequiresOnlyBackendConnection(t *testing.T) { t.Parallel() cmd := EdgeCmd() - for _, name := range []string{"backend-url", "bind-address", "http-port"} { + for _, name := range []string{"backend-url", "bind-address", "http-port", "endpoint", "edge-token-file", "dhcp-enabled", "dhcp-interface", "dhcp-server-ip", "dhcp-port"} { if cmd.Flags().Lookup(name) == nil { t.Errorf("edge command has no --%s flag", name) } diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index f38b897c9..3e8017098 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -65,6 +65,7 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { defaultNetbootPullSecret string capabilityKeyFile string capabilityKeyID string + edgeServiceAccount string ) components := componentsForRole(role) @@ -357,6 +358,10 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { Cache: ociCache, Capabilities: capabilities, StatusRecorder: &metalmachineops.SessionStatusRecorder{Client: mgr.GetClient()}, + EdgeAuthenticator: &netboot.TokenReviewEdgeAuthenticator{ + Client: clientset.AuthenticationV1(), + ServiceAccountName: edgeServiceAccount, + }, }).RegisterHandlers(httpMux) } if components.attestation { @@ -441,6 +446,9 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { cmd.Flags().StringVar(&capabilityKeyFile, "capability-key-file", "/var/run/secrets/metalman/capability.key", "File containing the shared capability HMAC key") cmd.Flags().StringVar(&capabilityKeyID, "capability-key-id", "v1", "Identifier for the active capability HMAC key") } + if components.sessionHTTP { + cmd.Flags().StringVar(&edgeServiceAccount, "edge-service-account", "metalman-edge", "ServiceAccount name accepted by internal edge APIs") + } return cmd } diff --git a/internal/metalman/dhcp/backend.go b/internal/metalman/dhcp/backend.go new file mode 100644 index 000000000..1d069229d --- /dev/null +++ b/internal/metalman/dhcp/backend.go @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package dhcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + pathpkg "path" + "strings" +) + +// HTTPDecisionProvider asks the Metalman server for an immutable session DHCP +// decision. It has no Kubernetes or OCI cache dependency. +type HTTPDecisionProvider struct { + backendURL *url.URL + endpoint string + token string + tokenFile string + client *http.Client +} + +func NewHTTPDecisionProviderFromTokenFile(backendURL, endpoint, tokenFile string, client *http.Client) (*HTTPDecisionProvider, error) { + if strings.TrimSpace(tokenFile) == "" { + return nil, errors.New("edge authentication token file is required") + } + provider, err := newHTTPDecisionProvider(backendURL, endpoint, client) + if err != nil { + return nil, err + } + provider.tokenFile = tokenFile + + return provider, nil +} + +func NewHTTPDecisionProvider(backendURL, endpoint, token string, client *http.Client) (*HTTPDecisionProvider, error) { + provider, err := newHTTPDecisionProvider(backendURL, endpoint, client) + if err != nil { + return nil, err + } + if strings.TrimSpace(token) == "" { + return nil, errors.New("edge authentication token is required") + } + provider.token = token + + return provider, nil +} + +func newHTTPDecisionProvider(backendURL, endpoint string, client *http.Client) (*HTTPDecisionProvider, error) { + backend, err := url.Parse(backendURL) + if err != nil { + return nil, fmt.Errorf("parsing backend URL: %w", err) + } + if (backend.Scheme != "http" && backend.Scheme != "https") || backend.Host == "" { + return nil, errors.New("backend URL must use HTTP or HTTPS and include a host") + } + if strings.TrimSpace(endpoint) == "" { + return nil, errors.New("netboot endpoint name is required") + } + if client == nil { + client = http.DefaultClient + } + + return &HTTPDecisionProvider{backendURL: backend, endpoint: endpoint, client: client}, nil +} + +func (p *HTTPDecisionProvider) Decide(ctx context.Context, mac string, httpClient bool) (*Decision, error) { + requestURL := *p.backendURL + requestURL.Path = pathpkg.Join(requestURL.Path, "v1/netboot/endpoints", p.endpoint, "dhcp", mac) + query := requestURL.Query() + query.Set("httpClient", fmt.Sprintf("%t", httpClient)) + requestURL.RawQuery = query.Encode() + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("creating DHCP decision request: %w", err) + } + token := p.token + if p.tokenFile != "" { + tokenBytes, err := os.ReadFile(p.tokenFile) + if err != nil { + return nil, fmt.Errorf("reading edge authentication token: %w", err) + } + token = strings.TrimSpace(string(tokenBytes)) + } + request.Header.Set("Authorization", "Bearer "+token) + + response, err := p.client.Do(request) + if err != nil { + return nil, fmt.Errorf("requesting DHCP decision: %w", err) + } + defer response.Body.Close() //nolint:errcheck // Response body is discarded after decoding. + + if response.StatusCode == http.StatusNotFound { + return nil, nil + } + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("DHCP decision server returned %s", response.Status) + } + + var decision Decision + if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { + return nil, fmt.Errorf("decoding DHCP decision: %w", err) + } + + return &decision, nil +} diff --git a/internal/metalman/dhcp/backend_test.go b/internal/metalman/dhcp/backend_test.go new file mode 100644 index 000000000..ec2c258f8 --- /dev/null +++ b/internal/metalman/dhcp/backend_test.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package dhcp + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +func TestHTTPDecisionProviderAuthenticatesEndpointRequest(t *testing.T) { + t.Parallel() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Path; got != "/v1/netboot/endpoints/edge-1/dhcp/aa:bb:cc:dd:ee:ff" { + t.Errorf("path = %q", got) + } + if got := r.URL.Query().Get("httpClient"); got != "true" { + t.Errorf("httpClient = %q", got) + } + if got := r.Header.Get("Authorization"); got != "Bearer edge-token" { + t.Errorf("Authorization = %q", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"lease":{"mac":"aa:bb:cc:dd:ee:ff","ipv4":"10.0.1.20","subnetMask":"255.255.255.0"},"transport":"HTTP","bootFile":"https://boot.example/shim.efi"}`)) + })) + defer backend.Close() + + provider, err := NewHTTPDecisionProvider(backend.URL, "edge-1", "edge-token", backend.Client()) + if err != nil { + t.Fatal(err) + } + decision, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", true) + if err != nil { + t.Fatal(err) + } + if decision.Transport != v1alpha3.NetbootTransportHTTP || decision.Lease.IPv4 != "10.0.1.20" { + t.Fatalf("decision = %#v", decision) + } +} + +func TestHTTPDecisionProviderTreatsMissingSessionAsNoDecision(t *testing.T) { + t.Parallel() + + backend := httptest.NewServer(http.NotFoundHandler()) + defer backend.Close() + + provider, err := NewHTTPDecisionProvider(backend.URL, "edge-1", "edge-token", backend.Client()) + if err != nil { + t.Fatal(err) + } + decision, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false) + if err != nil { + t.Fatal(err) + } + if decision != nil { + t.Fatalf("decision = %#v, want nil", decision) + } +} + +func TestHTTPDecisionProviderReloadsProjectedToken(t *testing.T) { + t.Parallel() + + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte("first"), 0o600); err != nil { + t.Fatal(err) + } + wantToken := "first" + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer "+wantToken { + t.Errorf("Authorization = %q, want token %q", got, wantToken) + } + http.NotFound(w, r) + })) + defer backend.Close() + + provider, err := NewHTTPDecisionProviderFromTokenFile(backend.URL, "edge-1", tokenFile, backend.Client()) + if err != nil { + t.Fatal(err) + } + if _, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false); err != nil { + t.Fatal(err) + } + wantToken = "second" + if err := os.WriteFile(tokenFile, []byte(wantToken), 0o600); err != nil { + t.Fatal(err) + } + if _, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false); err != nil { + t.Fatal(err) + } +} diff --git a/internal/metalman/dhcp/dhcp.go b/internal/metalman/dhcp/dhcp.go index 5f73d81e1..8f9d75e26 100644 --- a/internal/metalman/dhcp/dhcp.go +++ b/internal/metalman/dhcp/dhcp.go @@ -25,12 +25,26 @@ type Server struct { Interface string Port int Reader client.Reader + DecisionProvider DecisionProvider ServerIP net.IP OCICache *netboot.OCICache ServeURL string DefaultNetbootRef string } +// Decision is the immutable lease and boot information returned by the +// Metalman server for one ready netboot session. +type Decision struct { + Lease v1alpha3.DHCPLease `json:"lease"` + Transport v1alpha3.NetbootTransport `json:"transport"` + BootFile string `json:"bootFile"` +} + +// DecisionProvider resolves the active ready session for a DHCP client. +type DecisionProvider interface { + Decide(ctx context.Context, mac string, httpClient bool) (*Decision, error) +} + func (s *Server) NeedLeaderElection() bool { // Responding to unicast packets is always safe. // But we need election when binding to an interface (e.g. listening for multicast) @@ -125,33 +139,15 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { ctx := context.Background() - var list v1alpha3.MachineList - if err := s.Reader.List(ctx, &list, client.MatchingFields{indexing.IndexNodeByMAC: mac}); err != nil { - log.Error("listing Machines by MAC", "err", err) - return - } - - if len(list.Items) == 0 { - return - } - - node := &list.Items[0] - if node.Spec.Netboot() == nil { + decision, node, err := s.decision(ctx, mac, isHTTPClientRequest(m)) + if err != nil { + log.Error("resolving DHCP decision", "err", err) return } - - var lease *v1alpha3.DHCPLease - - for i := range node.Spec.Netboot().DHCPLeases { - if strings.EqualFold(node.Spec.Netboot().DHCPLeases[i].MAC, mac) { - lease = &node.Spec.Netboot().DHCPLeases[i] - break - } - } - - if lease == nil { + if decision == nil { return } + lease := &decision.Lease clientIP := net.ParseIP(lease.IPv4).To4() if clientIP == nil { @@ -189,12 +185,28 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { } } - netbootImage := node.Spec.Netboot().NetbootImage - if netbootImage == "" { - netbootImage = s.DefaultNetbootRef - } + if s.DecisionProvider != nil { + switch decision.Transport { + case v1alpha3.NetbootTransportHTTP: + if isHTTPClientRequest(m) && decision.BootFile != "" { + resp.UpdateOption(dhcpv4.OptBootFileName(decision.BootFile)) + } + case v1alpha3.NetbootTransportTFTP: + if decision.BootFile != "" { + resp.UpdateOption(dhcpv4.OptTFTPServerName(s.ServerIP.String())) + resp.UpdateOption(dhcpv4.OptBootFileName(decision.BootFile)) + resp.ServerIPAddr = s.ServerIP + } + } + } else if node != nil { + netbootImage := node.Spec.Netboot().NetbootImage + if netbootImage == "" { + netbootImage = s.DefaultNetbootRef + } - if netbootImage != "" && s.OCICache != nil { + if netbootImage == "" || s.OCICache == nil { + goto send + } architecture := node.Spec.Netboot().TargetArchitecture() meta, err := s.OCICache.MetadataForRefArchitecture(netbootImage, architecture) @@ -216,6 +228,7 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { } } +send: switch m.MessageType() { case dhcpv4.MessageTypeDiscover: resp.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeOffer)) @@ -230,13 +243,41 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { dest = peer } - log.Info("sending DHCP response", "node", node.Name, "ip", lease.IPv4, "response", resp.MessageType().String()) + log.Info("sending DHCP response", "ip", lease.IPv4, "response", resp.MessageType().String()) if _, err := conn.WriteTo(resp.ToBytes(), dest); err != nil { log.Error("sending DHCP response", "err", err) } } +func (s *Server) decision(ctx context.Context, mac string, httpClient bool) (*Decision, *v1alpha3.Machine, error) { + if s.DecisionProvider != nil { + decision, err := s.DecisionProvider.Decide(ctx, mac, httpClient) + return decision, nil, err + } + if s.Reader == nil { + return nil, nil, errors.New("DHCP decision provider is not configured") + } + + var list v1alpha3.MachineList + if err := s.Reader.List(ctx, &list, client.MatchingFields{indexing.IndexNodeByMAC: mac}); err != nil { + return nil, nil, fmt.Errorf("listing Machines by MAC: %w", err) + } + if len(list.Items) == 0 || list.Items[0].Spec.Netboot() == nil { + return nil, nil, nil + } + + node := &list.Items[0] + for i := range node.Spec.Netboot().DHCPLeases { + lease := &node.Spec.Netboot().DHCPLeases[i] + if strings.EqualFold(lease.MAC, mac) { + return &Decision{Lease: *lease}, node, nil + } + } + + return nil, nil, nil +} + func isHTTPClientRequest(m *dhcpv4.DHCPv4) bool { return strings.HasPrefix(m.ClassIdentifier(), "HTTPClient") } diff --git a/internal/metalman/dhcp/dhcp_test.go b/internal/metalman/dhcp/dhcp_test.go index 87d074eb9..b20004295 100644 --- a/internal/metalman/dhcp/dhcp_test.go +++ b/internal/metalman/dhcp/dhcp_test.go @@ -4,6 +4,7 @@ package dhcp import ( + "context" "net" "os" "path/filepath" @@ -413,6 +414,60 @@ func TestDHCPHandlerHTTPBootClientGetsHTTPBootURL(t *testing.T) { } } +func TestDHCPHandlerUsesBackendSessionDecision(t *testing.T) { + t.Parallel() + + mac, err := net.ParseMAC("aa:bb:cc:dd:ee:f5") + if err != nil { + t.Fatal(err) + } + serverIP := net.ParseIP("10.0.1.254").To4() + provider := &fakeDecisionProvider{decision: &Decision{ + Lease: v1alpha3.DHCPLease{ + MAC: mac.String(), + IPv4: "10.0.1.15", + SubnetMask: "255.255.255.0", + Gateway: "10.0.1.1", + }, + Transport: v1alpha3.NetbootTransportHTTP, + BootFile: "https://boot.example/v1/netboot/sessions/session/capability/artifacts/shimx64.efi", + }} + srv := &Server{ + Interface: "eth0", + ServerIP: serverIP, + DecisionProvider: provider, + } + + discover, err := dhcpv4.NewDiscovery(mac) + if err != nil { + t.Fatal(err) + } + discover.UpdateOption(dhcpv4.OptClassIdentifier("HTTPClient:Arch:00016:UNDI:003016")) + + conn := &fakePacketConn{} + srv.handler(conn, &net.UDPAddr{IP: net.ParseIP("10.0.1.15"), Port: 68}, discover) + + if provider.mac != mac.String() { + t.Errorf("provider MAC = %q, want %q", provider.mac, mac.String()) + } + if !provider.httpClient { + t.Error("provider did not receive HTTP client identity") + } + if conn.written == nil { + t.Fatal("expected DHCP response, got none") + } + response, err := dhcpv4.FromBytes(conn.written) + if err != nil { + t.Fatal(err) + } + if got := response.BootFileNameOption(); got != provider.decision.BootFile { + t.Errorf("bootfile = %q, want %q", got, provider.decision.BootFile) + } + if got := response.YourIPAddr.String(); got != provider.decision.Lease.IPv4 { + t.Errorf("lease IP = %q, want %q", got, provider.decision.Lease.IPv4) + } +} + func TestDHCPHandlerUnknownMAC(t *testing.T) { mac, _ := net.ParseMAC("ff:ff:ff:ff:ff:ff") serverIP := net.ParseIP("10.0.1.254").To4() @@ -772,6 +827,19 @@ type fakePacketConn struct { dest net.Addr } +type fakeDecisionProvider struct { + decision *Decision + mac string + httpClient bool +} + +func (f *fakeDecisionProvider) Decide(_ context.Context, mac string, httpClient bool) (*Decision, error) { + f.mac = mac + f.httpClient = httpClient + + return f.decision, nil +} + func (f *fakePacketConn) ReadFrom(b []byte) (int, net.Addr, error) { return 0, nil, nil } func (f *fakePacketConn) WriteTo(b []byte, addr net.Addr) (int, error) { f.written = make([]byte, len(b)) diff --git a/internal/metalman/netboot/capability.go b/internal/metalman/netboot/capability.go index 512212022..2e438d65b 100644 --- a/internal/metalman/netboot/capability.go +++ b/internal/metalman/netboot/capability.go @@ -97,3 +97,7 @@ func (s *CapabilitySigner) signature(payload string) string { return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) } + +func (s *CapabilitySigner) IsExpired(session *v1alpha3.NetbootSession) bool { + return session == nil || !s.now().Before(session.Spec.ExpiresAt.Time) +} diff --git a/internal/metalman/netboot/edge_auth.go b/internal/metalman/netboot/edge_auth.go new file mode 100644 index 000000000..5cadcf322 --- /dev/null +++ b/internal/metalman/netboot/edge_auth.go @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "context" + "net/http" + "slices" + "strings" + + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1" +) + +const EdgeTokenAudience = "metalman-edge" + +// TokenReviewEdgeAuthenticator validates audience-bound Kubernetes ServiceAccount tokens. +type TokenReviewEdgeAuthenticator struct { + Client authenticationclient.AuthenticationV1Interface + ServiceAccountName string +} + +func (a *TokenReviewEdgeAuthenticator) Authenticate(ctx context.Context, request *http.Request) bool { + if a == nil || a.Client == nil || strings.TrimSpace(a.ServiceAccountName) == "" { + return false + } + token, ok := strings.CutPrefix(request.Header.Get("Authorization"), "Bearer ") + if !ok || strings.TrimSpace(token) == "" { + return false + } + + review, err := a.Client.TokenReviews().Create(ctx, &authenticationv1.TokenReview{ + Spec: authenticationv1.TokenReviewSpec{Token: token, Audiences: []string{EdgeTokenAudience}}, + }, metav1.CreateOptions{}) + if err != nil || !review.Status.Authenticated || !slices.Contains(review.Status.Audiences, EdgeTokenAudience) { + return false + } + + prefix := "system:serviceaccount:" + if !strings.HasPrefix(review.Status.User.Username, prefix) { + return false + } + parts := strings.Split(strings.TrimPrefix(review.Status.User.Username, prefix), ":") + + return len(parts) == 2 && parts[1] == a.ServiceAccountName +} diff --git a/internal/metalman/netboot/edge_auth_test.go b/internal/metalman/netboot/edge_auth_test.go new file mode 100644 index 000000000..b6be81e2e --- /dev/null +++ b/internal/metalman/netboot/edge_auth_test.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "net/http/httptest" + "testing" + + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" +) + +func TestTokenReviewEdgeAuthenticatorRequiresAudienceAndServiceAccount(t *testing.T) { + t.Parallel() + + clientset := fake.NewClientset() + clientset.PrependReactor("create", "tokenreviews", func(action clienttesting.Action) (bool, runtime.Object, error) { + create := action.(clienttesting.CreateAction) + review := create.GetObject().(*authenticationv1.TokenReview) + if len(review.Spec.Audiences) != 1 || review.Spec.Audiences[0] != EdgeTokenAudience { + t.Errorf("audiences = %v", review.Spec.Audiences) + } + + return true, &authenticationv1.TokenReview{ + ObjectMeta: metav1.ObjectMeta{}, + Status: authenticationv1.TokenReviewStatus{ + Authenticated: true, + Audiences: []string{EdgeTokenAudience}, + User: authenticationv1.UserInfo{Username: "system:serviceaccount:unbounded-system:metalman-edge"}, + }, + }, nil + }) + authenticator := &TokenReviewEdgeAuthenticator{Client: clientset.AuthenticationV1(), ServiceAccountName: "metalman-edge"} + request := httptest.NewRequest("GET", "/", nil) + request.Header.Set("Authorization", "Bearer edge-token") + + if !authenticator.Authenticate(t.Context(), request) { + t.Fatal("expected edge token to authenticate") + } +} + +func TestTokenReviewEdgeAuthenticatorRejectsWrongAudience(t *testing.T) { + t.Parallel() + + clientset := fake.NewSimpleClientset() + clientset.PrependReactor("create", "tokenreviews", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, &authenticationv1.TokenReview{Status: authenticationv1.TokenReviewStatus{ + Authenticated: true, + Audiences: []string{"other"}, + User: authenticationv1.UserInfo{Username: "system:serviceaccount:unbounded-system:metalman-edge"}, + }}, nil + }) + authenticator := &TokenReviewEdgeAuthenticator{Client: clientset.AuthenticationV1(), ServiceAccountName: "metalman-edge"} + request := httptest.NewRequest("GET", "/", nil) + request.Header.Set("Authorization", "Bearer edge-token") + + if authenticator.Authenticate(t.Context(), request) { + t.Fatal("expected wrong audience to be rejected") + } +} diff --git a/internal/metalman/netboot/session_dhcp_test.go b/internal/metalman/netboot/session_dhcp_test.go new file mode 100644 index 000000000..bd24de6ac --- /dev/null +++ b/internal/metalman/netboot/session_dhcp_test.go @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/metalman/dhcp" + "github.com/Azure/unbounded/internal/metalman/netboot" +) + +func TestSessionDHCPDecisionUsesReadyImmutableSession(t *testing.T) { + t.Parallel() + + session := readyDHCPSession() + server := newSessionDHCPTestServer(t, session) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) + } + var decision dhcp.Decision + if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { + t.Fatal(err) + } + if decision.Lease.IPv4 != "10.0.1.20" { + t.Errorf("lease IP = %q", decision.Lease.IPv4) + } + if decision.Transport != v1alpha3.NetbootTransportHTTP { + t.Errorf("transport = %q", decision.Transport) + } + wantPrefix := "https://boot.example/v1/netboot/sessions/session-1/" + if len(decision.BootFile) <= len(wantPrefix) || decision.BootFile[:len(wantPrefix)] != wantPrefix { + t.Errorf("boot file = %q, want prefix %q", decision.BootFile, wantPrefix) + } +} + +func TestSessionDHCPDecisionRequiresEdgeAuthentication(t *testing.T) { + t.Parallel() + + server := newSessionDHCPTestServer(t, readyDHCPSession()) + server.EdgeAuthenticator = headerAuthenticator("Bearer edge-token") + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", response.Code, http.StatusUnauthorized) + } + + request = httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + request.Header.Set("Authorization", "Bearer edge-token") + response = httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("authenticated status = %d, want %d", response.Code, http.StatusOK) + } +} + +func TestSessionDHCPDecisionRejectsAmbiguousReadySessions(t *testing.T) { + t.Parallel() + + first := readyDHCPSession() + second := first.DeepCopy() + second.Name = "session-2" + second.UID = types.UID("session-uid-2") + server := newSessionDHCPTestServer(t, first, second) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict) + } +} + +func TestSessionDHCPDecisionOmitsBootFileForRedfishConfiguration(t *testing.T) { + t.Parallel() + + session := readyDHCPSession() + session.Spec.Boot.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + server := newSessionDHCPTestServer(t, session) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", response.Code, http.StatusOK) + } + var decision dhcp.Decision + if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { + t.Fatal(err) + } + if decision.BootFile != "" { + t.Errorf("boot file = %q, want empty", decision.BootFile) + } +} + +func TestSessionDHCPDecisionIgnoresExpiredReadySession(t *testing.T) { + t.Parallel() + + session := readyDHCPSession() + session.Spec.ExpiresAt = metav1.NewTime(time.Unix(999, 0)) + server := newSessionDHCPTestServer(t, session) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", response.Code, http.StatusNotFound) + } +} + +func newSessionDHCPTestServer(t *testing.T, sessions ...*v1alpha3.NetbootSession) *netboot.SessionHTTPServer { + t.Helper() + + scheme := runtime.NewScheme() + if err := v1alpha3.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := make([]runtime.Object, len(sessions)) + for i := range sessions { + objects[i] = sessions[i] + } + clientObjects := make([]runtime.Object, 0, len(objects)) + clientObjects = append(clientObjects, objects...) + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(clientObjects...).Build() + signer, err := netboot.NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test", func() time.Time { + return time.Unix(1000, 0) + }) + if err != nil { + t.Fatal(err) + } + + return &netboot.SessionHTTPServer{Client: client, Capabilities: signer} +} + +func readyDHCPSession() *v1alpha3.NetbootSession { + return &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: "session-1", UID: types.UID("session-uid-1")}, + Spec: v1alpha3.NetbootSessionSpec{ + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{Name: "edge", ExternalURL: "https://boot.example"}, + Boot: v1alpha3.NetbootSessionBoot{ + Transport: v1alpha3.NetbootTransportHTTP, + ConfigurationSource: v1alpha3.NetbootConfigurationSourceDHCP, + FirmwareArtifact: "shimx64.efi", + DHCPLeases: []v1alpha3.DHCPLease{{ + MAC: "aa:bb:cc:dd:ee:ff", + IPv4: "10.0.1.20", + SubnetMask: "255.255.255.0", + }}, + }, + Artifacts: v1alpha3.NetbootSessionArtifacts{Files: []v1alpha3.NetbootSessionArtifact{{Name: "shimx64.efi"}}}, + ExpiresAt: metav1.NewTime(time.Unix(2000, 0)), + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } +} + +type headerAuthenticator string + +func (h headerAuthenticator) Authenticate(_ context.Context, request *http.Request) bool { + return request.Header.Get("Authorization") == string(h) +} diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go index 67fb82605..2338b2687 100644 --- a/internal/metalman/netboot/session_http.go +++ b/internal/metalman/netboot/session_http.go @@ -5,8 +5,10 @@ package netboot import ( "context" + "encoding/json" "errors" "fmt" + "log/slog" "net/http" pathpkg "path" "strings" @@ -22,10 +24,16 @@ import ( // SessionHTTPServer serves only immutable artifacts authorized by a // session-scoped capability. type SessionHTTPServer struct { - Client client.Reader - Cache *OCICache - Capabilities *CapabilitySigner - StatusRecorder SessionConditionRecorder + Client client.Reader + Cache *OCICache + Capabilities *CapabilitySigner + StatusRecorder SessionConditionRecorder + EdgeAuthenticator EdgeAuthenticator +} + +// EdgeAuthenticator validates access to internal edge-only routes. +type EdgeAuthenticator interface { + Authenticate(ctx context.Context, request *http.Request) bool } // SessionConditionRecorder persists a milestone for an exact session identity. @@ -65,6 +73,88 @@ func (s *SessionHTTPServer) Handler() http.Handler { func (s *SessionHTTPServer) RegisterHandlers(mux *http.ServeMux) { mux.HandleFunc("GET /v1/netboot/sessions/{session}/{capability}/artifacts/{artifact...}", s.handleArtifact) mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/callbacks/{milestone}", s.handleCallback) + mux.HandleFunc("GET /v1/netboot/endpoints/{endpoint}/dhcp/{mac}", s.handleDHCPDecision) +} + +func (s *SessionHTTPServer) handleDHCPDecision(w http.ResponseWriter, r *http.Request) { + if s.Client == nil || s.Capabilities == nil { + http.Error(w, "session server unavailable", http.StatusServiceUnavailable) + return + } + if s.EdgeAuthenticator != nil && !s.EdgeAuthenticator.Authenticate(r.Context(), r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + endpoint := r.PathValue("endpoint") + mac := strings.ToLower(r.PathValue("mac")) + var sessions v1alpha3.NetbootSessionList + if err := s.Client.List(r.Context(), &sessions); err != nil { + http.Error(w, "loading sessions", http.StatusServiceUnavailable) + return + } + + matches := make([]*v1alpha3.NetbootSession, 0, 1) + for i := range sessions.Items { + session := &sessions.Items[i] + if session.Spec.Endpoint.Name != endpoint || s.Capabilities.IsExpired(session) || (session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive) { + continue + } + for _, lease := range session.Spec.Boot.DHCPLeases { + if strings.EqualFold(lease.MAC, mac) { + matches = append(matches, session) + break + } + } + } + if len(matches) == 0 { + http.NotFound(w, r) + return + } + if len(matches) != 1 { + http.Error(w, "multiple ready sessions match DHCP client", http.StatusConflict) + return + } + + session := matches[0] + var lease *v1alpha3.DHCPLease + for i := range session.Spec.Boot.DHCPLeases { + if strings.EqualFold(session.Spec.Boot.DHCPLeases[i].MAC, mac) { + lease = &session.Spec.Boot.DHCPLeases[i] + break + } + } + if lease == nil { + http.NotFound(w, r) + return + } + + bootFile := "" + if session.Spec.Boot.ConfigurationSource == v1alpha3.NetbootConfigurationSourceDHCP { + var err error + bootFile, err = SessionArtifactURL(s.Capabilities, session, session.Spec.Boot.FirmwareArtifact) + if err != nil { + http.Error(w, "building firmware URL", http.StatusServiceUnavailable) + return + } + if session.Spec.Boot.Transport == v1alpha3.NetbootTransportTFTP { + capability, err := s.Capabilities.Sign(session) + if err != nil { + http.Error(w, "building firmware capability", http.StatusServiceUnavailable) + return + } + bootFile = pathpkg.Join("v1/netboot/sessions", session.Name, capability, "artifacts", session.Spec.Boot.FirmwareArtifact) + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(struct { + Lease v1alpha3.DHCPLease `json:"lease"` + Transport v1alpha3.NetbootTransport `json:"transport"` + BootFile string `json:"bootFile"` + }{Lease: *lease, Transport: session.Spec.Boot.Transport, BootFile: bootFile}); err != nil { + slog.Warn("encoding DHCP decision", "err", err) + } } func (s *SessionHTTPServer) handleCallback(w http.ResponseWriter, r *http.Request) { From 00330ff5bc952212c8baa0dafebd1793aa3e971e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 17:24:39 +0000 Subject: [PATCH 11/43] metalman: proxy immutable TFTP artifacts --- internal/metalman/commands/edge.go | 20 ++ internal/metalman/commands/roles_test.go | 2 +- internal/metalman/netboot/tftp.go | 32 +++- internal/metalman/netboot/tftp_backend.go | 127 +++++++++++++ .../metalman/netboot/tftp_backend_test.go | 174 ++++++++++++++++++ 5 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 internal/metalman/netboot/tftp_backend.go create mode 100644 internal/metalman/netboot/tftp_backend_test.go diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go index 6d478a205..7fbea8042 100644 --- a/internal/metalman/commands/edge.go +++ b/internal/metalman/commands/edge.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "github.com/Azure/unbounded/internal/metalman/dhcp" + "github.com/Azure/unbounded/internal/metalman/netboot" ) const defaultHTTPReadHeaderTimeout = 10 * time.Second @@ -40,6 +41,9 @@ func EdgeCmd() *cobra.Command { dhcpInterface string dhcpServerIP string dhcpPort int + tftpEnabled bool + tftpBindAddr string + tftpPort int ) cmd := &cobra.Command{ @@ -77,6 +81,19 @@ func EdgeCmd() *cobra.Command { } }() } + if tftpEnabled { + artifactBackend, err := netboot.NewHTTPArtifactBackend(backend.String(), nil) + if err != nil { + return fmt.Errorf("creating TFTP backend: %w", err) + } + tftpServer := &netboot.TFTPServer{BindAddr: tftpBindAddr, Port: tftpPort, Backend: artifactBackend} + go func() { + if err := tftpServer.Start(ctx); err != nil && ctx.Err() == nil { + slog.ErrorContext(ctx, "Metalman edge TFTP server failed", "err", err) + stop() + } + }() + } go func() { <-ctx.Done() @@ -106,6 +123,9 @@ func EdgeCmd() *cobra.Command { cmd.Flags().StringVar(&dhcpInterface, "dhcp-interface", "", "Provisioning interface for direct DHCP; empty enables relay-only mode") cmd.Flags().StringVar(&dhcpServerIP, "dhcp-server-ip", "", "DHCP server IPv4 address; defaults to the interface or outbound address") cmd.Flags().IntVar(&dhcpPort, "dhcp-port", 67, "DHCP listener port") + cmd.Flags().BoolVar(&tftpEnabled, "tftp-enabled", false, "Enable the TFTP protocol edge") + cmd.Flags().StringVar(&tftpBindAddr, "tftp-bind-address", "0.0.0.0", "IP address to bind the TFTP listener") + cmd.Flags().IntVar(&tftpPort, "tftp-port", 69, "TFTP listener port") _ = cmd.MarkFlagRequired("backend-url") _ = cmd.MarkFlagRequired("endpoint") diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index 9174746b8..dbf0c8856 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -89,7 +89,7 @@ func TestEdgeCommandRequiresOnlyBackendConnection(t *testing.T) { t.Parallel() cmd := EdgeCmd() - for _, name := range []string{"backend-url", "bind-address", "http-port", "endpoint", "edge-token-file", "dhcp-enabled", "dhcp-interface", "dhcp-server-ip", "dhcp-port"} { + for _, name := range []string{"backend-url", "bind-address", "http-port", "endpoint", "edge-token-file", "dhcp-enabled", "dhcp-interface", "dhcp-server-ip", "dhcp-port", "tftp-enabled", "tftp-bind-address", "tftp-port"} { if cmd.Flags().Lookup(name) == nil { t.Errorf("edge command has no --%s flag", name) } diff --git a/internal/metalman/netboot/tftp.go b/internal/metalman/netboot/tftp.go index b268709da..e402e63b5 100644 --- a/internal/metalman/netboot/tftp.go +++ b/internal/metalman/netboot/tftp.go @@ -20,8 +20,14 @@ import ( type TFTPServer struct { BindAddr string + Port int FileResolver StatusRecorder TFTPStatusRecorder + Backend TFTPBackend +} + +type TFTPBackend interface { + Open(ctx context.Context, filename string) (io.ReadCloser, error) } type TFTPStatusRecorder interface { @@ -34,7 +40,11 @@ func (t *TFTPServer) Start(ctx context.Context) error { s := tftp.NewServer(t.readHandler, nil) s.SetAnticipate(0) - addr := net.JoinHostPort(t.BindAddr, "69") + port := t.Port + if port == 0 { + port = 69 + } + addr := net.JoinHostPort(t.BindAddr, fmt.Sprint(port)) conn, err := net.ListenPacket("udp", addr) if err != nil { @@ -56,6 +66,21 @@ func (t *TFTPServer) readHandler(filename string, rf io.ReaderFrom) error { ip := rf.(tftp.OutgoingTransfer).RemoteAddr().IP.String() //nolint:errcheck // Type is guaranteed by the tftp library. filename = strings.TrimPrefix(filename, "/") log := slog.With("proto", "tftp", "filename", filename, "ip", ip) + if t.Backend != nil { + if !validSessionArtifactPath(filename) { + return fmt.Errorf("invalid session artifact path %q", filename) + } + reader, err := t.Backend.Open(ctx, filename) + if err != nil { + return fmt.Errorf("opening backend artifact: %w", err) + } + defer reader.Close() //nolint:errcheck // Backend stream is no longer needed. + if _, err := rf.ReadFrom(reader); err != nil { + return fmt.Errorf("transferring backend artifact: %w", err) + } + + return nil + } node, err := t.LookupNodeByIP(ctx, ip) if err != nil { @@ -112,6 +137,11 @@ func (t *TFTPServer) readHandler(filename string, rf io.ReaderFrom) error { return nil } +func validSessionArtifactPath(filename string) bool { + parts := strings.Split(filename, "/") + return len(parts) >= 7 && parts[0] == "v1" && parts[1] == "netboot" && parts[2] == "sessions" && parts[3] != "" && parts[4] != "" && parts[5] == "artifacts" && parts[6] != "" +} + func (t *TFTPServer) recordBootLoaderDownloaded(ctx context.Context, log *slog.Logger, node *v1alpha3.Machine, imageRef, filename string) { if node == nil || t.StatusRecorder == nil || !t.isInitialBootLoaderDownload(imageRef, node.Spec.Netboot().TargetArchitecture(), filename) { return diff --git a/internal/metalman/netboot/tftp_backend.go b/internal/metalman/netboot/tftp_backend.go new file mode 100644 index 000000000..cda90d03e --- /dev/null +++ b/internal/metalman/netboot/tftp_backend.go @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + pathpkg "path" +) + +const maxTFTPArtifactBackendAttempts = 3 + +// HTTPArtifactBackend streams immutable session artifacts from a Metalman +// server and resumes a truncated backend response with exact byte ranges. +type HTTPArtifactBackend struct { + backendURL *url.URL + client *http.Client +} + +func NewHTTPArtifactBackend(backendURL string, client *http.Client) (*HTTPArtifactBackend, error) { + backend, err := url.Parse(backendURL) + if err != nil { + return nil, fmt.Errorf("parsing artifact backend URL: %w", err) + } + if (backend.Scheme != "http" && backend.Scheme != "https") || backend.Host == "" { + return nil, errors.New("artifact backend URL must use HTTP or HTTPS and include a host") + } + if client == nil { + client = http.DefaultClient + } + + return &HTTPArtifactBackend{backendURL: backend, client: client}, nil +} + +func (b *HTTPArtifactBackend) Open(ctx context.Context, filename string) (io.ReadCloser, error) { + requestURL := *b.backendURL + requestURL.Path = pathpkg.Join(requestURL.Path, filename) + + reader := &resumingArtifactReader{ctx: ctx, client: b.client, url: requestURL.String()} + if err := reader.open(0, -1); err != nil { + return nil, err + } + + return reader, nil +} + +type resumingArtifactReader struct { + ctx context.Context + client *http.Client + url string + body io.ReadCloser + offset int64 + end int64 + attempts int +} + +func (r *resumingArtifactReader) Read(buffer []byte) (int, error) { + for { + n, err := r.body.Read(buffer) + r.offset += int64(n) + truncated := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) + if !truncated || r.offset > r.end { + return n, err + } + if n > 0 { + return n, nil + } + if r.attempts >= maxTFTPArtifactBackendAttempts { + return 0, io.ErrUnexpectedEOF + } + if err := r.open(r.offset, r.end); err != nil { + if r.attempts >= maxTFTPArtifactBackendAttempts { + return 0, err + } + } + } +} + +func (r *resumingArtifactReader) Close() error { + if r.body == nil { + return nil + } + + return r.body.Close() +} + +func (r *resumingArtifactReader) open(start, end int64) error { + r.attempts++ + + request, err := http.NewRequestWithContext(r.ctx, http.MethodGet, r.url, nil) + if err != nil { + return err + } + if end >= 0 { + request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + } + response, err := r.client.Do(request) + if err != nil { + return fmt.Errorf("requesting TFTP artifact backend: %w", err) + } + wantStatus := http.StatusOK + if end >= 0 { + wantStatus = http.StatusPartialContent + } + if response.StatusCode != wantStatus || response.ContentLength <= 0 { + response.Body.Close() //nolint:errcheck // Invalid response. + return fmt.Errorf("TFTP artifact backend returned %s", response.Status) + } + if end < 0 { + end = response.ContentLength - 1 + } else if response.ContentLength != end-start+1 || response.Header.Get("Content-Range") != fmt.Sprintf("bytes %d-%d/%d", start, end, end+1) { + response.Body.Close() //nolint:errcheck // Invalid range response. + return errors.New("TFTP artifact backend returned a mismatched range") + } + if r.body != nil { + r.body.Close() //nolint:errcheck // Previous response reached EOF. + } + r.body = response.Body + r.end = end + + return nil +} diff --git a/internal/metalman/netboot/tftp_backend_test.go b/internal/metalman/netboot/tftp_backend_test.go new file mode 100644 index 000000000..51836fe7e --- /dev/null +++ b/internal/metalman/netboot/tftp_backend_test.go @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "bytes" + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/pin/tftp/v3" +) + +func TestTFTPServerFetchesTokenizedSessionArtifactFromBackend(t *testing.T) { + t.Parallel() + + backend := &recordingTFTPBackend{data: []byte("firmware")} + server := &TFTPServer{Backend: backend} + transfer := &memoryOutgoingTransfer{} + filename := "v1/netboot/sessions/session-1/capability/artifacts/shimx64.efi" + + if err := server.readHandler(filename, transfer); err != nil { + t.Fatal(err) + } + if backend.filename != filename { + t.Errorf("backend filename = %q, want %q", backend.filename, filename) + } + if got := transfer.String(); got != "firmware" { + t.Errorf("transfer = %q", got) + } +} + +func TestTFTPServerRejectsLegacySourceIPFilenameWithBackend(t *testing.T) { + t.Parallel() + + backend := &recordingTFTPBackend{data: []byte("firmware")} + server := &TFTPServer{Backend: backend} + if err := server.readHandler("shimx64.efi", &memoryOutgoingTransfer{}); err == nil { + t.Fatal("expected legacy filename to be rejected") + } + if backend.filename != "" { + t.Errorf("backend filename = %q, want empty", backend.filename) + } +} + +type recordingTFTPBackend struct { + filename string + data []byte +} + +func (b *recordingTFTPBackend) Open(_ context.Context, filename string) (io.ReadCloser, error) { + b.filename = filename + + return io.NopCloser(bytes.NewReader(b.data)), nil +} + +type memoryOutgoingTransfer struct { + bytes.Buffer +} + +func (m *memoryOutgoingTransfer) ReadFrom(reader io.Reader) (int64, error) { + return m.Buffer.ReadFrom(reader) +} + +func (m *memoryOutgoingTransfer) RemoteAddr() net.UDPAddr { + return net.UDPAddr{IP: net.ParseIP("10.0.1.20"), Port: 12345} +} + +func (m *memoryOutgoingTransfer) LocalIP() net.IP { + return net.ParseIP("10.0.1.254") +} + +func (m *memoryOutgoingTransfer) SetSize(int64) {} + +var _ tftp.OutgoingTransfer = (*memoryOutgoingTransfer)(nil) + +func TestHTTPArtifactBackendResumesTruncatedResponse(t *testing.T) { + t.Parallel() + + const artifact = "immutable-firmware" + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("Range = %q", got) + } + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + http.Error(w, "unexpected", http.StatusInternalServerError) + } + })) + defer server.Close() + + backend, err := NewHTTPArtifactBackend(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + reader, err := backend.Open(t.Context(), "v1/netboot/sessions/session/capability/artifacts/shimx64.efi") + if err != nil { + t.Fatal(err) + } + defer reader.Close() //nolint:errcheck // Test cleanup. + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if got := string(data); got != artifact { + t.Errorf("artifact = %q, want %q", got, artifact) + } +} + +func TestHTTPArtifactBackendRetriesFailedResumeRequest(t *testing.T) { + t.Parallel() + + const artifact = "immutable-firmware" + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijacking failed resume request: %v", err) + return + } + _ = conn.Close() + case 3: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("Range = %q", got) + } + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + http.Error(w, "unexpected", http.StatusInternalServerError) + } + })) + defer server.Close() + + backend, err := NewHTTPArtifactBackend(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + reader, err := backend.Open(t.Context(), "v1/netboot/sessions/session/capability/artifacts/shimx64.efi") + if err != nil { + t.Fatal(err) + } + defer reader.Close() //nolint:errcheck // Test cleanup. + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if got := string(data); got != artifact { + t.Errorf("artifact = %q, want %q", got, artifact) + } + if got := requests.Load(); got != 3 { + t.Errorf("backend requests = %d, want 3", got) + } +} From 4be4fe4b622b606670bcf4fc879d94cb037666d6 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 17:27:28 +0000 Subject: [PATCH 12/43] metalman: report session TFTP milestones --- internal/metalman/netboot/tftp.go | 9 ++++ internal/metalman/netboot/tftp_backend.go | 25 ++++++++++ .../metalman/netboot/tftp_backend_test.go | 49 ++++++++++++++++++- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/internal/metalman/netboot/tftp.go b/internal/metalman/netboot/tftp.go index e402e63b5..eac1a56cd 100644 --- a/internal/metalman/netboot/tftp.go +++ b/internal/metalman/netboot/tftp.go @@ -30,6 +30,10 @@ type TFTPBackend interface { Open(ctx context.Context, filename string) (io.ReadCloser, error) } +type TFTPBackendStatusRecorder interface { + RecordBootLoaderDownloaded(ctx context.Context, filename string) error +} + type TFTPStatusRecorder interface { RecordBootLoaderDownloaded(ctx context.Context, machineName, filename string) error } @@ -78,6 +82,11 @@ func (t *TFTPServer) readHandler(filename string, rf io.ReaderFrom) error { if _, err := rf.ReadFrom(reader); err != nil { return fmt.Errorf("transferring backend artifact: %w", err) } + if recorder, ok := t.Backend.(TFTPBackendStatusRecorder); ok { + if err := recorder.RecordBootLoaderDownloaded(ctx, filename); err != nil { + return fmt.Errorf("recording backend transfer: %w", err) + } + } return nil } diff --git a/internal/metalman/netboot/tftp_backend.go b/internal/metalman/netboot/tftp_backend.go index cda90d03e..0d65de583 100644 --- a/internal/metalman/netboot/tftp_backend.go +++ b/internal/metalman/netboot/tftp_backend.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" pathpkg "path" + "strings" ) const maxTFTPArtifactBackendAttempts = 3 @@ -49,6 +50,30 @@ func (b *HTTPArtifactBackend) Open(ctx context.Context, filename string) (io.Rea return reader, nil } +func (b *HTTPArtifactBackend) RecordBootLoaderDownloaded(ctx context.Context, filename string) error { + parts := strings.Split(strings.TrimPrefix(filename, "/"), "/") + if len(parts) < 7 || parts[5] != "artifacts" { + return fmt.Errorf("invalid session artifact path %q", filename) + } + + requestURL := *b.backendURL + requestURL.Path = pathpkg.Join(requestURL.Path, pathpkg.Join(parts[:5]...), "callbacks", "boot-loader-downloaded") + request, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), nil) + if err != nil { + return fmt.Errorf("creating TFTP milestone request: %w", err) + } + response, err := b.client.Do(request) + if err != nil { + return fmt.Errorf("reporting TFTP milestone: %w", err) + } + defer response.Body.Close() //nolint:errcheck // Response body is not reused. + if response.StatusCode != http.StatusNoContent { + return fmt.Errorf("reporting TFTP milestone: backend returned %s", response.Status) + } + + return nil +} + type resumingArtifactReader struct { ctx context.Context client *http.Client diff --git a/internal/metalman/netboot/tftp_backend_test.go b/internal/metalman/netboot/tftp_backend_test.go index 51836fe7e..da36a7f26 100644 --- a/internal/metalman/netboot/tftp_backend_test.go +++ b/internal/metalman/netboot/tftp_backend_test.go @@ -48,9 +48,25 @@ func TestTFTPServerRejectsLegacySourceIPFilenameWithBackend(t *testing.T) { } } +func TestTFTPServerReportsCompletedSessionTransfer(t *testing.T) { + t.Parallel() + + backend := &recordingTFTPBackend{data: []byte("firmware")} + server := &TFTPServer{Backend: backend} + filename := "v1/netboot/sessions/session-1/capability/artifacts/shimx64.efi" + + if err := server.readHandler(filename, &memoryOutgoingTransfer{}); err != nil { + t.Fatal(err) + } + if backend.completed != filename { + t.Errorf("completed filename = %q, want %q", backend.completed, filename) + } +} + type recordingTFTPBackend struct { - filename string - data []byte + filename string + completed string + data []byte } func (b *recordingTFTPBackend) Open(_ context.Context, filename string) (io.ReadCloser, error) { @@ -59,6 +75,12 @@ func (b *recordingTFTPBackend) Open(_ context.Context, filename string) (io.Read return io.NopCloser(bytes.NewReader(b.data)), nil } +func (b *recordingTFTPBackend) RecordBootLoaderDownloaded(_ context.Context, filename string) error { + b.completed = filename + + return nil +} + type memoryOutgoingTransfer struct { bytes.Buffer } @@ -172,3 +194,26 @@ func TestHTTPArtifactBackendRetriesFailedResumeRequest(t *testing.T) { t.Errorf("backend requests = %d, want 3", got) } } + +func TestHTTPArtifactBackendReportsSessionBootLoaderMilestone(t *testing.T) { + t.Parallel() + + var callbackPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callbackPath = r.URL.Path + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + backend, err := NewHTTPArtifactBackend(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + artifactPath := "v1/netboot/sessions/session/capability/artifacts/shimx64.efi" + if err := backend.RecordBootLoaderDownloaded(t.Context(), artifactPath); err != nil { + t.Fatal(err) + } + if want := "/v1/netboot/sessions/session/capability/callbacks/boot-loader-downloaded"; callbackPath != want { + t.Errorf("callback path = %q, want %q", callbackPath, want) + } +} From f2cddf14d0a7f29e2899029571176fbc96b5b39a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 17:32:51 +0000 Subject: [PATCH 13/43] metalman: support independent HTTP boot axes --- internal/metalman/machineops/controller.go | 54 ++++++---- .../metalman/machineops/controller_test.go | 98 ++++++++++++++++++- 2 files changed, 130 insertions(+), 22 deletions(-) diff --git a/internal/metalman/machineops/controller.go b/internal/metalman/machineops/controller.go index dbf51d98a..960863511 100644 --- a/internal/metalman/machineops/controller.go +++ b/internal/metalman/machineops/controller.go @@ -637,17 +637,26 @@ func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOpe } func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, machine *v1alpha3.Machine, session *v1alpha3.NetbootSession) error { - transport := machine.Spec.Netboot().TargetTransport() + netbootSpec := machine.Spec.Netboot() + transport := netbootSpec.TargetTransport() + configurationSource := targetConfigurationSource(netbootSpec) + networkMode := targetNetworkMode(netbootSpec) if session != nil { transport = session.Spec.Boot.Transport + configurationSource = session.Spec.Boot.ConfigurationSource + networkMode = session.Spec.Boot.NetworkMode } if transport == v1alpha3.NetbootTransportHTTP { + if configurationSource == v1alpha3.NetbootConfigurationSourceDHCP { + return pc.SetBootOverride(ctx, redfish.BootTargetUefiHTTP, redfish.BootOnce) + } + bootURL, staticConfig, err := r.httpBootConfig(machine, session) if err != nil { return err } - if err := setHTTPBootOverride(ctx, pc, bootURL, staticConfig); err != nil { + if err := setHTTPBootOverride(ctx, pc, bootURL, staticConfig, networkMode == v1alpha3.NetbootNetworkModeStatic); err != nil { return err } } else if err := pc.SetBootOverride(ctx, redfish.BootTargetPxe, redfish.BootContinuous); err != nil { @@ -677,9 +686,12 @@ func (r *Reconciler) httpBootConfig(machine *v1alpha3.Machine, session *v1alpha3 return "", redfish.StaticIPv4Config{}, err } - staticConfig, err := httpBootStaticNetworkConfig(machine, session) - if err != nil { - return "", redfish.StaticIPv4Config{}, err + staticConfig := redfish.StaticIPv4Config{} + if session == nil || session.Spec.Boot.NetworkMode == v1alpha3.NetbootNetworkModeStatic { + staticConfig, err = httpBootStaticNetworkConfig(machine, session) + if err != nil { + return "", redfish.StaticIPv4Config{}, err + } } return bootURL, staticConfig, nil @@ -716,22 +728,24 @@ func httpBootStaticNetworkConfig(machine *v1alpha3.Machine, session *v1alpha3.Ne return config, nil } -func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config) error { +func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config, configureStaticNetwork bool) error { config, err := pc.GetBootConfig(ctx) if err != nil { return err } if !config.HasHTTPBootURI { - return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig) + return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig, configureStaticNetwork) } - if err := pc.SetStaticIPv4(ctx, staticConfig); err != nil { - if !errors.Is(err, redfish.ErrUnsupported) { - return err - } + if configureStaticNetwork { + if err := pc.SetStaticIPv4(ctx, staticConfig); err != nil { + if !errors.Is(err, redfish.ErrUnsupported) { + return err + } - return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig) + return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig, true) + } } if err := pc.SetHTTPBootOverride(ctx, bootURL); err != nil { @@ -739,12 +753,14 @@ func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, st return err } - return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig) + return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig, configureStaticNetwork) } // Some BMCs expose both locations but boot from the vendor BIOS setting. - if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil && !errors.Is(err, redfish.ErrUnsupported) { - return err + if configureStaticNetwork { + if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil && !errors.Is(err, redfish.ErrUnsupported) { + return err + } } if err := pc.SetBIOSHTTPBootURI(ctx, bootURL); err != nil && !errors.Is(err, redfish.ErrUnsupported) { @@ -754,9 +770,11 @@ func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, st return nil } -func setBIOSHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config) error { - if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil { - return err +func setBIOSHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config, configureStaticNetwork bool) error { + if configureStaticNetwork { + if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil { + return err + } } if err := pc.SetBIOSHTTPBootURI(ctx, bootURL); err != nil { diff --git a/internal/metalman/machineops/controller_test.go b/internal/metalman/machineops/controller_test.go index 9207faf45..b06055442 100644 --- a/internal/metalman/machineops/controller_test.go +++ b/internal/metalman/machineops/controller_test.go @@ -461,6 +461,88 @@ func TestReconcilerUsesSessionCapabilityURLForHTTPBoot(t *testing.T) { require.Contains(t, power.calls, "machine-session-http:SetHTTPBootOverride:https://boot.example.com/v1/netboot/sessions/netboot-session/capability/artifacts/bootx64.efi") } +func TestConfigureRepaveBootSupportsIndependentBootAxes(t *testing.T) { + t.Parallel() + + const bootURL = "https://boot.example.com/v1/netboot/sessions/session/capability/artifacts/bootx64.efi" + tests := []struct { + name string + transport v1alpha3.NetbootTransport + configurationSource v1alpha3.NetbootConfigurationSource + networkMode v1alpha3.NetbootNetworkMode + wantCalls []string + wantURLResolution bool + }{ + { + name: "TFTP configured by DHCP", + transport: v1alpha3.NetbootTransportTFTP, + configurationSource: v1alpha3.NetbootConfigurationSourceDHCP, + networkMode: v1alpha3.NetbootNetworkModeDHCP, + wantCalls: []string{"machine:SetBootOverride:Pxe:Continuous"}, + }, + { + name: "HTTP configured by DHCP", + transport: v1alpha3.NetbootTransportHTTP, + configurationSource: v1alpha3.NetbootConfigurationSourceDHCP, + networkMode: v1alpha3.NetbootNetworkModeDHCP, + wantCalls: []string{"machine:SetBootOverride:UefiHttp:Once"}, + }, + { + name: "HTTP URL configured by Redfish with DHCP networking", + transport: v1alpha3.NetbootTransportHTTP, + configurationSource: v1alpha3.NetbootConfigurationSourceRedfish, + networkMode: v1alpha3.NetbootNetworkModeDHCP, + wantURLResolution: true, + wantCalls: []string{ + "machine:GetBootConfig", + "machine:SetHTTPBootOverride:" + bootURL, + "machine:SetBIOSHTTPBootURI:" + bootURL, + }, + }, + { + name: "HTTP URL and static network configured by Redfish", + transport: v1alpha3.NetbootTransportHTTP, + configurationSource: v1alpha3.NetbootConfigurationSourceRedfish, + networkMode: v1alpha3.NetbootNetworkModeStatic, + wantURLResolution: true, + wantCalls: []string{ + "machine:GetBootConfig", + "machine:SetStaticIPv4:aa:bb:cc:dd:ee:01:10.0.0.20:255.255.255.0:10.0.0.1:10.0.0.53", + "machine:SetHTTPBootOverride:" + bootURL, + "machine:SetBIOSStaticIPv4:10.0.0.20:255.255.255.0:10.0.0.1", + "machine:SetBIOSHTTPBootURI:" + bootURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + power := &recordingPowerClient{} + client, err := power.ForMachine(t.Context(), &v1alpha3.Machine{ObjectMeta: metav1.ObjectMeta{Name: "machine"}}) + require.NoError(t, err) + resolved := false + reconciler := &Reconciler{SessionHTTPBootURL: func(*v1alpha3.NetbootSession) (string, error) { + resolved = true + + return bootURL, nil + }} + session := &v1alpha3.NetbootSession{Spec: v1alpha3.NetbootSessionSpec{Boot: v1alpha3.NetbootSessionBoot{ + Transport: tt.transport, + ConfigurationSource: tt.configurationSource, + NetworkMode: tt.networkMode, + DHCPLeases: []v1alpha3.DHCPLease{httpBootLease()}, + }}} + + err = reconciler.configureRepaveBoot(t.Context(), client, testBareMetalMachine("machine", "rack-a"), session) + require.NoError(t, err) + require.Equal(t, tt.wantURLResolution, resolved) + require.Equal(t, tt.wantCalls, power.calls) + }) + } +} + func TestReconcilerPowersOnHostReplaceTargetWhenOff(t *testing.T) { t.Parallel() @@ -553,6 +635,8 @@ func TestReconcilerFallsBackToBIOSHTTPBootURIForHostReplace(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -594,6 +678,8 @@ func TestReconcilerUsesBIOSHTTPBootURIWhenStandardURIAbsent(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-bios", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -635,6 +721,8 @@ func TestReconcilerFallsBackToBIOSWhenStaticInterfaceIsReadOnly(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-read-only-nic", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -681,7 +769,7 @@ func TestSetHTTPBootOverrideRefreshesStandardAndBIOSURLs(t *testing.T) { DNS: []string{"10.0.0.53"}, } - require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", staticConfig)) + require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", staticConfig, true)) require.Equal(t, []string{ "machine-1:GetBootConfig", "machine-1:SetStaticIPv4:aa:bb:cc:dd:ee:01:10.0.0.20:255.255.255.0:10.0.0.1:10.0.0.53", @@ -697,12 +785,10 @@ func TestSetHTTPBootOverrideAllowsUnsupportedBIOSURL(t *testing.T) { power := &recordingPowerClient{biosHTTPBootUnsupported: map[string]bool{"machine-1": true}} client := &recordingMachinePowerClient{parent: power, machine: "machine-1"} - require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", redfish.StaticIPv4Config{})) + require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", redfish.StaticIPv4Config{}, false)) require.Equal(t, []string{ "machine-1:GetBootConfig", - "machine-1:SetStaticIPv4:::::", "machine-1:SetHTTPBootOverride:http://10.0.0.10:8880/http/shimx64.efi", - "machine-1:SetBIOSStaticIPv4:::", "machine-1:SetBIOSHTTPBootURI:http://10.0.0.10:8880/http/shimx64.efi", }, power.calls) } @@ -713,6 +799,8 @@ func TestReconcilerRetriesHTTPHostReplaceWithoutStaticLease(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic op := testOperation("op-replace-http-no-lease", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -746,6 +834,8 @@ func TestReconcilerWaitsForHTTPBootImage(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-wait-image", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name From 9d358cdec62478198b0e016cb5c2287620de8681 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 17:37:15 +0000 Subject: [PATCH 14/43] metalman: snapshot netboot rendering inputs --- .../v1alpha3/netboot_validation_test.go | 8 +- api/machina/v1alpha3/netbootsession_types.go | 35 +++ api/machina/v1alpha3/zz_generated.deepcopy.go | 49 ++++ .../unbounded-cloud.io_netbootsessions.yaml | 251 ++++++++++++++++++ internal/metalman/commands/serve_pxe.go | 4 + .../metalman/machineops/session_manager.go | 51 ++++ .../machineops/session_manager_test.go | 39 ++- 7 files changed, 432 insertions(+), 5 deletions(-) diff --git a/api/machina/v1alpha3/netboot_validation_test.go b/api/machina/v1alpha3/netboot_validation_test.go index cef7975c1..9ab5f4d04 100644 --- a/api/machina/v1alpha3/netboot_validation_test.go +++ b/api/machina/v1alpha3/netboot_validation_test.go @@ -51,7 +51,7 @@ func TestNetbootSessionSchema(t *testing.T) { } spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] - for _, field := range []string{"machine", "operation", "endpoint", "boot", "artifacts", "expiresAt"} { + for _, field := range []string{"machine", "operation", "endpoint", "boot", "provisioning", "artifacts", "expiresAt"} { if _, ok := spec.Properties[field]; !ok { t.Errorf("session spec is missing %q", field) } @@ -67,6 +67,12 @@ func TestNetbootSessionSchema(t *testing.T) { if firmware.MinLength == nil || *firmware.MinLength != 1 { t.Error("session boot firmwareArtifact must be non-empty") } + provisioning := spec.Properties["provisioning"] + for _, field := range []string{"cluster", "kubernetes", "agent", "providerLabels", "userData"} { + if _, ok := provisioning.Properties[field]; !ok { + t.Errorf("session provisioning snapshot is missing %q", field) + } + } status := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] for _, field := range []string{"phase", "conditions"} { diff --git a/api/machina/v1alpha3/netbootsession_types.go b/api/machina/v1alpha3/netbootsession_types.go index 6bdd00e3c..462cdf1e7 100644 --- a/api/machina/v1alpha3/netbootsession_types.go +++ b/api/machina/v1alpha3/netbootsession_types.go @@ -86,6 +86,10 @@ type NetbootSessionSpec struct { // Boot snapshots firmware and provisioning network configuration. Boot NetbootSessionBoot `json:"boot"` + // Provisioning snapshots inputs used to render installer and first-boot + // configuration. + Provisioning NetbootSessionProvisioning `json:"provisioning"` + // Artifacts identifies immutable OCI sources and files for this session. Artifacts NetbootSessionArtifacts `json:"artifacts"` @@ -144,6 +148,37 @@ type NetbootSessionBoot struct { TargetDisk string `json:"targetDisk,omitempty"` } +// NetbootSessionProvisioning snapshots installer and first-boot inputs. +type NetbootSessionProvisioning struct { + Cluster NetbootSessionCluster `json:"cluster"` + + // Kubernetes contains the target's immutable kubelet configuration. + // +optional + Kubernetes *KubernetesSpec `json:"kubernetes,omitempty"` + + // Agent contains the immutable agent installation configuration. + // +optional + Agent *AgentSpec `json:"agent,omitempty"` + + // ProviderLabels are merged into the rendered kubelet labels. + // +optional + ProviderLabels map[string]string `json:"providerLabels,omitempty"` + + // UserData is the resolved cloud-init user-data content. + UserData string `json:"userData"` +} + +// NetbootSessionCluster snapshots cluster connection inputs used by the agent. +type NetbootSessionCluster struct { + APIServerURL string `json:"apiServerURL"` + CACertBase64 string `json:"caCertBase64"` + DNS string `json:"dns"` + + // KubernetesVersion is the cluster version used when the Machine does not + // specify one. + KubernetesVersion string `json:"kubernetesVersion"` +} + // NetbootSessionArtifacts snapshots immutable OCI sources and artifact paths. type NetbootSessionArtifacts struct { MachineImage NetbootSessionImage `json:"machineImage"` diff --git a/api/machina/v1alpha3/zz_generated.deepcopy.go b/api/machina/v1alpha3/zz_generated.deepcopy.go index d9738b74b..0630c6e96 100644 --- a/api/machina/v1alpha3/zz_generated.deepcopy.go +++ b/api/machina/v1alpha3/zz_generated.deepcopy.go @@ -1432,6 +1432,21 @@ func (in *NetbootSessionBoot) DeepCopy() *NetbootSessionBoot { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionCluster) DeepCopyInto(out *NetbootSessionCluster) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionCluster. +func (in *NetbootSessionCluster) DeepCopy() *NetbootSessionCluster { + if in == nil { + return nil + } + out := new(NetbootSessionCluster) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetbootSessionEndpointSnapshot) DeepCopyInto(out *NetbootSessionEndpointSnapshot) { *out = *in @@ -1514,6 +1529,39 @@ func (in *NetbootSessionObjectSnapshot) DeepCopy() *NetbootSessionObjectSnapshot return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionProvisioning) DeepCopyInto(out *NetbootSessionProvisioning) { + *out = *in + out.Cluster = in.Cluster + if in.Kubernetes != nil { + in, out := &in.Kubernetes, &out.Kubernetes + *out = new(KubernetesSpec) + (*in).DeepCopyInto(*out) + } + if in.Agent != nil { + in, out := &in.Agent, &out.Agent + *out = new(AgentSpec) + (*in).DeepCopyInto(*out) + } + if in.ProviderLabels != nil { + in, out := &in.ProviderLabels, &out.ProviderLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionProvisioning. +func (in *NetbootSessionProvisioning) DeepCopy() *NetbootSessionProvisioning { + if in == nil { + return nil + } + out := new(NetbootSessionProvisioning) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetbootSessionReference) DeepCopyInto(out *NetbootSessionReference) { *out = *in @@ -1536,6 +1584,7 @@ func (in *NetbootSessionSpec) DeepCopyInto(out *NetbootSessionSpec) { out.Operation = in.Operation out.Endpoint = in.Endpoint in.Boot.DeepCopyInto(&out.Boot) + in.Provisioning.DeepCopyInto(&out.Provisioning) in.Artifacts.DeepCopyInto(&out.Artifacts) in.ExpiresAt.DeepCopyInto(&out.ExpiresAt) } diff --git a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml index 4f0c1404b..aa5fef35d 100644 --- a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml @@ -319,6 +319,256 @@ spec: - name - uid type: object + provisioning: + description: |- + Provisioning snapshots inputs used to render installer and first-boot + configuration. + properties: + agent: + description: Agent contains the immutable agent installation configuration. + properties: + baseURL: + description: |- + BaseURL overrides the base URL used to construct the + unbounded-agent download URL. Defaults to the upstream GitHub + releases URL. The layout under BaseURL must match the GitHub + releases layout (/latest/download/ and + /download//). + type: string + downloads: + description: |- + Downloads overrides the download sources for the binaries the + agent installs into the nspawn rootfs (kubelet, containerd, runc, + CNI plugins, crictl). When unset the agent downloads each + artifact from its upstream default host. + properties: + cni: + description: |- + CNI overrides the download source for CNI plugins + (upstream default: https://github.com/containernetworking/plugins). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + containerd: + description: |- + Containerd overrides the download source for containerd + (upstream default: https://github.com/containerd/containerd). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + crictl: + description: |- + Crictl overrides the download source for crictl + (upstream default: https://github.com/kubernetes-sigs/cri-tools). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + kubernetes: + description: |- + Kubernetes overrides the download source for kubelet/kubectl/kube-proxy + (upstream default: https://dl.k8s.io). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + runc: + description: |- + Runc overrides the download source for runc + (upstream default: https://github.com/opencontainers/runc). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + type: object + image: + description: |- + Image is the OCI image reference used for provisioning the + nspawn machine (e.g. "ghcr.io/org/repo:tag"). When empty the + agent falls back to its built-in default image. + type: string + url: + description: |- + URL is a fully qualified download URL for the unbounded-agent + tarball. When set it overrides Version and BaseURL entirely. + type: string + version: + description: |- + Version pins the unbounded-agent release tag that is downloaded + onto the host (e.g. "v0.0.10"). When empty the install script + tracks the latest published release. + type: string + type: object + cluster: + description: NetbootSessionCluster snapshots cluster connection + inputs used by the agent. + properties: + apiServerURL: + type: string + caCertBase64: + type: string + dns: + type: string + kubernetesVersion: + description: |- + KubernetesVersion is the cluster version used when the Machine does not + specify one. + type: string + required: + - apiServerURL + - caCertBase64 + - dns + - kubernetesVersion + type: object + kubernetes: + description: Kubernetes contains the target's immutable kubelet + configuration. + properties: + bootstrapTokenRef: + description: |- + BootstrapTokenRef references a bootstrap token Secret in + kube-system. The secret must be of type + bootstrap.kubernetes.io/token with the well-known keys + "token-id" and "token-secret". + properties: + name: + description: Name of the referenced resource. + minLength: 1 + type: string + required: + - name + type: object + nodeLabels: + additionalProperties: + type: string + description: NodeLabels are labels passed to kubelet's --node-labels + flag. + type: object + nodeRef: + description: NodeRef references the Node that corresponds + to this Machine. + properties: + name: + description: Name of the referenced resource. + minLength: 1 + type: string + required: + - name + type: object + registerWithTaints: + description: |- + RegisterWithTaints are taints passed to kubelet's --register-with-taints flag. + Each entry uses the standard Kubernetes taint format: key=value:Effect. + items: + type: string + type: array + version: + description: |- + Version is the Kubernetes version to install (e.g., "v1.34.0"). + When omitted the controller falls back to the cluster's + Kubernetes version. + type: string + type: object + providerLabels: + additionalProperties: + type: string + description: ProviderLabels are merged into the rendered kubelet + labels. + type: object + userData: + description: UserData is the resolved cloud-init user-data content. + type: string + required: + - cluster + - userData + type: object required: - artifacts - boot @@ -326,6 +576,7 @@ spec: - expiresAt - machine - operation + - provisioning type: object x-kubernetes-validations: - message: netboot session spec is immutable diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index 3e8017098..5ebc51022 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -231,6 +231,10 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { Cache: ociCache, DefaultNetbootRef: defaultNetbootImage, DefaultNetbootPullSecret: defaultNetbootPullSecretRef, + Cluster: clusterInfoWatcher, + KubernetesVersion: kubeVersion, + ClusterDNS: clusterDNS, + ProviderLabels: providerLabels, } var statusQueue *metalmachineops.StatusQueue diff --git a/internal/metalman/machineops/session_manager.go b/internal/metalman/machineops/session_manager.go index d6f4339af..809178a3c 100644 --- a/internal/metalman/machineops/session_manager.go +++ b/internal/metalman/machineops/session_manager.go @@ -7,9 +7,11 @@ import ( "context" "errors" "fmt" + "maps" "strings" "time" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,6 +34,10 @@ type KubernetesSessionManager struct { Cache *netboot.OCICache DefaultNetbootRef string DefaultNetbootPullSecret *v1alpha3.NamespacedSecretReference + Cluster netboot.ClusterInfoProvider + KubernetesVersion string + ClusterDNS string + ProviderLabels map[string]string Now func() metav1.Time } @@ -96,6 +102,14 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph } now := m.now() + userData, err := m.resolveUserData(ctx, netbootSpec) + if err != nil { + return nil, err + } + clusterInfo := netboot.ClusterInfo{} + if m.Cluster != nil { + clusterInfo = m.Cluster.ClusterInfo() + } session := &v1alpha3.NetbootSession{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -121,6 +135,18 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph DHCPLeases: append([]v1alpha3.DHCPLease(nil), netbootSpec.DHCPLeases...), TargetDisk: netbootSpec.TargetDisk, }, + Provisioning: v1alpha3.NetbootSessionProvisioning{ + Cluster: v1alpha3.NetbootSessionCluster{ + APIServerURL: clusterInfo.ApiserverURL, + CACertBase64: clusterInfo.CACertBase64, + DNS: m.ClusterDNS, + KubernetesVersion: m.KubernetesVersion, + }, + Kubernetes: machine.Spec.Kubernetes.DeepCopy(), + Agent: machine.Spec.Agent.DeepCopy(), + ProviderLabels: maps.Clone(m.ProviderLabels), + UserData: userData, + }, Artifacts: v1alpha3.NetbootSessionArtifacts{ MachineImage: v1alpha3.NetbootSessionImage{ Reference: netbootSpec.Image, @@ -153,6 +179,31 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph return m.refreshEndpointReadinessWithEndpoint(ctx, session, &endpoint) } +func (m *KubernetesSessionManager) resolveUserData(ctx context.Context, spec *v1alpha3.PXESpec) (string, error) { + if spec.CloudInit == nil || spec.CloudInit.UserDataConfigMapRef == nil { + return "#cloud-config\n", nil + } + + ref := spec.CloudInit.UserDataConfigMapRef + var configMap corev1.ConfigMap + if err := m.Client.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, &configMap); err != nil { + return "", fmt.Errorf("get cloud-init user-data ConfigMap %s/%s: %w", ref.Namespace, ref.Name, err) + } + + key := ref.Key + if key == "" { + key = "user-data" + } + if value, ok := configMap.Data[key]; ok { + return value, nil + } + if value, ok := configMap.BinaryData[key]; ok { + return string(value), nil + } + + return "", fmt.Errorf("cloud-init user-data key %q not found in ConfigMap %s/%s", key, ref.Namespace, ref.Name) +} + func (m *KubernetesSessionManager) refreshEndpointReadiness(ctx context.Context, session *v1alpha3.NetbootSession) (*v1alpha3.NetbootSession, error) { var endpoint v1alpha3.NetbootEndpoint if err := m.Client.Get(ctx, client.ObjectKey{Name: session.Spec.Endpoint.Name}, &endpoint); err != nil { diff --git a/internal/metalman/machineops/session_manager_test.go b/internal/metalman/machineops/session_manager_test.go index ae1b4d130..ccede976a 100644 --- a/internal/metalman/machineops/session_manager_test.go +++ b/internal/metalman/machineops/session_manager_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -27,6 +28,15 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { machine.Spec.PXE.EndpointRef = "rack-a-edge" machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.Kubernetes = &v1alpha3.KubernetesSpec{ + Version: "v1.35.0", + NodeLabels: map[string]string{"user-label": "original"}, + RegisterWithTaints: []string{"dedicated=metal:NoSchedule"}, + } + machine.Spec.Agent = &v1alpha3.AgentSpec{Image: "ghcr.io/test/agent:v1", Version: "v1.2.3"} + machine.Spec.PXE.CloudInit = &v1alpha3.CloudInitSpec{UserDataConfigMapRef: &v1alpha3.ConfigMapKeySelector{ + Name: "machine-session-user-data", Namespace: "default", + }} op := testOperation("replace-session", v1alpha3.OperationHostReplace) op.UID = "operation-uid" op.Generation = 2 @@ -40,11 +50,18 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { require.NoError(t, writeSessionMetadata(cache, netbootDigest, "http/bootx64.efi")) s := testScheme(t) - c := fake.NewClientBuilder().WithScheme(s).WithObjects(endpoint).WithStatusSubresource(&v1alpha3.NetbootSession{}).Build() + c := fake.NewClientBuilder().WithScheme(s).WithObjects(endpoint, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "machine-session-user-data", Namespace: "default"}, + Data: map[string]string{"user-data": "#cloud-config\nhostname: original\n"}, + }).WithStatusSubresource(&v1alpha3.NetbootSession{}).Build() manager := &KubernetesSessionManager{ - Client: c, - Cache: cache, - Now: func() metav1.Time { return fixedNow() }, + Client: c, + Cache: cache, + Cluster: &netboot.StaticClusterInfo{Info: netboot.ClusterInfo{ApiserverURL: "https://api.example.com:6443", CACertBase64: "cluster-ca"}}, + KubernetesVersion: "v1.34.0", + ClusterDNS: "10.96.0.10", + ProviderLabels: map[string]string{"provider-label": "original"}, + Now: func() metav1.Time { return fixedNow() }, } session, err := manager.Ensure(t.Context(), op, machine) @@ -56,6 +73,14 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { require.Equal(t, op.UID, session.Spec.Operation.UID) require.Equal(t, endpoint.Spec.ExternalURL, session.Spec.Endpoint.ExternalURL) require.Equal(t, "http/bootx64.efi", session.Spec.Boot.FirmwareArtifact) + require.Equal(t, "https://api.example.com:6443", session.Spec.Provisioning.Cluster.APIServerURL) + require.Equal(t, "cluster-ca", session.Spec.Provisioning.Cluster.CACertBase64) + require.Equal(t, "10.96.0.10", session.Spec.Provisioning.Cluster.DNS) + require.Equal(t, "v1.34.0", session.Spec.Provisioning.Cluster.KubernetesVersion) + require.Equal(t, "original", session.Spec.Provisioning.Kubernetes.NodeLabels["user-label"]) + require.Equal(t, "ghcr.io/test/agent:v1", session.Spec.Provisioning.Agent.Image) + require.Equal(t, "original", session.Spec.Provisioning.ProviderLabels["provider-label"]) + require.Equal(t, "#cloud-config\nhostname: original\n", session.Spec.Provisioning.UserData) require.Contains(t, session.Spec.Artifacts.Files, v1alpha3.NetbootSessionArtifact{ Name: "http/bootx64.efi", Source: "NetbootImage", @@ -64,10 +89,16 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { require.True(t, session.Spec.ExpiresAt.Time.Equal(fixedNow().Add(24*time.Hour))) machine.Spec.PXE.Image = "ghcr.io/test/changed:v2" + machine.Spec.Kubernetes.NodeLabels["user-label"] = "changed" + machine.Spec.Agent.Image = "ghcr.io/test/agent:changed" + manager.ProviderLabels["provider-label"] = "changed" reused, err := manager.Ensure(t.Context(), op, machine) require.NoError(t, err) require.Equal(t, session.Name, reused.Name) require.Equal(t, "ghcr.io/test/host:v1", reused.Spec.Artifacts.MachineImage.Reference) + require.Equal(t, "original", reused.Spec.Provisioning.Kubernetes.NodeLabels["user-label"]) + require.Equal(t, "ghcr.io/test/agent:v1", reused.Spec.Provisioning.Agent.Image) + require.Equal(t, "original", reused.Spec.Provisioning.ProviderLabels["provider-label"]) var sessions v1alpha3.NetbootSessionList require.NoError(t, c.List(t.Context(), &sessions, client.MatchingLabels{sessionOperationUIDLabel: string(op.UID)})) From 150890ee40e817b4370e6f7c2b522f4994a402eb Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 17:48:52 +0000 Subject: [PATCH 15/43] metalman: render capability-scoped boot artifacts --- .../v1alpha3/netboot_validation_test.go | 15 ++ api/machina/v1alpha3/netbootsession_types.go | 5 +- .../unbounded-cloud.io_netbootsessions.yaml | 6 +- images/netboot/assets/grub.cfg.tmpl | 9 +- images/netboot/assets/init | 9 +- images/netboot/assets/vendor-data.tmpl | 4 +- .../metalman/machineops/session_manager.go | 27 ++- .../machineops/session_manager_test.go | 14 ++ internal/metalman/netboot/netboot.go | 56 ++++-- internal/metalman/netboot/session_http.go | 161 +++++++++++++++- .../metalman/netboot/session_http_test.go | 177 ++++++++++++++++++ 11 files changed, 449 insertions(+), 34 deletions(-) diff --git a/api/machina/v1alpha3/netboot_validation_test.go b/api/machina/v1alpha3/netboot_validation_test.go index 9ab5f4d04..dea7545a8 100644 --- a/api/machina/v1alpha3/netboot_validation_test.go +++ b/api/machina/v1alpha3/netboot_validation_test.go @@ -73,6 +73,8 @@ func TestNetbootSessionSchema(t *testing.T) { t.Errorf("session provisioning snapshot is missing %q", field) } } + artifactSource := spec.Properties["artifacts"].Properties["files"].Items.Schema.Properties["source"] + requireEnumValue(t, artifactSource.Enum, "Session") status := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] for _, field := range []string{"phase", "conditions"} { @@ -82,6 +84,19 @@ func TestNetbootSessionSchema(t *testing.T) { } } +func requireEnumValue(t *testing.T, values []apiextensionsv1.JSON, want string) { + t.Helper() + + for _, value := range values { + var got string + if err := yaml.Unmarshal(value.Raw, &got); err == nil && got == want { + return + } + } + + t.Errorf("enum does not contain %q", want) +} + func TestMachineOperationTargetInputHasNetbootSessionRef(t *testing.T) { t.Parallel() diff --git a/api/machina/v1alpha3/netbootsession_types.go b/api/machina/v1alpha3/netbootsession_types.go index 462cdf1e7..416c8911b 100644 --- a/api/machina/v1alpha3/netbootsession_types.go +++ b/api/machina/v1alpha3/netbootsession_types.go @@ -212,8 +212,9 @@ type NetbootSessionArtifact struct { // +kubebuilder:validation:MinLength=1 Name string `json:"name"` - // Source selects the OCI image containing this file. - // +kubebuilder:validation:Enum=MachineImage;NetbootImage + // Source selects the OCI image containing this file, or Session for + // content snapshotted directly into the session. + // +kubebuilder:validation:Enum=MachineImage;NetbootImage;Session Source string `json:"source"` // Path is the absolute path within the unpacked OCI image. diff --git a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml index aa5fef35d..697e42dfb 100644 --- a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml @@ -89,11 +89,13 @@ spec: minimum: 0 type: integer source: - description: Source selects the OCI image containing this - file. + description: |- + Source selects the OCI image containing this file, or Session for + content snapshotted directly into the session. enum: - MachineImage - NetbootImage + - Session type: string required: - name diff --git a/images/netboot/assets/grub.cfg.tmpl b/images/netboot/assets/grub.cfg.tmpl index 1af1b9994..d05c33e29 100644 --- a/images/netboot/assets/grub.cfg.tmpl +++ b/images/netboot/assets/grub.cfg.tmpl @@ -27,10 +27,11 @@ set default=0 set timeout=0 menuentry "Unbounded Metal Install" { - linux /vmlinuz \ - unbounded.image_url={{ .ServeURL }}/disk.img.gz \ + linux {{ .ArtifactBaseURL }}/vmlinuz \ + unbounded.image_url={{ .ArtifactBaseURL }}/disk.img.gz \ unbounded.serve_url={{ .ServeURL }} \ - unbounded.ds_url={{ .ServeURL }}/cloud-init/ \ + unbounded.boot_image_written_url={{ .BootImageWrittenURL }} \ + unbounded.ds_url={{ .ArtifactBaseURL }}/cloud-init/ \ unbounded.node_name={{ .Machine.Name }} \ unbounded.node_namespace={{ .Machine.Namespace }} \ unbounded.apiserver_url={{ .ApiserverURL }} \ @@ -46,5 +47,5 @@ menuentry "Unbounded Metal Install" { {{- end }} console=tty0 console=ttyS0,115200n8 \ --- - initrd /initrd /init.cpio + initrd {{ .ArtifactBaseURL }}/initrd {{ .ArtifactBaseURL }}/init.cpio } diff --git a/images/netboot/assets/init b/images/netboot/assets/init index 53829033e..6aafee75a 100644 --- a/images/netboot/assets/init +++ b/images/netboot/assets/init @@ -121,6 +121,7 @@ done IMAGE_URL=$(get_param unbounded.image_url) || fatal "unbounded.image_url not set" SERVE_URL=$(get_param unbounded.serve_url || true) +BOOT_IMAGE_WRITTEN_URL=$(get_param unbounded.boot_image_written_url || true) TARGET_DISK=$(get_param unbounded.disk || true) BOOT_MAC=$(get_param unbounded.boot_mac || true) BOOTIF=$(get_param BOOTIF || true) @@ -416,10 +417,10 @@ if [ -d /sys/firmware/efi ]; then done fi -if [ -n "$SERVE_URL" ]; then - log "disabling PXE boot" - retry 5 2 "disable PXE" wget -q -O /dev/null "$SERVE_URL/pxe/disable" \ - || log "WARNING: failed to disable PXE boot" +if [ -n "$BOOT_IMAGE_WRITTEN_URL" ]; then + log "reporting completed disk installation" + retry 5 2 "report completed disk installation" wget -q -O /dev/null --post-data='' "$BOOT_IMAGE_WRITTEN_URL" \ + || log "WARNING: failed to report completed disk installation" fi log "installation complete, rebooting" diff --git a/images/netboot/assets/vendor-data.tmpl b/images/netboot/assets/vendor-data.tmpl index 94a35ad01..9fb032b3e 100644 --- a/images/netboot/assets/vendor-data.tmpl +++ b/images/netboot/assets/vendor-data.tmpl @@ -16,7 +16,7 @@ ssh_pwauth: false reporting: unbounded: type: webhook - endpoint: {{ .ServeURL }}/cloudinit/log + endpoint: {{ .CloudInitURL }} write_files: - path: /etc/unbounded/agent/config.json permissions: '0600' @@ -67,7 +67,7 @@ runcmd: else echo "could not read ${install_log}: file does not exist" fi - } | curl -fsS -m 10 -X POST -H "Content-Type: text/plain" --data-binary @- "{{ .ServeURL }}/unbounded-agent/install-log" >/dev/null 2>&1 || true + } | curl -fsS -m 10 -X POST -H "Content-Type: text/plain" --data-binary @- "{{ .InstallLogURL }}" >/dev/null 2>&1 || true exit "$status" } trap report_unbounded_failure ERR diff --git a/internal/metalman/machineops/session_manager.go b/internal/metalman/machineops/session_manager.go index 809178a3c..c1281e0cd 100644 --- a/internal/metalman/machineops/session_manager.go +++ b/internal/metalman/machineops/session_manager.go @@ -283,11 +283,36 @@ func targetNetworkMode(spec *v1alpha3.PXESpec) v1alpha3.NetbootNetworkMode { } func sessionArtifacts(firmwareArtifact string) []v1alpha3.NetbootSessionArtifact { - return []v1alpha3.NetbootSessionArtifact{ + artifacts := []v1alpha3.NetbootSessionArtifact{ {Name: "disk.img.gz", Source: "MachineImage", Path: "/disk/disk.img.gz"}, {Name: "metadata.yaml", Source: "NetbootImage", Path: "/disk/metadata.yaml"}, {Name: firmwareArtifact, Source: "NetbootImage", Path: "/disk/" + firmwareArtifact}, + {Name: "vmlinuz", Source: "NetbootImage", Path: "/disk/vmlinuz"}, + {Name: "initrd", Source: "NetbootImage", Path: "/disk/initrd"}, + {Name: "init.cpio", Source: "NetbootImage", Path: "/disk/init.cpio"}, + {Name: "grub/grub.cfg", Source: "NetbootImage", Path: "/disk/grub/grub.cfg"}, + {Name: "cloud-init/meta-data", Source: "NetbootImage", Path: "/disk/cloud-init/meta-data"}, + {Name: "cloud-init/user-data", Source: "Session", Path: "/session/cloud-init/user-data"}, + {Name: "cloud-init/vendor-data", Source: "NetbootImage", Path: "/disk/cloud-init/vendor-data"}, + {Name: "cloud-init/network-config", Source: "NetbootImage", Path: "/disk/cloud-init/network-config"}, } + if strings.Contains(firmwareArtifact, "aa64") { + artifacts = append(artifacts, v1alpha3.NetbootSessionArtifact{Name: "grubaa64.efi", Source: "NetbootImage", Path: "/disk/grubaa64.efi"}) + } else { + artifacts = append(artifacts, v1alpha3.NetbootSessionArtifact{Name: "grubx64.efi", Source: "NetbootImage", Path: "/disk/grubx64.efi"}) + } + + unique := make([]v1alpha3.NetbootSessionArtifact, 0, len(artifacts)) + names := make(map[string]struct{}, len(artifacts)) + for _, artifact := range artifacts { + if _, exists := names[artifact.Name]; exists { + continue + } + names[artifact.Name] = struct{}{} + unique = append(unique, artifact) + } + + return unique } func firmwareArtifactForTransport(metadata *netboot.ImageMetadata, transport v1alpha3.NetbootTransport) string { diff --git a/internal/metalman/machineops/session_manager_test.go b/internal/metalman/machineops/session_manager_test.go index ccede976a..06db36c40 100644 --- a/internal/metalman/machineops/session_manager_test.go +++ b/internal/metalman/machineops/session_manager_test.go @@ -86,6 +86,20 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { Source: "NetbootImage", Path: "/disk/http/bootx64.efi", }) + for _, name := range []string{ + "vmlinuz", "initrd", "init.cpio", "grub/grub.cfg", "grubx64.efi", + "cloud-init/meta-data", "cloud-init/user-data", "cloud-init/vendor-data", "cloud-init/network-config", + } { + require.Condition(t, func() bool { + for _, artifact := range session.Spec.Artifacts.Files { + if artifact.Name == name { + return true + } + } + + return false + }, "session artifact list is missing %s", name) + } require.True(t, session.Spec.ExpiresAt.Time.Equal(fixedNow().Add(24*time.Hour))) machine.Spec.PXE.Image = "ghcr.io/test/changed:v2" diff --git a/internal/metalman/netboot/netboot.go b/internal/metalman/netboot/netboot.go index bb504552e..4621920e7 100644 --- a/internal/metalman/netboot/netboot.go +++ b/internal/metalman/netboot/netboot.go @@ -303,14 +303,18 @@ func (f *FileResolver) resolveUserDataFromConfigMap(ctx context.Context, node *v } type templateData struct { - Machine *v1alpha3.Machine - BootLease *v1alpha3.DHCPLease - ApiserverURL string - ServeURL string - AgentConfigJSON string - InstallScript string - InstallEnv []string - InstallRequested bool + Machine *v1alpha3.Machine + BootLease *v1alpha3.DHCPLease + ApiserverURL string + ServeURL string + ArtifactBaseURL string + BootImageWrittenURL string + CloudInitURL string + InstallLogURL string + AgentConfigJSON string + InstallScript string + InstallEnv []string + InstallRequested bool } func newTemplateData(node *v1alpha3.Machine, ci ClusterInfo, serveURL, agentConfigJSON, requestIP string, installRequested bool) templateData { @@ -324,15 +328,21 @@ func newTemplateData(node *v1alpha3.Machine, ci ClusterInfo, serveURL, agentConf bootLease = selectBootLease(node, requestIP) } + serveURL = strings.TrimRight(serveURL, "/") + return templateData{ - Machine: node, - BootLease: bootLease, - ApiserverURL: ci.ApiserverURL, - ServeURL: serveURL, - AgentConfigJSON: agentConfigJSON, - InstallScript: provision.UnboundedAgentInstallScript(), - InstallEnv: provision.AgentInstallEnv(agent), - InstallRequested: installRequested, + Machine: node, + BootLease: bootLease, + ApiserverURL: ci.ApiserverURL, + ServeURL: serveURL, + ArtifactBaseURL: serveURL, + BootImageWrittenURL: serveURL + "/pxe/disable", + CloudInitURL: serveURL + "/cloudinit/log", + InstallLogURL: serveURL + "/unbounded-agent/install-log", + AgentConfigJSON: agentConfigJSON, + InstallScript: provision.UnboundedAgentInstallScript(), + InstallEnv: provision.AgentInstallEnv(agent), + InstallRequested: installRequested, } } @@ -440,6 +450,20 @@ func indentTemplateBlock(spaces int, value string) string { } func renderTemplate(tmplStr string, data templateData) ([]byte, error) { + serveURL := strings.TrimRight(data.ServeURL, "/") + if data.ArtifactBaseURL == "" { + data.ArtifactBaseURL = serveURL + } + if data.BootImageWrittenURL == "" { + data.BootImageWrittenURL = serveURL + "/pxe/disable" + } + if data.CloudInitURL == "" { + data.CloudInitURL = serveURL + "/cloudinit/log" + } + if data.InstallLogURL == "" { + data.InstallLogURL = serveURL + "/unbounded-agent/install-log" + } + t, err := template.New("").Funcs(templateFuncMap).Parse(tmplStr) if err != nil { return nil, fmt.Errorf("parsing template: %w", err) diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go index 2338b2687..8aa95e9bd 100644 --- a/internal/metalman/netboot/session_http.go +++ b/internal/metalman/netboot/session_http.go @@ -4,12 +4,15 @@ package netboot import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "log/slog" "net/http" + "os" pathpkg "path" "strings" @@ -19,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/provision" ) // SessionHTTPServer serves only immutable artifacts authorized by a @@ -54,12 +58,26 @@ func SessionArtifactURL(signer *CapabilitySigner, session *v1alpha3.NetbootSessi if cleanArtifact == "" || pathpkg.Clean(cleanArtifact) != cleanArtifact || strings.HasPrefix(artifactName, "/") { return "", fmt.Errorf("invalid artifact name %q", artifactName) } + baseURL, err := SessionBaseURL(signer, session) + if err != nil { + return "", err + } + + return JoinServeURLPath(baseURL, pathpkg.Join("artifacts", cleanArtifact)) +} + +// SessionBaseURL returns the externally advertised capability root for a +// session's artifacts and callbacks. +func SessionBaseURL(signer *CapabilitySigner, session *v1alpha3.NetbootSession) (string, error) { + if signer == nil || session == nil { + return "", errors.New("capability signer and session are required") + } capability, err := signer.Sign(session) if err != nil { return "", err } - return JoinServeURLPath(session.Spec.Endpoint.ExternalURL, pathpkg.Join("v1/netboot/sessions", session.Name, capability, "artifacts", cleanArtifact)) + return JoinServeURLPath(session.Spec.Endpoint.ExternalURL, pathpkg.Join("v1/netboot/sessions", session.Name, capability)) } func (s *SessionHTTPServer) Handler() http.Handler { @@ -73,6 +91,7 @@ func (s *SessionHTTPServer) Handler() http.Handler { func (s *SessionHTTPServer) RegisterHandlers(mux *http.ServeMux) { mux.HandleFunc("GET /v1/netboot/sessions/{session}/{capability}/artifacts/{artifact...}", s.handleArtifact) mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/callbacks/{milestone}", s.handleCallback) + mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/logs/agent-install", s.handleInstallLog) mux.HandleFunc("GET /v1/netboot/endpoints/{endpoint}/dhcp/{mac}", s.handleDHCPDecision) } @@ -167,7 +186,13 @@ func (s *SessionHTTPServer) handleCallback(w http.ResponseWriter, r *http.Reques return } - conditionType, ok := sessionMilestoneCondition(r.PathValue("milestone")) + milestone := r.PathValue("milestone") + if milestone == "cloud-init" { + s.handleSessionCloudInit(w, r, session) + return + } + + conditionType, ok := sessionMilestoneCondition(milestone) if !ok { http.NotFound(w, r) return @@ -185,6 +210,42 @@ func (s *SessionHTTPServer) handleCallback(w http.ResponseWriter, r *http.Reques w.WriteHeader(http.StatusNoContent) } +func (s *SessionHTTPServer) handleSessionCloudInit(w http.ResponseWriter, r *http.Request, session *v1alpha3.NetbootSession) { + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, "reading cloud-init event", http.StatusBadRequest) + return + } + var event cloudInitEvent + if err := json.Unmarshal(body, &event); err != nil { + http.Error(w, "invalid cloud-init event", http.StatusBadRequest) + return + } + condition := buildCloudInitCondition(&event, session.Spec.Machine.Generation) + if condition != nil { + condition.Type = v1alpha3.NetbootSessionConditionCloudInitDone + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, *condition); err != nil { + http.Error(w, "recording session status", http.StatusServiceUnavailable) + return + } + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *SessionHTTPServer) handleInstallLog(w http.ResponseWriter, r *http.Request) { + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, "reading install log", http.StatusBadRequest) + return + } + slog.Warn("unbounded-agent install log", "session", session.Name, "body", strings.TrimSpace(string(body))) + w.WriteHeader(http.StatusNoContent) +} + func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Request) { if s.Cache == nil { http.Error(w, "session server unavailable", http.StatusServiceUnavailable) @@ -201,6 +262,14 @@ func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Reques http.NotFound(w, r) return } + if artifact.Source == "Session" { + if artifact.Name != "cloud-init/user-data" { + http.NotFound(w, r) + return + } + http.ServeContent(w, r, artifact.Name, session.CreationTimestamp.Time, strings.NewReader(session.Spec.Provisioning.UserData)) + return + } image, ok := sessionArtifactImage(session, artifact.Source) if !ok { http.NotFound(w, r) @@ -223,11 +292,97 @@ func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Reques return } if isTemplate { - http.Error(w, "rendered artifact unavailable", http.StatusServiceUnavailable) + data, err := s.renderSessionTemplate(diskPath, session) + if err != nil { + http.Error(w, "rendering artifact", http.StatusServiceUnavailable) + return + } + http.ServeContent(w, r, artifact.Name, session.CreationTimestamp.Time, bytes.NewReader(data)) + s.recordFirmwareDownloaded(r.Context(), session, artifact.Name) return } http.ServeFile(w, r, diskPath) + s.recordFirmwareDownloaded(r.Context(), session, artifact.Name) +} + +func (s *SessionHTTPServer) recordFirmwareDownloaded(ctx context.Context, session *v1alpha3.NetbootSession, artifactName string) { + if s.StatusRecorder == nil || artifactName != session.Spec.Boot.FirmwareArtifact { + return + } + if err := s.StatusRecorder.RecordCondition(ctx, session.Name, session.UID, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionBootLoaderDownloaded, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "firmware artifact downloaded", + }); err != nil { + slog.Warn("recording firmware artifact download", "session", session.Name, "err", err) + } +} + +func (s *SessionHTTPServer) renderSessionTemplate(templatePath string, session *v1alpha3.NetbootSession) ([]byte, error) { + content, err := os.ReadFile(templatePath) + if err != nil { + return nil, fmt.Errorf("reading template: %w", err) + } + baseURL, err := SessionBaseURL(s.Capabilities, session) + if err != nil { + return nil, err + } + machine := sessionMachine(session) + cluster := session.Spec.Provisioning.Cluster + agentConfig := provision.BuildAgentConfig(provision.BuildAgentConfigParams{ + Machine: machine, + Cluster: provision.ClusterEndpoint{ + APIServer: cluster.APIServerURL, + CACertBase64: cluster.CACertBase64, + ClusterDNS: cluster.DNS, + KubeVersion: cluster.KubernetesVersion, + }, + ProviderLabels: session.Spec.Provisioning.ProviderLabels, + AttestURL: baseURL, + }) + agentConfigJSON, err := json.MarshalIndent(agentConfig, " ", " ") + if err != nil { + return nil, fmt.Errorf("marshaling agent config: %w", err) + } + data := newTemplateData(machine, ClusterInfo{ApiserverURL: cluster.APIServerURL, CACertBase64: cluster.CACertBase64}, baseURL, string(agentConfigJSON), "", true) + data.ArtifactBaseURL, err = JoinServeURLPath(baseURL, "artifacts") + if err != nil { + return nil, err + } + data.BootImageWrittenURL, err = JoinServeURLPath(baseURL, "callbacks/boot-image-written") + if err != nil { + return nil, err + } + data.CloudInitURL, err = JoinServeURLPath(baseURL, "callbacks/cloud-init") + if err != nil { + return nil, err + } + data.InstallLogURL, err = JoinServeURLPath(baseURL, "logs/agent-install") + if err != nil { + return nil, err + } + + return renderTemplate(string(content), data) +} + +func sessionMachine(session *v1alpha3.NetbootSession) *v1alpha3.Machine { + return &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: session.Spec.Machine.Name, UID: session.Spec.Machine.UID, Generation: session.Spec.Machine.Generation}, + Spec: v1alpha3.MachineSpec{ + Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{ + Transport: session.Spec.Boot.Transport, + ConfigurationSource: session.Spec.Boot.ConfigurationSource, + NetworkMode: session.Spec.Boot.NetworkMode, + Architecture: session.Spec.Boot.Architecture, + DHCPLeases: append([]v1alpha3.DHCPLease(nil), session.Spec.Boot.DHCPLeases...), + TargetDisk: session.Spec.Boot.TargetDisk, + }}, + Kubernetes: session.Spec.Provisioning.Kubernetes.DeepCopy(), + Agent: session.Spec.Provisioning.Agent.DeepCopy(), + }, + } } func (s *SessionHTTPServer) authorizeSession(w http.ResponseWriter, r *http.Request) (*v1alpha3.NetbootSession, bool) { diff --git a/internal/metalman/netboot/session_http_test.go b/internal/metalman/netboot/session_http_test.go index df2503bdc..a61aedfdd 100644 --- a/internal/metalman/netboot/session_http_test.go +++ b/internal/metalman/netboot/session_http_test.go @@ -8,6 +8,9 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" "time" @@ -115,6 +118,71 @@ func TestSessionHTTPRecordsAuthenticatedSessionCallback(t *testing.T) { require.Equal(t, metav1.ConditionTrue, recorder.condition.Status) } +func TestSessionHTTPRecordsCloudInitCompletionOnlyForFinalSuccess(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-cloud-init", "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd") + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + recorder := &recordingSessionConditionsRecorder{} + handler := (&SessionHTTPServer{Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, StatusRecorder: recorder}).Handler() + path := "/v1/netboot/sessions/session-cloud-init/" + capability + "/callbacks/cloud-init" + + for _, body := range []string{ + `{"event_type":"start","name":"modules-final","description":"running"}`, + `{"event_type":"finish","name":"modules-config","description":"done","result":"SUCCESS"}`, + `{"event_type":"finish","name":"modules-final","description":"done","result":"SUCCESS"}`, + } { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))) + require.Equal(t, http.StatusNoContent, response.Code) + } + + require.Len(t, recorder.conditions, 3) + require.Equal(t, metav1.ConditionFalse, recorder.conditions[0].Status) + require.Equal(t, metav1.ConditionFalse, recorder.conditions[1].Status) + require.Equal(t, metav1.ConditionTrue, recorder.conditions[2].Status) + for _, condition := range recorder.conditions { + require.Equal(t, v1alpha3.NetbootSessionConditionCloudInitDone, condition.Type) + } +} + +func TestSessionHTTPRecordsFirmwareDownloadForExactSession(t *testing.T) { + t.Parallel() + + const digest = "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + cache := setupOCICache(t, "unused.example/netboot:latest", strings.TrimPrefix(digest, "sha256:"), map[string][]byte{ + "bootx64.efi": []byte("firmware"), + }) + session := testNetbootSession("session-firmware", digest) + session.Spec.Boot.FirmwareArtifact = "bootx64.efi" + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, v1alpha3.NetbootSessionArtifact{ + Name: "bootx64.efi", Source: "NetbootImage", Path: "/disk/bootx64.efi", + }) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + recorder := &recordingSessionConditionRecorder{} + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-firmware/"+capability+"/artifacts/bootx64.efi", nil) + (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer, StatusRecorder: recorder}).Handler().ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code) + require.Equal(t, session.Name, recorder.sessionName) + require.Equal(t, session.UID, recorder.sessionUID) + require.Equal(t, v1alpha3.NetbootSessionConditionBootLoaderDownloaded, recorder.condition.Type) +} + func TestSessionArtifactURLUsesEndpointAndCapability(t *testing.T) { t.Parallel() @@ -136,12 +204,121 @@ func TestSessionArtifactURLUsesEndpointAndCapability(t *testing.T) { require.Equal(t, "https://boot.example.com/base/v1/netboot/sessions/session-url/"+capability+"/artifacts/http/bootx64.efi", bootURL) } +func TestSessionHTTPRendersBootArtifactsFromImmutableSnapshot(t *testing.T) { + t.Parallel() + + const digest = "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + cache := NewOCICache(t.TempDir()) + diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.PXEArchitectureAMD64) + require.NoError(t, os.MkdirAll(filepath.Join(diskDir, "grub"), 0o755)) + templateContent, err := os.ReadFile(filepath.Join("..", "..", "..", "images", "netboot", "assets", "grub.cfg.tmpl")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(diskDir, "grub", "grub.cfg.tmpl"), templateContent, 0o600)) + + session := testNetbootSession("session-render", digest) + session.Spec.Boot = v1alpha3.NetbootSessionBoot{ + Architecture: v1alpha3.PXEArchitectureAMD64, + TargetDisk: "/dev/sda", + DHCPLeases: []v1alpha3.DHCPLease{{ + MAC: "aa:bb:cc:dd:ee:ff", IPv4: "192.0.2.20", SubnetMask: "255.255.255.0", Gateway: "192.0.2.1", + }}, + } + session.Spec.Provisioning = v1alpha3.NetbootSessionProvisioning{ + Cluster: v1alpha3.NetbootSessionCluster{APIServerURL: "https://api.snapshot.example:6443"}, + } + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, + v1alpha3.NetbootSessionArtifact{Name: "grub/grub.cfg", Source: "NetbootImage", Path: "/disk/grub/grub.cfg"}, + v1alpha3.NetbootSessionArtifact{Name: "vmlinuz", Source: "NetbootImage", Path: "/disk/vmlinuz"}, + v1alpha3.NetbootSessionArtifact{Name: "initrd", Source: "NetbootImage", Path: "/disk/initrd"}, + v1alpha3.NetbootSessionArtifact{Name: "init.cpio", Source: "NetbootImage", Path: "/disk/init.cpio"}, + ) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-render/"+capability+"/artifacts/grub/grub.cfg", nil) + (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler().ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code) + body := response.Body.String() + capabilityBase := "https://boot.example.com/v1/netboot/sessions/session-render/" + capability + require.Contains(t, body, "linux "+capabilityBase+"/artifacts/vmlinuz") + require.Contains(t, body, "initrd "+capabilityBase+"/artifacts/initrd "+capabilityBase+"/artifacts/init.cpio") + require.Contains(t, body, "unbounded.image_url="+capabilityBase+"/artifacts/disk.img.gz") + require.Contains(t, body, "unbounded.serve_url="+capabilityBase) + require.Contains(t, body, "unbounded.ds_url="+capabilityBase+"/artifacts/cloud-init/") + require.Contains(t, body, "unbounded.apiserver_url=https://api.snapshot.example:6443") + require.Contains(t, body, "unbounded.disk=/dev/sda") +} + +func TestSessionHTTPRendersCapabilityScopedCallbacks(t *testing.T) { + t.Parallel() + + const digest = "sha256:abababababababababababababababababababababababababababababababab" + cache := NewOCICache(t.TempDir()) + diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.PXEArchitectureAMD64) + require.NoError(t, os.MkdirAll(filepath.Join(diskDir, "cloud-init"), 0o755)) + for _, artifact := range []string{"grub.cfg", "vendor-data"} { + source, err := os.ReadFile(filepath.Join("..", "..", "..", "images", "netboot", "assets", artifact+".tmpl")) + require.NoError(t, err) + destination := filepath.Join(diskDir, artifact+".tmpl") + if artifact == "vendor-data" { + destination = filepath.Join(diskDir, "cloud-init", artifact+".tmpl") + } + require.NoError(t, os.WriteFile(destination, source, 0o600)) + } + + session := testNetbootSession("session-callbacks", digest) + session.Spec.Boot.Architecture = v1alpha3.PXEArchitectureAMD64 + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, + v1alpha3.NetbootSessionArtifact{Name: "grub.cfg", Source: "NetbootImage", Path: "/disk/grub.cfg"}, + v1alpha3.NetbootSessionArtifact{Name: "cloud-init/vendor-data", Source: "NetbootImage", Path: "/disk/cloud-init/vendor-data"}, + ) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() + capabilityBase := "https://boot.example.com/v1/netboot/sessions/session-callbacks/" + capability + + grubResponse := httptest.NewRecorder() + handler.ServeHTTP(grubResponse, httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-callbacks/"+capability+"/artifacts/grub.cfg", nil)) + require.Equal(t, http.StatusOK, grubResponse.Code) + require.Contains(t, grubResponse.Body.String(), "unbounded.boot_image_written_url="+capabilityBase+"/callbacks/boot-image-written") + require.NotContains(t, grubResponse.Body.String(), "/pxe/disable") + + vendorResponse := httptest.NewRecorder() + handler.ServeHTTP(vendorResponse, httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-callbacks/"+capability+"/artifacts/cloud-init/vendor-data", nil)) + require.Equal(t, http.StatusOK, vendorResponse.Code) + require.Contains(t, vendorResponse.Body.String(), "endpoint: "+capabilityBase+"/callbacks/cloud-init") + require.Contains(t, vendorResponse.Body.String(), `"URL": "`+capabilityBase+`"`) + require.Contains(t, vendorResponse.Body.String(), `"`+capabilityBase+`/logs/agent-install"`) +} + type recordingSessionConditionRecorder struct { sessionName string sessionUID types.UID condition metav1.Condition } +type recordingSessionConditionsRecorder struct { + conditions []metav1.Condition +} + +func (r *recordingSessionConditionsRecorder) RecordCondition(_ context.Context, _ string, _ types.UID, condition metav1.Condition) error { + r.conditions = append(r.conditions, condition) + + return nil +} + func (r *recordingSessionConditionRecorder) RecordCondition(_ context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error { r.sessionName = sessionName r.sessionUID = sessionUID From 1fb8a1138bffb30fd6a19a50114613546d119855 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 17:53:41 +0000 Subject: [PATCH 16/43] metalman: bind attestation to netboot sessions --- internal/metalman/attestation/attestation.go | 15 ++++ .../metalman/attestation/attestation_test.go | 24 ++++++ internal/metalman/commands/serve_pxe.go | 18 ++-- internal/metalman/netboot/session_http.go | 83 ++++++++++++++++++- .../metalman/netboot/session_http_test.go | 41 +++++++++ 5 files changed, 172 insertions(+), 9 deletions(-) diff --git a/internal/metalman/attestation/attestation.go b/internal/metalman/attestation/attestation.go index 98f5206ce..27e50708c 100644 --- a/internal/metalman/attestation/attestation.go +++ b/internal/metalman/attestation/attestation.go @@ -324,6 +324,21 @@ func (h *Handler) Attest(w http.ResponseWriter, r *http.Request) { } } +// AttestMachine performs attestation for an already authenticated Machine +// identity instead of deriving identity from the request network path. +func (h *Handler) AttestMachine(w http.ResponseWriter, r *http.Request, machine *v1alpha3.Machine) { + if machine == nil { + http.Error(w, "node not found", http.StatusNotFound) + return + } + + exact := *h + exact.LookupNodeByIP = func(context.Context, string) (*v1alpha3.Machine, error) { + return machine, nil + } + exact.Attest(w, r) +} + // publicKeysEqual compares two crypto.PublicKey values structurally. func publicKeysEqual(a, b crypto.PublicKey) bool { type equaler interface { diff --git a/internal/metalman/attestation/attestation_test.go b/internal/metalman/attestation/attestation_test.go index b1f8eae01..b0849bdd1 100644 --- a/internal/metalman/attestation/attestation_test.go +++ b/internal/metalman/attestation/attestation_test.go @@ -342,6 +342,30 @@ func TestAttestTOFUStoresKey(t *testing.T) { } } +func TestAttestMachineDoesNotUseRequestSourceIP(t *testing.T) { + _, ekPub := testEKKeyPair(t) + srkPub, _ := testSRKPub(t) + node := &v1alpha3.Machine{ObjectMeta: metav1.ObjectMeta{Name: "session-machine"}} + handler := testHandler(t, node) + handler.LookupNodeByIP = func(context.Context, string) (*v1alpha3.Machine, error) { + t.Fatal("session attestation must not resolve a Machine by source IP") + return nil, nil + } + body, err := json.Marshal(AttestRequest{EKPub: ekPub, SRKPub: srkPub}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/session/attest", bytes.NewReader(body)) + request.RemoteAddr = "198.51.100.20:12345" + response := httptest.NewRecorder() + + handler.AttestMachine(response, request, node) + + if response.Code != http.StatusOK { + t.Fatalf("attest exact Machine: expected 200, got %d: %s", response.Code, response.Body.String()) + } +} + func TestAttestTOFURejectsNewKey(t *testing.T) { _, ekPub := testEKKeyPair(t) srkPub, _ := testSRKPub(t) diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index 5ebc51022..78a2cfb84 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -348,6 +348,15 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { if components.http { httpMux := http.NewServeMux() + var attestHandler *attestation.Handler + if components.attestation { + attestHandler = &attestation.Handler{ + Clientset: clientset, + ClusterCA: clusterCA, + LookupNodeByIP: resolver.LookupNodeByIP, + StatusUpdater: &StatusUpdater{Client: mgr.GetClient()}, + } + } if components.sessionHTTP { capabilityKey, err := os.ReadFile(capabilityKeyFile) if err != nil { @@ -362,19 +371,14 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { Cache: ociCache, Capabilities: capabilities, StatusRecorder: &metalmachineops.SessionStatusRecorder{Client: mgr.GetClient()}, + Attestation: attestHandler, EdgeAuthenticator: &netboot.TokenReviewEdgeAuthenticator{ Client: clientset.AuthenticationV1(), ServiceAccountName: edgeServiceAccount, }, }).RegisterHandlers(httpMux) } - if components.attestation { - attestHandler := &attestation.Handler{ - Clientset: clientset, - ClusterCA: clusterCA, - LookupNodeByIP: resolver.LookupNodeByIP, - StatusUpdater: &StatusUpdater{Client: mgr.GetClient()}, - } + if attestHandler != nil && !components.sessionHTTP { httpMux.HandleFunc("POST /attest", attestHandler.Attest) } diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go index 8aa95e9bd..2f6aa038a 100644 --- a/internal/metalman/netboot/session_http.go +++ b/internal/metalman/netboot/session_http.go @@ -33,6 +33,7 @@ type SessionHTTPServer struct { Capabilities *CapabilitySigner StatusRecorder SessionConditionRecorder EdgeAuthenticator EdgeAuthenticator + Attestation SessionAttester } // EdgeAuthenticator validates access to internal edge-only routes. @@ -45,6 +46,11 @@ type SessionConditionRecorder interface { RecordCondition(ctx context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error } +// SessionAttester performs TPM attestation for an exact authenticated Machine. +type SessionAttester interface { + AttestMachine(w http.ResponseWriter, r *http.Request, machine *v1alpha3.Machine) +} + // SessionArtifactURL returns the externally advertised capability URL for one // artifact listed by the immutable session. func SessionArtifactURL(signer *CapabilitySigner, session *v1alpha3.NetbootSession, artifactName string) (string, error) { @@ -92,6 +98,7 @@ func (s *SessionHTTPServer) RegisterHandlers(mux *http.ServeMux) { mux.HandleFunc("GET /v1/netboot/sessions/{session}/{capability}/artifacts/{artifact...}", s.handleArtifact) mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/callbacks/{milestone}", s.handleCallback) mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/logs/agent-install", s.handleInstallLog) + mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/attest", s.handleAttest) mux.HandleFunc("GET /v1/netboot/endpoints/{endpoint}/dhcp/{mac}", s.handleDHCPDecision) } @@ -246,6 +253,48 @@ func (s *SessionHTTPServer) handleInstallLog(w http.ResponseWriter, r *http.Requ w.WriteHeader(http.StatusNoContent) } +func (s *SessionHTTPServer) handleAttest(w http.ResponseWriter, r *http.Request) { + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + if s.Attestation == nil || s.StatusRecorder == nil { + http.Error(w, "session attestation unavailable", http.StatusServiceUnavailable) + return + } + + var machine v1alpha3.Machine + if err := s.Client.Get(r.Context(), client.ObjectKey{Name: session.Spec.Machine.Name}, &machine); err != nil { + if apierrors.IsNotFound(err) { + http.NotFound(w, r) + return + } + http.Error(w, "loading Machine", http.StatusServiceUnavailable) + return + } + if machine.UID != session.Spec.Machine.UID { + http.Error(w, "Machine identity changed", http.StatusConflict) + return + } + + response := newBufferedResponseWriter() + s.Attestation.AttestMachine(response, r, &machine) + if response.status < http.StatusOK || response.status >= http.StatusMultipleChoices { + response.writeTo(w) + return + } + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionAttested, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "TPM attestation succeeded", + }); err != nil { + http.Error(w, "recording session status", http.StatusServiceUnavailable) + return + } + response.writeTo(w) +} + func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Request) { if s.Cache == nil { http.Error(w, "session server unavailable", http.StatusServiceUnavailable) @@ -420,13 +469,43 @@ func sessionMilestoneCondition(milestone string) (string, bool) { return v1alpha3.NetbootSessionConditionBootImageWritten, true case "cloud-init-done": return v1alpha3.NetbootSessionConditionCloudInitDone, true - case "attested": - return v1alpha3.NetbootSessionConditionAttested, true default: return "", false } } +type bufferedResponseWriter struct { + header http.Header + body bytes.Buffer + status int +} + +func newBufferedResponseWriter() *bufferedResponseWriter { + return &bufferedResponseWriter{header: make(http.Header), status: http.StatusOK} +} + +func (w *bufferedResponseWriter) Header() http.Header { + return w.header +} + +func (w *bufferedResponseWriter) WriteHeader(status int) { + w.status = status +} + +func (w *bufferedResponseWriter) Write(data []byte) (int, error) { + return w.body.Write(data) +} + +func (w *bufferedResponseWriter) writeTo(destination http.ResponseWriter) { + for key, values := range w.header { + destination.Header()[key] = append([]string(nil), values...) + } + destination.WriteHeader(w.status) + if _, err := destination.Write(w.body.Bytes()); err != nil { + slog.Warn("writing buffered attestation response", "err", err) + } +} + func sessionArtifact(session *v1alpha3.NetbootSession, name string) (v1alpha3.NetbootSessionArtifact, bool) { for _, artifact := range session.Spec.Artifacts.Files { if artifact.Name == name { diff --git a/internal/metalman/netboot/session_http_test.go b/internal/metalman/netboot/session_http_test.go index a61aedfdd..98f011581 100644 --- a/internal/metalman/netboot/session_http_test.go +++ b/internal/metalman/netboot/session_http_test.go @@ -152,6 +152,37 @@ func TestSessionHTTPRecordsCloudInitCompletionOnlyForFinalSuccess(t *testing.T) } } +func TestSessionHTTPAttestsExactSessionMachineAndRecordsMilestone(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-attest", "sha256:dededededededededededededededededededededededededededededededede") + machine := &v1alpha3.Machine{ObjectMeta: metav1.ObjectMeta{Name: session.Spec.Machine.Name, UID: session.Spec.Machine.UID}} + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session, machine).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + attester := &recordingSessionAttester{} + recorder := &recordingSessionConditionRecorder{} + handler := (&SessionHTTPServer{ + Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, + StatusRecorder: recorder, Attestation: attester, + }).Handler() + + request := httptest.NewRequest(http.MethodPost, "/v1/netboot/sessions/session-attest/"+capability+"/attest", strings.NewReader(`{"ekPub":"a2V5","srkPub":"a2V5"}`)) + request.RemoteAddr = "198.51.100.25:12345" + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code) + require.Equal(t, machine.Name, attester.machine.Name) + require.Equal(t, machine.UID, attester.machine.UID) + require.Equal(t, v1alpha3.NetbootSessionConditionAttested, recorder.condition.Type) + require.Equal(t, session.UID, recorder.sessionUID) +} + func TestSessionHTTPRecordsFirmwareDownloadForExactSession(t *testing.T) { t.Parallel() @@ -313,6 +344,16 @@ type recordingSessionConditionsRecorder struct { conditions []metav1.Condition } +type recordingSessionAttester struct { + machine *v1alpha3.Machine +} + +func (a *recordingSessionAttester) AttestMachine(w http.ResponseWriter, _ *http.Request, machine *v1alpha3.Machine) { + a.machine = machine.DeepCopy() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"credentialBlob":"YQ=="}`)) +} + func (r *recordingSessionConditionsRecorder) RecordCondition(_ context.Context, _ string, _ types.UID, condition metav1.Condition) error { r.conditions = append(r.conditions, condition) From caddde3cf5601715f7dc219eedda931c8e32fd79 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:04:10 +0000 Subject: [PATCH 17/43] metalman: split operator control and server planes --- deploy/machina/06-metalman-rbac.yaml.tmpl | 88 ++++++- .../operator/components/metalman/metalman.go | 174 ++++++++++---- .../components/metalman/metalman_test.go | 219 +++++++++++++++--- 3 files changed, 410 insertions(+), 71 deletions(-) diff --git a/deploy/machina/06-metalman-rbac.yaml.tmpl b/deploy/machina/06-metalman-rbac.yaml.tmpl index e7b2eff83..2e5c8c527 100644 --- a/deploy/machina/06-metalman-rbac.yaml.tmpl +++ b/deploy/machina/06-metalman-rbac.yaml.tmpl @@ -5,6 +5,13 @@ metadata: name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} + --- apiVersion: v1 kind: ServiceAccount @@ -33,12 +40,38 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] - # Bootstrap token creation (scoped to metalman-bootstrap SA) + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + # Bootstrap token creation is scoped to the bootstrap ServiceAccount. - apiGroups: [""] resources: ["serviceaccounts/token"] resourceNames: ["metalman-bootstrap"] verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} +subjects: + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} +roleRef: + kind: Role + name: metalman-server + apiGroup: rbac.authorization.k8s.io + --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -49,6 +82,9 @@ subjects: - kind: ServiceAccount name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} roleRef: kind: Role name: metalman-controller @@ -77,6 +113,9 @@ subjects: - kind: ServiceAccount name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} roleRef: kind: Role name: metalman-controller @@ -152,6 +191,47 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metalman-controller +subjects: + - kind: ServiceAccount + name: metalman-controller + namespace: {{ default "unbounded-system" .Namespace }} +roleRef: + kind: ClusterRole + name: metalman-controller + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metalman-server +rules: + - apiGroups: ["unbounded-cloud.io"] + resources: ["machines"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machines/status"] + verbs: ["get", "update"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machineoperations"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machineoperations/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints", "netbootsessions"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootsessions/status"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["secrets", "configmaps", "nodes"] + verbs: ["get", "list", "watch"] # Authenticate audience-bound edge ServiceAccount tokens. - apiGroups: ["authentication.k8s.io"] resources: ["tokenreviews"] @@ -161,14 +241,14 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: metalman-controller + name: metalman-server subjects: - kind: ServiceAccount - name: metalman-controller + name: metalman-server namespace: {{ default "unbounded-system" .Namespace }} roleRef: kind: ClusterRole - name: metalman-controller + name: metalman-server apiGroup: rbac.authorization.k8s.io --- diff --git a/internal/operator/components/metalman/metalman.go b/internal/operator/components/metalman/metalman.go index 57e6f2a1a..a03925de8 100644 --- a/internal/operator/components/metalman/metalman.go +++ b/internal/operator/components/metalman/metalman.go @@ -1,25 +1,29 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Package metalman implements the per-Site metalman PXE controller component. +// Package metalman implements the per-Site Metalman control and serving plane. package metalman import ( "context" + "crypto/rand" + "fmt" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" machinamanifests "github.com/Azure/unbounded/deploy/machina" "github.com/Azure/unbounded/internal/operator/component" ) -// Component reconciles the per-Site metalman PXE controller. +// Component reconciles the per-Site Metalman workloads. type Component struct{} // New returns the metalman per-Site component. @@ -40,27 +44,44 @@ func (Component) Enabled(site *unboundedv1alpha3.Site) bool { return unboundedv1alpha3.ComponentEnabled(&site.Spec.Components.Metalman.SiteComponentSpec) } -// Reconcile deploys the per-site metalman PXE controller and its RBAC. +// Reconcile deploys the per-site Metalman controller and server plane. func (Component) Reconcile(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) component.Result { if err := env.ApplyManifestFS(ctx, machinamanifests.Manifests, mutateSupportObject); err != nil { return component.Failed(err) } - - if err := env.ApplyObject(ctx, deployment(site, env.Namespace, env.Config)); err != nil { + if err := ensureCapabilitySecret(ctx, env, site); err != nil { return component.Failed(err) } + for _, obj := range []client.Object{ + controllerDeployment(site, env.Namespace, env.Config), + serverDeployment(site, env.Namespace, env.Config), + serverService(site, env.Namespace), + } { + if err := env.ApplyObject(ctx, obj); err != nil { + return component.Failed(err) + } + } + return component.Reconciled() } -// Cleanup removes the per-site metalman Deployment. The shared metalman RBAC is +// Cleanup removes the per-site Metalman workloads. The shared Metalman RBAC is // left in place; it is harmless when unreferenced and may still be used by other // sites. func (Component) Cleanup(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) error { - return env.DeleteIfExists(ctx, &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, - ObjectMeta: metav1.ObjectMeta{Name: DeploymentName(site.Name), Namespace: env.Namespace}, - }) + for _, obj := range []client.Object{ + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: DeploymentName(site.Name), Namespace: env.Namespace}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, ObjectMeta: metav1.ObjectMeta{Name: CapabilitySecretName(site.Name), Namespace: env.Namespace}}, + } { + if err := env.DeleteIfExists(ctx, obj); err != nil { + return err + } + } + + return nil } // SetupWatches recreates the per-site Deployment if it is deleted or drifts, via @@ -68,11 +89,20 @@ func (Component) Cleanup(ctx context.Context, env *component.Env, site *unbounde // updates so pod-count churn does not re-apply the Deployment. func (Component) SetupWatches(b *builder.Builder, env *component.Env) { b.Owns(&appsv1.Deployment{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) + b.Owns(&corev1.Service{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) } // DeploymentName is the per-site metalman Deployment name. func DeploymentName(site string) string { return "metalman-controller-" + site } +// ServerName is the per-site Metalman server Deployment and Service name. +func ServerName(site string) string { return "metalman-server-" + site } + +// CapabilitySecretName is the per-site capability signing Secret name. +func CapabilitySecretName(site string) string { return "metalman-capability-" + site } + +const capabilitySecretKey = "capability.key" + // SupportObjectNameSubstring identifies the metalman RBAC objects that ship in // the machina manifest set. It is exported so the machina component can skip // exactly the objects the metalman component owns and applies. @@ -95,25 +125,34 @@ func mutateSupportObject(obj *unstructured.Unstructured) error { return nil } -func deployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config) *appsv1.Deployment { +func controllerDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config) *appsv1.Deployment { + return roleDeployment(site, namespace, cfg, metalmanControllerRole, 1) +} + +func serverDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config) *appsv1.Deployment { + return roleDeployment(site, namespace, cfg, metalmanServerRole, 2) +} + +const ( + metalmanControllerRole = "controller" + metalmanServerRole = "server" +) + +func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config, role string, replicas int32) *appsv1.Deployment { image := cfg.Image("metalman") name := DeploymentName(site.Name) - labels := map[string]string{ - "app": "unbounded-pxe", - "app.kubernetes.io/name": "metalman-controller", - "app.kubernetes.io/component": "metalman", - unboundedv1alpha3.MachineSiteLabelKey: site.Name, + if role == metalmanServerRole { + name = ServerName(site.Name) } - args := []string{"serve-pxe", "--site=" + site.Name} - if site.Spec.Components.Metalman.DHCPAutoInterface != nil && *site.Spec.Components.Metalman.DHCPAutoInterface { - args = append(args, "--dhcp-auto-interface") + labels := map[string]string{ + "app": "unbounded-metalman", + "app.kubernetes.io/name": "metalman-" + role, + "app.kubernetes.io/component": role, + unboundedv1alpha3.MachineSiteLabelKey: site.Name, } - replicas := int32(1) - if site.Spec.Components.Metalman.Replicas != nil { - replicas = *site.Spec.Components.Metalman.Replicas - } + args := []string{role, "--site=" + site.Name, "--cache-dir=/var/cache/metalman"} env := []corev1.EnvVar{{ // Metalman resolves its leader-election lease namespace from @@ -135,11 +174,14 @@ func deployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Co env = append(env, corev1.EnvVar{Name: "METALMAN_APISERVER_URL", Value: cfg.APIServerEndpoint}) } - // metalman is hostNetwork and binds host ports (DHCP/TFTP/HTTP), so a surge - // pod cannot start while the old pod holds them on the same node. Terminate - // the old pod before creating the new one to avoid a rollout deadlock. - maxSurge := intstr.FromInt32(0) - maxUnavailable := intstr.FromInt32(1) + maxSurge := intstr.FromInt32(1) + maxUnavailable := intstr.FromInt32(0) + ports := []corev1.ContainerPort{{Name: "health", ContainerPort: 8081, Protocol: corev1.ProtocolTCP}} + serviceAccountName := "metalman-controller" + if role == metalmanServerRole { + serviceAccountName = "metalman-server" + ports = append(ports, corev1.ContainerPort{Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}) + } return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, @@ -162,35 +204,87 @@ func deployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Co Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ - HostNetwork: true, - ServiceAccountName: "metalman-controller", - // Match either the canonical or deprecated site label during - // the node-label deprecation window. Storage scopes its - // DaemonSet the same way. - Affinity: component.SiteNodeAffinity(site.Name), + ServiceAccountName: serviceAccountName, Containers: []corev1.Container{{ Name: "metalman", Image: image, ImagePullPolicy: corev1.PullAlways, Args: args, Env: env, - Ports: []corev1.ContainerPort{ - {Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}, - {Name: "health", ContainerPort: 8081, Protocol: corev1.ProtocolTCP}, - {Name: "dhcp", ContainerPort: 67, Protocol: corev1.ProtocolUDP}, - {Name: "tftp", ContainerPort: 69, Protocol: corev1.ProtocolUDP}, - }, + Ports: ports, VolumeMounts: []corev1.VolumeMount{ {Name: "tmp", MountPath: "/tmp"}, {Name: "cache", MountPath: "/var/cache/metalman"}, + {Name: "capability-key", MountPath: "/var/run/secrets/metalman", ReadOnly: true}, }, }}, Volumes: []corev1.Volume{ {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "cache", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "capability-key", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: CapabilitySecretName(site.Name), + }}}, }, }, }, }, } } + +func ensureCapabilitySecret(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) error { + secret := &corev1.Secret{} + key := client.ObjectKey{Namespace: env.Namespace, Name: CapabilitySecretName(site.Name)} + if err := env.Client.Get(ctx, key, secret); err == nil { + return nil + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("get capability Secret: %w", err) + } + + capabilityKey := make([]byte, 32) + if _, err := rand.Read(capabilityKey); err != nil { + return fmt.Errorf("generate capability key: %w", err) + } + + secret = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Data: map[string][]byte{capabilitySecretKey: capabilityKey}, + } + if err := env.Client.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create capability Secret: %w", err) + } + + return nil +} + +func serverService(site *unboundedv1alpha3.Site, namespace string) *corev1.Service { + labels := map[string]string{ + "app": "unbounded-metalman", + "app.kubernetes.io/name": "metalman-server", + "app.kubernetes.io/component": metalmanServerRole, + unboundedv1alpha3.MachineSiteLabelKey: site.Name, + } + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: ServerName(site.Name), + Namespace: namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: corev1.ServiceSpec{ + Selector: labels, + Ports: []corev1.ServicePort{{ + Name: "http", + Port: 8880, + TargetPort: intstr.FromInt32(8880), + Protocol: corev1.ProtocolTCP, + }}, + }, + } +} diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index a7ace1aaa..ff1c7cde7 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -4,6 +4,8 @@ package metalman import ( + "os" + "strings" "testing" appsv1 "k8s.io/api/apps/v1" @@ -75,20 +77,54 @@ func TestMutateSupportObject(t *testing.T) { } } +func TestRBACSeparatesControllerAndServerIdentities(t *testing.T) { + data, err := os.ReadFile("../../../../deploy/machina/06-metalman-rbac.yaml.tmpl") + if err != nil { + t.Fatalf("read Metalman RBAC template: %v", err) + } + manifest := string(data) + + for _, required := range []string{ + "name: metalman-controller", + "name: metalman-server", + "name: metalman-edge", + "resources: [\"tokenreviews\"]", + "resources: [\"serviceaccounts/token\"]", + } { + if !strings.Contains(manifest, required) { + t.Fatalf("RBAC template missing %q", required) + } + } + + serverNamespaceRole := manifest[strings.Index(manifest, "kind: Role\nmetadata:\n name: metalman-server"):] + serverNamespaceRole = serverNamespaceRole[:strings.Index(serverNamespaceRole, "\n---")] + if !strings.Contains(serverNamespaceRole, "resources: [\"serviceaccounts/token\"]") { + t.Fatalf("metalman-server Role lacks bootstrap-token issuance:\n%s", serverNamespaceRole) + } + + serverClusterRole := manifest[strings.Index(manifest, "kind: ClusterRole\nmetadata:\n name: metalman-server"):] + serverClusterRole = serverClusterRole[:strings.Index(serverClusterRole, "\n---")] + if !strings.Contains(serverClusterRole, "resources: [\"tokenreviews\"]") { + t.Fatalf("metalman-server ClusterRole lacks TokenReview permission:\n%s", serverClusterRole) + } + + controllerRole := manifest[strings.Index(manifest, "kind: ClusterRole\nmetadata:\n name: metalman-controller\nrules:"):] + controllerRole = controllerRole[:strings.Index(controllerRole, "\n---")] + if strings.Contains(controllerRole, "resources: [\"tokenreviews\"]") { + t.Fatalf("metalman-controller retains server TokenReview permission:\n%s", controllerRole) + } +} + func TestDeployment(t *testing.T) { enabled := true - dhcpAuto := true - replicas := int32(3) site := &unboundedv1alpha3.Site{ ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}, Spec: unboundedv1alpha3.SiteSpec{Components: unboundedv1alpha3.SiteComponents{Metalman: &unboundedv1alpha3.MetalmanComponentSpec{ SiteComponentSpec: unboundedv1alpha3.SiteComponentSpec{Enabled: &enabled}, - DHCPAutoInterface: &dhcpAuto, - Replicas: &replicas, }}}, } - d := deployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3", APIServerEndpoint: "https://api.example:6443"}) + d := controllerDeployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3", APIServerEndpoint: "https://api.example:6443"}) if d.Name != "metalman-controller-rack-a" { t.Fatalf("name = %q", d.Name) } @@ -102,12 +138,12 @@ func TestDeployment(t *testing.T) { t.Fatalf("image = %q", container.Image) } - if got := container.Args; len(got) != 3 || got[0] != "serve-pxe" || got[1] != "--site=rack-a" || got[2] != "--dhcp-auto-interface" { + if got := container.Args; len(got) != 3 || got[0] != "controller" || got[1] != "--site=rack-a" || got[2] != "--cache-dir=/var/cache/metalman" { t.Fatalf("args = %#v", got) } - if d.Spec.Replicas == nil || *d.Spec.Replicas != 3 { - t.Fatalf("replicas = %v, want 3", d.Spec.Replicas) + if d.Spec.Replicas == nil || *d.Spec.Replicas != 1 { + t.Fatalf("replicas = %v, want 1", d.Spec.Replicas) } for _, path := range []string{"deployment", "selector", "pod"} { @@ -138,19 +174,117 @@ func TestDeployment(t *testing.T) { } assertSiteOwnerRef(t, d.OwnerReferences, "rack-a", "site-uid") - assertSiteAffinity(t, d.Spec.Template.Spec.Affinity, "rack-a") + assertOrdinaryPodNetworking(t, &d.Spec.Template.Spec) strategy := d.Spec.Strategy if strategy.Type != appsv1.RollingUpdateDeploymentStrategyType || strategy.RollingUpdate == nil { t.Fatalf("expected RollingUpdate strategy, got %+v", strategy) } - if got := strategy.RollingUpdate.MaxSurge; got == nil || got.IntValue() != 0 { - t.Fatalf("expected maxSurge=0, got %v", got) + if got := strategy.RollingUpdate.MaxSurge; got == nil || got.IntValue() != 1 { + t.Fatalf("expected maxSurge=1, got %v", got) + } + + if got := strategy.RollingUpdate.MaxUnavailable; got == nil || got.IntValue() != 0 { + t.Fatalf("expected maxUnavailable=0, got %v", got) + } +} + +func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { + site := &unboundedv1alpha3.Site{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}, + } + cfg := component.Config{ + ImageRegistry: "registry.example.com", + ImageTag: "v1.2.3", + APIServerEndpoint: "https://api.example:6443", + } + + controller := controllerDeployment(site, component.DefaultNamespace, cfg) + if controller.Spec.Replicas == nil || *controller.Spec.Replicas != 1 { + t.Fatalf("controller replicas = %v, want 1", controller.Spec.Replicas) + } + assertOrdinaryPodNetworking(t, &controller.Spec.Template.Spec) + + controllerContainer := controller.Spec.Template.Spec.Containers[0] + if got := controllerContainer.Args; len(got) < 2 || got[0] != "controller" || got[1] != "--site=rack-a" { + t.Fatalf("controller args = %#v", got) + } + if controller.Spec.Template.Spec.ServiceAccountName != "metalman-controller" { + t.Fatalf("controller service account = %q", controller.Spec.Template.Spec.ServiceAccountName) + } + assertCapabilityKeyMount(t, &controller.Spec.Template.Spec, &controllerContainer, "rack-a") + for _, port := range controllerContainer.Ports { + if port.Name == "http" || port.Name == "dhcp" || port.Name == "tftp" { + t.Fatalf("controller exposes data-plane port %#v", port) + } + } + + server := serverDeployment(site, component.DefaultNamespace, cfg) + if server.Spec.Replicas == nil || *server.Spec.Replicas != 2 { + t.Fatalf("server replicas = %v, want 2", server.Spec.Replicas) + } + assertOrdinaryPodNetworking(t, &server.Spec.Template.Spec) + + serverContainer := server.Spec.Template.Spec.Containers[0] + if got := serverContainer.Args; len(got) < 3 || got[0] != "server" || got[1] != "--site=rack-a" || got[2] != "--cache-dir=/var/cache/metalman" { + t.Fatalf("server args = %#v", got) + } + if server.Spec.Template.Spec.ServiceAccountName != "metalman-server" { + t.Fatalf("server service account = %q", server.Spec.Template.Spec.ServiceAccountName) + } + assertCapabilityKeyMount(t, &server.Spec.Template.Spec, &serverContainer, "rack-a") + if !hasContainerPort(serverContainer.Ports, "http", 8880) { + t.Fatalf("server ports = %#v, want HTTP 8880", serverContainer.Ports) + } + if got := server.Spec.Strategy.RollingUpdate; got == nil || got.MaxUnavailable == nil || got.MaxUnavailable.IntValue() != 0 { + t.Fatalf("server maxUnavailable = %#v, want 0", got) + } + + service := serverService(site, component.DefaultNamespace) + if service.Name != ServerName("rack-a") { + t.Fatalf("service name = %q, want %q", service.Name, ServerName("rack-a")) + } + if service.Spec.Selector["app.kubernetes.io/name"] != "metalman-server" || service.Spec.Selector[unboundedv1alpha3.MachineSiteLabelKey] != "rack-a" { + t.Fatalf("service selector = %#v", service.Spec.Selector) + } + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != 8880 || service.Spec.Ports[0].TargetPort.IntValue() != 8880 { + t.Fatalf("service ports = %#v", service.Spec.Ports) + } +} + +func TestEnsureCapabilitySecretPreservesExistingKey(t *testing.T) { + scheme := testScheme(t) + env := &component.Env{ + Client: fake.NewClientBuilder().WithScheme(scheme).Build(), + Scheme: scheme, + Namespace: component.DefaultNamespace, + } + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + + if err := ensureCapabilitySecret(t.Context(), env, site); err != nil { + t.Fatalf("first ensureCapabilitySecret: %v", err) } - if got := strategy.RollingUpdate.MaxUnavailable; got == nil || got.IntValue() != 1 { - t.Fatalf("expected maxUnavailable=1, got %v", got) + key := client.ObjectKey{Namespace: component.DefaultNamespace, Name: CapabilitySecretName("rack-a")} + first := &corev1.Secret{} + if err := env.Client.Get(t.Context(), key, first); err != nil { + t.Fatalf("get first capability Secret: %v", err) + } + if len(first.Data[capabilitySecretKey]) != 32 { + t.Fatalf("capability key length = %d, want 32", len(first.Data[capabilitySecretKey])) + } + + if err := ensureCapabilitySecret(t.Context(), env, site); err != nil { + t.Fatalf("second ensureCapabilitySecret: %v", err) + } + + second := &corev1.Secret{} + if err := env.Client.Get(t.Context(), key, second); err != nil { + t.Fatalf("get second capability Secret: %v", err) + } + if string(second.Data[capabilitySecretKey]) != string(first.Data[capabilitySecretKey]) { + t.Fatal("capability key changed across reconciliation") } } @@ -163,7 +297,7 @@ func TestDeploymentRespectsNamespaceAndDefaults(t *testing.T) { }}}, } - d := deployment(site, "custom-ns", component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) + d := controllerDeployment(site, "custom-ns", component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) if d.Namespace != "custom-ns" { t.Fatalf("namespace = %q, want custom-ns", d.Namespace) } @@ -178,19 +312,12 @@ func TestDeploymentRespectsNamespaceAndDefaults(t *testing.T) { } func TestDeploymentAllowsZeroReplicas(t *testing.T) { - enabled := true - replicas := int32(0) - site := &unboundedv1alpha3.Site{ - ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}, - Spec: unboundedv1alpha3.SiteSpec{Components: unboundedv1alpha3.SiteComponents{Metalman: &unboundedv1alpha3.MetalmanComponentSpec{ - SiteComponentSpec: unboundedv1alpha3.SiteComponentSpec{Enabled: &enabled}, - Replicas: &replicas, - }}}, - } - - d := deployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) - if d.Spec.Replicas == nil || *d.Spec.Replicas != 0 { - t.Fatalf("replicas = %v, want 0", d.Spec.Replicas) + // The split roles have fixed availability semantics; the former Site-level + // replica knob no longer scales the controller and data plane together. + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}} + d := serverDeployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) + if d.Spec.Replicas == nil || *d.Spec.Replicas != 2 { + t.Fatalf("server replicas = %v, want 2", d.Spec.Replicas) } } @@ -220,6 +347,44 @@ func findEnv(env []corev1.EnvVar, name string) *corev1.EnvVar { return nil } +func assertOrdinaryPodNetworking(t *testing.T, podSpec *corev1.PodSpec) { + t.Helper() + + if podSpec.HostNetwork { + t.Fatal("pod unexpectedly uses host networking") + } + if podSpec.Affinity != nil && podSpec.Affinity.NodeAffinity != nil && + podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + t.Fatalf("pod unexpectedly has required node affinity: %#v", podSpec.Affinity) + } +} + +func hasContainerPort(ports []corev1.ContainerPort, name string, port int32) bool { + for _, candidate := range ports { + if candidate.Name == name && candidate.ContainerPort == port { + return true + } + } + + return false +} + +func assertCapabilityKeyMount(t *testing.T, podSpec *corev1.PodSpec, container *corev1.Container, site string) { + t.Helper() + + for _, mount := range container.VolumeMounts { + if mount.Name == "capability-key" && mount.MountPath == "/var/run/secrets/metalman" && mount.ReadOnly { + for _, volume := range podSpec.Volumes { + if volume.Name == mount.Name && volume.Secret != nil && volume.Secret.SecretName == CapabilitySecretName(site) { + return + } + } + } + } + + t.Fatalf("missing read-only capability key mount: mounts=%#v volumes=%#v", container.VolumeMounts, podSpec.Volumes) +} + func assertSiteOwnerRef(t *testing.T, refs []metav1.OwnerReference, siteName, uid string) { t.Helper() From 1d5ad8237d5575e6e58677ba9594d936b83da555 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:06:34 +0000 Subject: [PATCH 18/43] metalman: add server availability policy --- .../operator/components/metalman/metalman.go | 62 +++++++++++++++++-- .../components/metalman/metalman_test.go | 36 ++++++++++- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/internal/operator/components/metalman/metalman.go b/internal/operator/components/metalman/metalman.go index a03925de8..3d597ddb4 100644 --- a/internal/operator/components/metalman/metalman.go +++ b/internal/operator/components/metalman/metalman.go @@ -11,7 +11,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" @@ -57,6 +59,7 @@ func (Component) Reconcile(ctx context.Context, env *component.Env, site *unboun controllerDeployment(site, env.Namespace, env.Config), serverDeployment(site, env.Namespace, env.Config), serverService(site, env.Namespace), + serverPodDisruptionBudget(site, env.Namespace), } { if err := env.ApplyObject(ctx, obj); err != nil { return component.Failed(err) @@ -75,6 +78,7 @@ func (Component) Cleanup(ctx context.Context, env *component.Env, site *unbounde &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, &corev1.Service{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, &corev1.Secret{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, ObjectMeta: metav1.ObjectMeta{Name: CapabilitySecretName(site.Name), Namespace: env.Namespace}}, + &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{APIVersion: "policy/v1", Kind: "PodDisruptionBudget"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, } { if err := env.DeleteIfExists(ctx, obj); err != nil { return err @@ -90,6 +94,7 @@ func (Component) Cleanup(ctx context.Context, env *component.Env, site *unbounde func (Component) SetupWatches(b *builder.Builder, env *component.Env) { b.Owns(&appsv1.Deployment{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) b.Owns(&corev1.Service{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) + b.Owns(&policyv1.PodDisruptionBudget{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) } // DeploymentName is the per-site metalman Deployment name. @@ -182,6 +187,12 @@ func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg componen serviceAccountName = "metalman-server" ports = append(ports, corev1.ContainerPort{Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}) } + probe := func(path string) *corev1.Probe { + return &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: path, + Port: intstr.FromInt32(8081), + }}} + } return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, @@ -212,6 +223,18 @@ func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg componen Args: args, Env: env, Ports: ports, + LivenessProbe: probe("/healthz"), + ReadinessProbe: probe("/readyz"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, VolumeMounts: []corev1.VolumeMount{ {Name: "tmp", MountPath: "/tmp"}, {Name: "cache", MountPath: "/var/cache/metalman"}, @@ -225,6 +248,12 @@ func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg componen SecretName: CapabilitySecretName(site.Name), }}}, }, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ + MaxSkew: 1, + TopologyKey: corev1.LabelHostname, + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + }}, }, }, }, @@ -262,12 +291,7 @@ func ensureCapabilitySecret(ctx context.Context, env *component.Env, site *unbou } func serverService(site *unboundedv1alpha3.Site, namespace string) *corev1.Service { - labels := map[string]string{ - "app": "unbounded-metalman", - "app.kubernetes.io/name": "metalman-server", - "app.kubernetes.io/component": metalmanServerRole, - unboundedv1alpha3.MachineSiteLabelKey: site.Name, - } + labels := serverLabels(site.Name) return &corev1.Service{ TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, @@ -288,3 +312,29 @@ func serverService(site *unboundedv1alpha3.Site, namespace string) *corev1.Servi }, } } + +func serverPodDisruptionBudget(site *unboundedv1alpha3.Site, namespace string) *policyv1.PodDisruptionBudget { + minAvailable := intstr.FromInt32(1) + + return &policyv1.PodDisruptionBudget{ + TypeMeta: metav1.TypeMeta{APIVersion: "policy/v1", Kind: "PodDisruptionBudget"}, + ObjectMeta: metav1.ObjectMeta{ + Name: ServerName(site.Name), + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &minAvailable, + Selector: &metav1.LabelSelector{MatchLabels: serverLabels(site.Name)}, + }, + } +} + +func serverLabels(site string) map[string]string { + return map[string]string{ + "app": "unbounded-metalman", + "app.kubernetes.io/name": "metalman-server", + "app.kubernetes.io/component": metalmanServerRole, + unboundedv1alpha3.MachineSiteLabelKey: site, + } +} diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index ff1c7cde7..fefec7aca 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -10,6 +10,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -25,7 +26,7 @@ func testScheme(t *testing.T) *runtime.Scheme { t.Helper() scheme := runtime.NewScheme() - for _, add := range []func(*runtime.Scheme) error{appsv1.AddToScheme, corev1.AddToScheme, unboundedv1alpha3.AddToScheme} { + for _, add := range []func(*runtime.Scheme) error{appsv1.AddToScheme, corev1.AddToScheme, policyv1.AddToScheme, unboundedv1alpha3.AddToScheme} { if err := add(scheme); err != nil { t.Fatalf("add to scheme: %v", err) } @@ -237,6 +238,14 @@ func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { if !hasContainerPort(serverContainer.Ports, "http", 8880) { t.Fatalf("server ports = %#v, want HTTP 8880", serverContainer.Ports) } + assertWorkloadHealthAndResources(t, &serverContainer) + if len(server.Spec.Template.Spec.TopologySpreadConstraints) != 1 { + t.Fatalf("server topology spread constraints = %#v", server.Spec.Template.Spec.TopologySpreadConstraints) + } + spread := server.Spec.Template.Spec.TopologySpreadConstraints[0] + if spread.TopologyKey != corev1.LabelHostname || spread.MaxSkew != 1 || spread.WhenUnsatisfiable != corev1.ScheduleAnyway { + t.Fatalf("server topology spread = %#v", spread) + } if got := server.Spec.Strategy.RollingUpdate; got == nil || got.MaxUnavailable == nil || got.MaxUnavailable.IntValue() != 0 { t.Fatalf("server maxUnavailable = %#v, want 0", got) } @@ -251,6 +260,14 @@ func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != 8880 || service.Spec.Ports[0].TargetPort.IntValue() != 8880 { t.Fatalf("service ports = %#v", service.Spec.Ports) } + + pdb := serverPodDisruptionBudget(site, component.DefaultNamespace) + if pdb.Spec.MinAvailable == nil || pdb.Spec.MinAvailable.IntValue() != 1 { + t.Fatalf("PDB minAvailable = %#v, want 1", pdb.Spec.MinAvailable) + } + if pdb.Spec.Selector == nil || pdb.Spec.Selector.MatchLabels["app.kubernetes.io/name"] != "metalman-server" { + t.Fatalf("PDB selector = %#v", pdb.Spec.Selector) + } } func TestEnsureCapabilitySecretPreservesExistingKey(t *testing.T) { @@ -385,6 +402,23 @@ func assertCapabilityKeyMount(t *testing.T, podSpec *corev1.PodSpec, container * t.Fatalf("missing read-only capability key mount: mounts=%#v volumes=%#v", container.VolumeMounts, podSpec.Volumes) } +func assertWorkloadHealthAndResources(t *testing.T, container *corev1.Container) { + t.Helper() + + for name, probe := range map[string]*corev1.Probe{"liveness": container.LivenessProbe, "readiness": container.ReadinessProbe} { + if probe == nil || probe.HTTPGet == nil || probe.HTTPGet.Port.IntValue() != 8081 { + t.Fatalf("%s probe = %#v, want HTTP probe on 8081", name, probe) + } + } + if container.LivenessProbe.HTTPGet.Path != "/healthz" || container.ReadinessProbe.HTTPGet.Path != "/readyz" { + t.Fatalf("probe paths = %q, %q", container.LivenessProbe.HTTPGet.Path, container.ReadinessProbe.HTTPGet.Path) + } + if container.Resources.Requests.Cpu().IsZero() || container.Resources.Requests.Memory().IsZero() || + container.Resources.Limits.Cpu().IsZero() || container.Resources.Limits.Memory().IsZero() { + t.Fatalf("resources are incomplete: %#v", container.Resources) + } +} + func assertSiteOwnerRef(t *testing.T, refs []metav1.OwnerReference, siteName, uid string) { t.Helper() From 2be1148f1209fac25b418b588dce126d4a81acd5 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:15:48 +0000 Subject: [PATCH 19/43] metalman: reconcile endpoint edge workloads --- deploy/unbounded-operator/02-rbac.yaml.tmpl | 6 + .../operator/components/metalman/metalman.go | 248 ++++++++++++++++++ .../components/metalman/metalman_test.go | 205 +++++++++++++++ 3 files changed, 459 insertions(+) diff --git a/deploy/unbounded-operator/02-rbac.yaml.tmpl b/deploy/unbounded-operator/02-rbac.yaml.tmpl index 66a3f9716..e84ebf48c 100644 --- a/deploy/unbounded-operator/02-rbac.yaml.tmpl +++ b/deploy/unbounded-operator/02-rbac.yaml.tmpl @@ -16,6 +16,9 @@ rules: - apiGroups: ["unbounded-cloud.io"] resources: ["sites/status"] verbs: ["get", "patch", "update"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints"] + verbs: ["get", "list", "watch"] # Legacy net-group Sites are read during migration and translated into the # machina-group Sites above; their finalizers are cleared (update/patch) so the # CRD can be deleted once drained. @@ -45,6 +48,9 @@ rules: - apiGroups: ["apps"] resources: ["deployments", "daemonsets"] verbs: ["get", "list", "watch", "create", "patch", "update", "delete", "deletecollection"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete", "deletecollection"] - apiGroups: ["apps"] resources: ["statefulsets", "replicasets"] verbs: ["get", "list", "watch"] diff --git a/internal/operator/components/metalman/metalman.go b/internal/operator/components/metalman/metalman.go index 3d597ddb4..6283197ef 100644 --- a/internal/operator/components/metalman/metalman.go +++ b/internal/operator/components/metalman/metalman.go @@ -8,6 +8,7 @@ import ( "context" "crypto/rand" "fmt" + "sort" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -17,8 +18,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" machinamanifests "github.com/Azure/unbounded/deploy/machina" @@ -65,6 +68,9 @@ func (Component) Reconcile(ctx context.Context, env *component.Env, site *unboun return component.Failed(err) } } + if err := reconcileEndpointEdges(ctx, env, site); err != nil { + return component.Failed(err) + } return component.Reconciled() } @@ -92,6 +98,14 @@ func (Component) Cleanup(ctx context.Context, env *component.Env, site *unbounde // its controller owner reference to the Site. The predicate drops status-only // updates so pod-count churn does not re-apply the Deployment. func (Component) SetupWatches(b *builder.Builder, env *component.Env) { + b.Watches(&unboundedv1alpha3.NetbootEndpoint{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []ctrl.Request { + endpoint, ok := obj.(*unboundedv1alpha3.NetbootEndpoint) + if !ok || endpoint.Spec.SiteRef == "" { + return nil + } + + return []ctrl.Request{{NamespacedName: client.ObjectKey{Name: endpoint.Spec.SiteRef}}} + })) b.Owns(&appsv1.Deployment{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) b.Owns(&corev1.Service{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) b.Owns(&policyv1.PodDisruptionBudget{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) @@ -141,6 +155,8 @@ func serverDeployment(site *unboundedv1alpha3.Site, namespace string, cfg compon const ( metalmanControllerRole = "controller" metalmanServerRole = "server" + metalmanEdgeRole = "edge" + netbootEndpointLabel = "unbounded-cloud.io/netboot-endpoint" ) func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config, role string, replicas int32) *appsv1.Deployment { @@ -338,3 +354,235 @@ func serverLabels(site string) map[string]string { unboundedv1alpha3.MachineSiteLabelKey: site, } } + +func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) error { + var endpoints unboundedv1alpha3.NetbootEndpointList + if err := env.Client.List(ctx, &endpoints); err != nil { + return fmt.Errorf("list NetbootEndpoints: %w", err) + } + sort.Slice(endpoints.Items, func(i, j int) bool { return endpoints.Items[i].Name < endpoints.Items[j].Name }) + + desiredDeployments := map[string]struct{}{} + desiredServices := map[string]struct{}{} + for i := range endpoints.Items { + endpoint := &endpoints.Items[i] + if endpoint.Spec.SiteRef != site.Name { + continue + } + deployment, service, err := endpointEdgeObjects(endpoint, site, env.Namespace, env.Config) + if err != nil { + return fmt.Errorf("build edge for NetbootEndpoint %s: %w", endpoint.Name, err) + } + if deployment != nil { + if err := env.ApplyObject(ctx, deployment); err != nil { + return err + } + desiredDeployments[deployment.Name] = struct{}{} + } + if service != nil { + if err := env.ApplyObject(ctx, service); err != nil { + return err + } + desiredServices[service.Name] = struct{}{} + } + } + match := client.MatchingLabels{ + "app.kubernetes.io/component": metalmanEdgeRole, + unboundedv1alpha3.MachineSiteLabelKey: site.Name, + } + var deployments appsv1.DeploymentList + if err := env.Client.List(ctx, &deployments, client.InNamespace(env.Namespace), match); err != nil { + return fmt.Errorf("list managed edge Deployments: %w", err) + } + for i := range deployments.Items { + if _, ok := desiredDeployments[deployments.Items[i].Name]; !ok { + if err := env.DeleteIfExists(ctx, &deployments.Items[i]); err != nil { + return err + } + } + } + var services corev1.ServiceList + if err := env.Client.List(ctx, &services, client.InNamespace(env.Namespace), match); err != nil { + return fmt.Errorf("list managed edge Services: %w", err) + } + for i := range services.Items { + if _, ok := desiredServices[services.Items[i].Name]; !ok { + if err := env.DeleteIfExists(ctx, &services.Items[i]); err != nil { + return err + } + } + } + + return nil +} + +func endpointEdgeObjects( + endpoint *unboundedv1alpha3.NetbootEndpoint, + site *unboundedv1alpha3.Site, + namespace string, + cfg component.Config, +) (*appsv1.Deployment, *corev1.Service, error) { + if endpoint.Spec.SiteRef != site.Name { + return nil, nil, fmt.Errorf("endpoint site %q does not match %q", endpoint.Spec.SiteRef, site.Name) + } + if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeExternalL2 { + return nil, nil, nil + } + + labels := endpointEdgeLabels(endpoint, site.Name) + replicas := int32(2) + name := EdgeName(endpoint.Name) + backendURL := fmt.Sprintf("http://%s.%s.svc:8880", ServerName(site.Name), namespace) + args := []string{ + metalmanEdgeRole, + "--backend-url=" + backendURL, + "--endpoint=" + endpoint.Name, + } + ports := []corev1.ContainerPort{{Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}} + podSpec := corev1.PodSpec{ServiceAccountName: "metalman-edge"} + maxSurge := intstr.FromInt32(1) + maxUnavailable := intstr.FromInt32(0) + strategy := appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + } + + switch endpoint.Spec.Type { + case unboundedv1alpha3.NetbootEndpointTypeManagedL2: + if endpoint.Spec.ManagedL2 == nil { + return nil, nil, fmt.Errorf("managedL2 configuration is required") + } + replicas = 1 + args = append(args, + "--bind-address="+endpoint.Spec.ManagedL2.Address, + "--dhcp-enabled", + "--dhcp-interface="+endpoint.Spec.ManagedL2.Interface, + "--dhcp-server-ip="+endpoint.Spec.ManagedL2.Address, + "--tftp-enabled", + "--tftp-bind-address="+endpoint.Spec.ManagedL2.Address, + ) + ports = append(ports, + corev1.ContainerPort{Name: "dhcp", ContainerPort: 67, Protocol: corev1.ProtocolUDP}, + corev1.ContainerPort{Name: "tftp", ContainerPort: 69, Protocol: corev1.ProtocolUDP}, + ) + affinity, err := requiredNodeAffinity(endpoint.Spec.ManagedL2.NodeSelector) + if err != nil { + return nil, nil, err + } + podSpec.HostNetwork = true + podSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + podSpec.Affinity = affinity + strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} + case unboundedv1alpha3.NetbootEndpointTypeHTTP: + if endpoint.Spec.HTTP == nil { + return nil, nil, fmt.Errorf("http configuration is required") + } + default: + return nil, nil, fmt.Errorf("unsupported endpoint type %q", endpoint.Spec.Type) + } + + expirationSeconds := int64(3600) + container := corev1.Container{ + Name: "metalman", + Image: cfg.Image("metalman"), + ImagePullPolicy: corev1.PullAlways, + Args: args, + Ports: ports, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("50m"), corev1.ResourceMemory: resource.MustParse("64Mi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + } + if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeManagedL2 { + container.VolumeMounts = []corev1.VolumeMount{{Name: "edge-token", MountPath: "/var/run/secrets/metalman", ReadOnly: true}} + podSpec.Volumes = []corev1.Volume{{Name: "edge-token", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "metalman-edge", + ExpirationSeconds: &expirationSeconds, + Path: "token", + }}}, + }}}} + } + podSpec.Containers = []corev1.Container{container} + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: strategy, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: podSpec}, + }, + } + + if endpoint.Spec.Type != unboundedv1alpha3.NetbootEndpointTypeHTTP { + return deployment, nil, nil + } + serviceType := endpoint.Spec.HTTP.ServiceType + if serviceType == "" { + serviceType = corev1.ServiceTypeClusterIP + } + service := &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: corev1.ServiceSpec{ + Type: serviceType, + Selector: labels, + Ports: []corev1.ServicePort{{Name: "http", Port: 8880, TargetPort: intstr.FromInt32(8880), Protocol: corev1.ProtocolTCP}}, + }, + } + + return deployment, service, nil +} + +// EdgeName returns the in-cluster workload name for an endpoint. +func EdgeName(endpoint string) string { return "metalman-edge-" + endpoint } + +func endpointEdgeLabels(endpoint *unboundedv1alpha3.NetbootEndpoint, site string) map[string]string { + return map[string]string{ + "app": "unbounded-metalman", + "app.kubernetes.io/name": "metalman-edge", + "app.kubernetes.io/component": metalmanEdgeRole, + unboundedv1alpha3.MachineSiteLabelKey: site, + netbootEndpointLabel: endpoint.Name, + } +} + +func requiredNodeAffinity(selector metav1.LabelSelector) (*corev1.Affinity, error) { + requirements := make([]corev1.NodeSelectorRequirement, 0, len(selector.MatchLabels)+len(selector.MatchExpressions)) + keys := make([]string, 0, len(selector.MatchLabels)) + for key := range selector.MatchLabels { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + requirements = append(requirements, corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{selector.MatchLabels[key]}}) + } + for _, expression := range selector.MatchExpressions { + operator := corev1.NodeSelectorOperator(expression.Operator) + switch operator { + case corev1.NodeSelectorOpIn, corev1.NodeSelectorOpNotIn, corev1.NodeSelectorOpExists, corev1.NodeSelectorOpDoesNotExist: + default: + return nil, fmt.Errorf("unsupported node selector operator %q", expression.Operator) + } + requirements = append(requirements, corev1.NodeSelectorRequirement{Key: expression.Key, Operator: operator, Values: expression.Values}) + } + + return &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: requirements}}, + }}}, nil +} diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index fefec7aca..8a178bb1d 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -116,6 +116,22 @@ func TestRBACSeparatesControllerAndServerIdentities(t *testing.T) { } } +func TestOperatorRBACCanReconcileNetbootEndpoints(t *testing.T) { + data, err := os.ReadFile("../../../../deploy/unbounded-operator/02-rbac.yaml.tmpl") + if err != nil { + t.Fatalf("read operator RBAC template: %v", err) + } + manifest := string(data) + for _, required := range []string{ + "resources: [\"netbootendpoints\"]", + "resources: [\"poddisruptionbudgets\"]", + } { + if !strings.Contains(manifest, required) { + t.Fatalf("operator RBAC template missing %q", required) + } + } +} + func TestDeployment(t *testing.T) { enabled := true site := &unboundedv1alpha3.Site{ @@ -270,6 +286,164 @@ func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { } } +func TestEndpointEdgeWorkloadMatrix(t *testing.T) { + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + cfg := component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"} + + t.Run("managed L2 is the only host-network edge", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a-lan"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeManagedL2, + ExternalURL: "http://192.0.2.10:8880", + ManagedL2: &unboundedv1alpha3.NetbootManagedL2Spec{ + NodeSelector: metav1.LabelSelector{MatchLabels: map[string]string{"provisioning-lan": "rack-a"}}, + Interface: "eno2", + Address: "192.0.2.10", + }, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + if deployment == nil || service != nil { + t.Fatalf("managed L2 objects = deployment %v, service %v", deployment, service) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("managed L2 replicas = %v, want 1", deployment.Spec.Replicas) + } + pod := &deployment.Spec.Template.Spec + if !pod.HostNetwork || pod.DNSPolicy != corev1.DNSClusterFirstWithHostNet { + t.Fatalf("managed L2 networking = hostNetwork %v, dnsPolicy %q", pod.HostNetwork, pod.DNSPolicy) + } + if pod.Affinity == nil || pod.Affinity.NodeAffinity == nil || + pod.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + t.Fatalf("managed L2 edge lacks required placement: %#v", pod.Affinity) + } + container := &pod.Containers[0] + for _, arg := range []string{ + "edge", + "--backend-url=http://metalman-server-rack-a." + component.DefaultNamespace + ".svc:8880", + "--endpoint=rack-a-lan", + "--dhcp-enabled", + "--dhcp-interface=eno2", + "--dhcp-server-ip=192.0.2.10", + "--tftp-enabled", + } { + if !containsString(container.Args, arg) { + t.Fatalf("managed L2 args %#v lack %q", container.Args, arg) + } + } + if pod.ServiceAccountName != "metalman-edge" { + t.Fatalf("managed L2 service account = %q", pod.ServiceAccountName) + } + assertProjectedEdgeToken(t, pod, container) + if !hasContainerPort(container.Ports, "http", 8880) || + !hasContainerPort(container.Ports, "dhcp", 67) || + !hasContainerPort(container.Ports, "tftp", 69) { + t.Fatalf("managed L2 ports = %#v", container.Ports) + } + }) + + t.Run("HTTP edge uses ordinary replicated pods and a Service", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-http"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + ExternalURL: "https://boot.example.com", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSExternal, + }, + HTTP: &unboundedv1alpha3.NetbootHTTPEndpointSpec{ServiceType: corev1.ServiceTypeNodePort}, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + if deployment == nil || service == nil { + t.Fatalf("HTTP objects = deployment %v, service %v", deployment, service) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 2 { + t.Fatalf("HTTP replicas = %v, want 2", deployment.Spec.Replicas) + } + assertOrdinaryPodNetworking(t, &deployment.Spec.Template.Spec) + args := deployment.Spec.Template.Spec.Containers[0].Args + if containsString(args, "--dhcp-enabled") || containsString(args, "--tftp-enabled") { + t.Fatalf("HTTP edge enables L2 protocols: %#v", args) + } + if service.Spec.Type != corev1.ServiceTypeNodePort || service.Spec.Selector[netbootEndpointLabel] != endpoint.Name { + t.Fatalf("HTTP edge Service = %#v", service.Spec) + } + }) + + t.Run("external L2 has no in-cluster workload", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "admin-laptop"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeExternalL2, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + if deployment != nil || service != nil { + t.Fatalf("external L2 objects = deployment %v, service %v", deployment, service) + } + }) +} + +func TestReconcileEndpointEdgesDeletesStaleManagedWorkloads(t *testing.T) { + scheme := testScheme(t) + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + staleLabels := map[string]string{ + "app.kubernetes.io/component": metalmanEdgeRole, + unboundedv1alpha3.MachineSiteLabelKey: site.Name, + netbootEndpointLabel: "removed-endpoint", + } + staleDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ + Name: EdgeName("removed-endpoint"), + Namespace: component.DefaultNamespace, + Labels: staleLabels, + }} + staleService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: EdgeName("removed-endpoint"), + Namespace: component.DefaultNamespace, + Labels: staleLabels, + }} + external := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "external-endpoint"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeExternalL2, + }, + } + env := &component.Env{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(staleDeployment, staleService, external).Build(), + Scheme: scheme, Namespace: component.DefaultNamespace, + Config: component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}, + } + + if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { + t.Fatalf("reconcileEndpointEdges: %v", err) + } + for _, object := range []client.Object{staleDeployment, staleService} { + err := env.Client.Get(t.Context(), client.ObjectKeyFromObject(object), object) + if !apierrors.IsNotFound(err) { + t.Fatalf("stale %T still exists: err=%v", object, err) + } + } +} + func TestEnsureCapabilitySecretPreservesExistingKey(t *testing.T) { scheme := testScheme(t) env := &component.Env{ @@ -442,6 +616,37 @@ func assertSiteOwnerRef(t *testing.T, refs []metav1.OwnerReference, siteName, ui } } +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + + return false +} + +func assertProjectedEdgeToken(t *testing.T, pod *corev1.PodSpec, container *corev1.Container) { + t.Helper() + + for _, mount := range container.VolumeMounts { + if mount.Name != "edge-token" || mount.MountPath != "/var/run/secrets/metalman" || !mount.ReadOnly { + continue + } + for _, volume := range pod.Volumes { + if volume.Name != mount.Name || volume.Projected == nil || len(volume.Projected.Sources) != 1 { + continue + } + token := volume.Projected.Sources[0].ServiceAccountToken + if token != nil && token.Audience == "metalman-edge" && token.Path == "token" { + return + } + } + } + + t.Fatalf("missing audience-bound projected edge token: mounts=%#v volumes=%#v", container.VolumeMounts, pod.Volumes) +} + func assertSiteAffinity(t *testing.T, affinity *corev1.Affinity, siteName string) { t.Helper() From d04bb40779ca29f62a0d57e0c11d377dccca13a7 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:19:33 +0000 Subject: [PATCH 20/43] metalman: support TLS edge listeners --- internal/metalman/commands/edge.go | 25 +++++++++++++++++++++- internal/metalman/commands/roles_test.go | 27 +++++++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go index 7fbea8042..d0e380f6f 100644 --- a/internal/metalman/commands/edge.go +++ b/internal/metalman/commands/edge.go @@ -35,6 +35,8 @@ func EdgeCmd() *cobra.Command { backendURL string bindAddress string httpPort int + tlsCertFile string + tlsKeyFile string endpoint string edgeTokenFile string dhcpEnabled bool @@ -54,6 +56,9 @@ func EdgeCmd() *cobra.Command { if err != nil { return err } + if err := validateEdgeTLSFiles(tlsCertFile, tlsKeyFile); err != nil { + return err + } ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -106,7 +111,7 @@ func EdgeCmd() *cobra.Command { PrintReady() slog.InfoContext(ctx, "starting Metalman edge", "addr", addr, "backend", backend.String()) - if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := serveEdgeHTTP(server, tlsCertFile, tlsKeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) { return fmt.Errorf("serving edge HTTP: %w", err) } @@ -117,6 +122,8 @@ func EdgeCmd() *cobra.Command { cmd.Flags().StringVar(&backendURL, "backend-url", "", "Metalman server base URL") cmd.Flags().StringVar(&bindAddress, "bind-address", "0.0.0.0", "IP address to bind the edge HTTP listener") cmd.Flags().IntVar(&httpPort, "http-port", 8880, "Port for the edge HTTP listener") + cmd.Flags().StringVar(&tlsCertFile, "tls-cert-file", "", "TLS certificate file; requires --tls-key-file") + cmd.Flags().StringVar(&tlsKeyFile, "tls-key-file", "", "TLS private key file; requires --tls-cert-file") cmd.Flags().StringVar(&endpoint, "endpoint", "", "NetbootEndpoint served by this edge") cmd.Flags().StringVar(&edgeTokenFile, "edge-token-file", "/var/run/secrets/metalman/token", "Audience-bound ServiceAccount token file") cmd.Flags().BoolVar(&dhcpEnabled, "dhcp-enabled", false, "Enable the DHCP protocol edge") @@ -132,6 +139,22 @@ func EdgeCmd() *cobra.Command { return cmd } +func validateEdgeTLSFiles(certFile, keyFile string) error { + if (certFile == "") != (keyFile == "") { + return errors.New("--tls-cert-file and --tls-key-file must be set together") + } + + return nil +} + +func serveEdgeHTTP(server *http.Server, certFile, keyFile string) error { + if certFile != "" { + return server.ListenAndServeTLS(certFile, keyFile) + } + + return server.ListenAndServe() +} + func edgeDHCPServerIP(configured, iface string) (net.IP, error) { if configured != "" { ip := net.ParseIP(configured).To4() diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index dbf0c8856..4fdb139d2 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -89,7 +89,7 @@ func TestEdgeCommandRequiresOnlyBackendConnection(t *testing.T) { t.Parallel() cmd := EdgeCmd() - for _, name := range []string{"backend-url", "bind-address", "http-port", "endpoint", "edge-token-file", "dhcp-enabled", "dhcp-interface", "dhcp-server-ip", "dhcp-port", "tftp-enabled", "tftp-bind-address", "tftp-port"} { + for _, name := range []string{"backend-url", "bind-address", "http-port", "tls-cert-file", "tls-key-file", "endpoint", "edge-token-file", "dhcp-enabled", "dhcp-interface", "dhcp-server-ip", "dhcp-port", "tftp-enabled", "tftp-bind-address", "tftp-port"} { if cmd.Flags().Lookup(name) == nil { t.Errorf("edge command has no --%s flag", name) } @@ -102,6 +102,31 @@ func TestEdgeCommandRequiresOnlyBackendConnection(t *testing.T) { } } +func TestEdgeTLSRequiresCertificateAndKeyTogether(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + certFile string + keyFile string + wantErr bool + }{ + {name: "plaintext"}, + {name: "TLS", certFile: "tls.crt", keyFile: "tls.key"}, + {name: "certificate only", certFile: "tls.crt", wantErr: true}, + {name: "key only", keyFile: "tls.key", wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := validateEdgeTLSFiles(tt.certFile, tt.keyFile) + if (err != nil) != tt.wantErr { + t.Fatalf("validateEdgeTLSFiles(%q, %q) error = %v, wantErr %v", tt.certFile, tt.keyFile, err, tt.wantErr) + } + }) + } +} + func TestEdgeProxyPreservesSessionPathAndRange(t *testing.T) { t.Parallel() From c8c6f11e511ba2c0d0d469d5f28cc9313f4961a6 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:24:03 +0000 Subject: [PATCH 21/43] metalman: terminate edge TLS from Secrets --- .../operator/components/metalman/metalman.go | 142 ++++++++++++++++- .../components/metalman/metalman_test.go | 150 ++++++++++++++++++ 2 files changed, 287 insertions(+), 5 deletions(-) diff --git a/internal/operator/components/metalman/metalman.go b/internal/operator/components/metalman/metalman.go index 6283197ef..c209c2520 100644 --- a/internal/operator/components/metalman/metalman.go +++ b/internal/operator/components/metalman/metalman.go @@ -7,6 +7,7 @@ package metalman import ( "context" "crypto/rand" + "crypto/sha256" "fmt" "sort" @@ -106,8 +107,17 @@ func (Component) SetupWatches(b *builder.Builder, env *component.Env) { return []ctrl.Request{{NamespacedName: client.ObjectKey{Name: endpoint.Spec.SiteRef}}} })) + b.Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []ctrl.Request { + secret, ok := obj.(*corev1.Secret) + if !ok { + return nil + } + + return requestsForTLSSecret(ctx, env.Client, secret) + })) b.Owns(&appsv1.Deployment{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) b.Owns(&corev1.Service{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) + b.Owns(&corev1.Secret{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) b.Owns(&policyv1.PodDisruptionBudget{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) } @@ -153,10 +163,11 @@ func serverDeployment(site *unboundedv1alpha3.Site, namespace string, cfg compon } const ( - metalmanControllerRole = "controller" - metalmanServerRole = "server" - metalmanEdgeRole = "edge" - netbootEndpointLabel = "unbounded-cloud.io/netboot-endpoint" + metalmanControllerRole = "controller" + metalmanServerRole = "server" + metalmanEdgeRole = "edge" + netbootEndpointLabel = "unbounded-cloud.io/netboot-endpoint" + edgeTLSChecksumAnnotation = "unbounded-cloud.io/tls-checksum" ) func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config, role string, replicas int32) *appsv1.Deployment { @@ -364,6 +375,7 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou desiredDeployments := map[string]struct{}{} desiredServices := map[string]struct{}{} + desiredTLSSecrets := map[string]struct{}{} for i := range endpoints.Items { endpoint := &endpoints.Items[i] if endpoint.Spec.SiteRef != site.Name { @@ -374,6 +386,19 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou return fmt.Errorf("build edge for NetbootEndpoint %s: %w", endpoint.Name, err) } if deployment != nil { + if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { + secret, err := mirroredTLSSecret(ctx, env.Client, endpoint, site, env.Namespace) + if err != nil { + return fmt.Errorf("mirror TLS Secret for NetbootEndpoint %s: %w", endpoint.Name, err) + } + if err := env.ApplyObject(ctx, secret); err != nil { + return err + } + desiredTLSSecrets[secret.Name] = struct{}{} + deployment.Spec.Template.Annotations = map[string]string{ + edgeTLSChecksumAnnotation: tlsSecretChecksum(secret), + } + } if err := env.ApplyObject(ctx, deployment); err != nil { return err } @@ -412,10 +437,96 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou } } } + var tlsSecrets corev1.SecretList + if err := env.Client.List(ctx, &tlsSecrets, client.InNamespace(env.Namespace), match); err != nil { + return fmt.Errorf("list managed edge TLS Secrets: %w", err) + } + for i := range tlsSecrets.Items { + if _, ok := desiredTLSSecrets[tlsSecrets.Items[i].Name]; !ok { + if err := env.DeleteIfExists(ctx, &tlsSecrets.Items[i]); err != nil { + return err + } + } + } return nil } +func tlsSecretChecksum(secret *corev1.Secret) string { + digest := sha256.New() + _, _ = digest.Write(secret.Data[corev1.TLSCertKey]) + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write(secret.Data[corev1.TLSPrivateKeyKey]) + + return fmt.Sprintf("%x", digest.Sum(nil)) +} + +func mirroredTLSSecret( + ctx context.Context, + kubeClient client.Client, + endpoint *unboundedv1alpha3.NetbootEndpoint, + site *unboundedv1alpha3.Site, + namespace string, +) (*corev1.Secret, error) { + if endpoint.Spec.TLS.SecretRef == nil { + return nil, fmt.Errorf("TLS secretRef is required") + } + ref := endpoint.Spec.TLS.SecretRef + source := &corev1.Secret{} + if err := kubeClient.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, source); err != nil { + return nil, fmt.Errorf("get source Secret %s/%s: %w", ref.Namespace, ref.Name, err) + } + certificate, certOK := source.Data[corev1.TLSCertKey] + privateKey, keyOK := source.Data[corev1.TLSPrivateKeyKey] + if !certOK || !keyOK { + return nil, fmt.Errorf("source Secret %s/%s must contain %s and %s", ref.Namespace, ref.Name, corev1.TLSCertKey, corev1.TLSPrivateKeyKey) + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EdgeTLSSecretName(endpoint.Name), + Namespace: namespace, + Labels: endpointEdgeLabels(endpoint, site.Name), + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: append([]byte(nil), certificate...), + corev1.TLSPrivateKeyKey: append([]byte(nil), privateKey...), + }, + }, nil +} + +func requestsForTLSSecret(ctx context.Context, kubeClient client.Client, secret *corev1.Secret) []ctrl.Request { + var endpoints unboundedv1alpha3.NetbootEndpointList + if err := kubeClient.List(ctx, &endpoints); err != nil { + return nil + } + + sites := map[string]struct{}{} + for i := range endpoints.Items { + ref := endpoints.Items[i].Spec.TLS.SecretRef + if endpoints.Items[i].Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret && ref != nil && + ref.Namespace == secret.Namespace && ref.Name == secret.Name { + sites[endpoints.Items[i].Spec.SiteRef] = struct{}{} + } + } + names := make([]string, 0, len(sites)) + for site := range sites { + if site != "" { + names = append(names, site) + } + } + sort.Strings(names) + requests := make([]ctrl.Request, 0, len(names)) + for _, site := range names { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKey{Name: site}}) + } + + return requests +} + func endpointEdgeObjects( endpoint *unboundedv1alpha3.NetbootEndpoint, site *unboundedv1alpha3.Site, @@ -506,6 +617,19 @@ func endpointEdgeObjects( }}}, }}}} } + if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { + args = append(args, + "--tls-cert-file=/var/run/secrets/metalman-tls/tls.crt", + "--tls-key-file=/var/run/secrets/metalman-tls/tls.key", + ) + container.Args = args + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: "tls", MountPath: "/var/run/secrets/metalman-tls", ReadOnly: true, + }) + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: EdgeTLSSecretName(endpoint.Name)}}, + }) + } podSpec.Containers = []corev1.Container{container} deployment := &appsv1.Deployment{ @@ -531,6 +655,11 @@ func endpointEdgeObjects( if serviceType == "" { serviceType = corev1.ServiceTypeClusterIP } + servicePort := corev1.ServicePort{Name: "http", Port: 8880, TargetPort: intstr.FromInt32(8880), Protocol: corev1.ProtocolTCP} + if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { + servicePort.Name = "https" + servicePort.Port = 443 + } service := &corev1.Service{ TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, ObjectMeta: metav1.ObjectMeta{ @@ -542,7 +671,7 @@ func endpointEdgeObjects( Spec: corev1.ServiceSpec{ Type: serviceType, Selector: labels, - Ports: []corev1.ServicePort{{Name: "http", Port: 8880, TargetPort: intstr.FromInt32(8880), Protocol: corev1.ProtocolTCP}}, + Ports: []corev1.ServicePort{servicePort}, }, } @@ -552,6 +681,9 @@ func endpointEdgeObjects( // EdgeName returns the in-cluster workload name for an endpoint. func EdgeName(endpoint string) string { return "metalman-edge-" + endpoint } +// EdgeTLSSecretName returns the mirrored serving-certificate Secret name. +func EdgeTLSSecretName(endpoint string) string { return EdgeName(endpoint) + "-tls" } + func endpointEdgeLabels(endpoint *unboundedv1alpha3.NetbootEndpoint, site string) map[string]string { return map[string]string{ "app": "unbounded-metalman", diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index 8a178bb1d..ea4e01bad 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -383,6 +383,51 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { } }) + t.Run("Secret TLS mounts the mirrored certificate", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-https"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + ExternalURL: "https://boot.example.com", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSSecret, + SecretRef: &unboundedv1alpha3.NamespacedSecretReference{ + Namespace: "certificates", + Name: "boot-example-com", + }, + }, + HTTP: &unboundedv1alpha3.NetbootHTTPEndpointSpec{}, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + container := &deployment.Spec.Template.Spec.Containers[0] + for _, arg := range []string{ + "--tls-cert-file=/var/run/secrets/metalman-tls/tls.crt", + "--tls-key-file=/var/run/secrets/metalman-tls/tls.key", + } { + if !containsString(container.Args, arg) { + t.Fatalf("Secret TLS args %#v lack %q", container.Args, arg) + } + } + if len(deployment.Spec.Template.Spec.Volumes) != 1 || + deployment.Spec.Template.Spec.Volumes[0].Secret == nil || + deployment.Spec.Template.Spec.Volumes[0].Secret.SecretName != EdgeTLSSecretName(endpoint.Name) { + t.Fatalf("Secret TLS volumes = %#v", deployment.Spec.Template.Spec.Volumes) + } + if len(container.VolumeMounts) != 1 || container.VolumeMounts[0].MountPath != "/var/run/secrets/metalman-tls" || !container.VolumeMounts[0].ReadOnly { + t.Fatalf("Secret TLS mounts = %#v", container.VolumeMounts) + } + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Name != "https" || service.Spec.Ports[0].Port != 443 { + t.Fatalf("Secret TLS Service ports = %#v", service.Spec.Ports) + } + }) + t.Run("external L2 has no in-cluster workload", func(t *testing.T) { endpoint := &unboundedv1alpha3.NetbootEndpoint{ ObjectMeta: metav1.ObjectMeta{Name: "admin-laptop"}, @@ -402,6 +447,111 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { }) } +func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { + scheme := testScheme(t) + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + source := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "boot-example-com", Namespace: "certificates"}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{corev1.TLSCertKey: []byte("certificate-v1"), corev1.TLSPrivateKeyKey: []byte("private-key-v1")}, + } + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-https"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + ExternalURL: "https://boot.example.com", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSSecret, + SecretRef: &unboundedv1alpha3.NamespacedSecretReference{Namespace: source.Namespace, Name: source.Name}, + }, + HTTP: &unboundedv1alpha3.NetbootHTTPEndpointSpec{}, + }, + } + env := &component.Env{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(source, endpoint).Build(), + Scheme: scheme, Namespace: component.DefaultNamespace, + Config: component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}, + } + + assertMirror := func(wantCert string) string { + t.Helper() + + if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { + t.Fatalf("reconcileEndpointEdges: %v", err) + } + mirror := &corev1.Secret{} + key := client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeTLSSecretName(endpoint.Name)} + if err := env.Client.Get(t.Context(), key, mirror); err != nil { + t.Fatalf("get mirrored TLS Secret: %v", err) + } + if got := string(mirror.Data[corev1.TLSCertKey]); got != wantCert { + t.Fatalf("mirrored certificate = %q, want %q", got, wantCert) + } + if got := string(mirror.Data[corev1.TLSPrivateKeyKey]); got != "private-key-v1" { + t.Fatalf("mirrored private key = %q", got) + } + deployment := &appsv1.Deployment{} + if err := env.Client.Get(t.Context(), client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeName(endpoint.Name)}, deployment); err != nil { + t.Fatalf("get TLS edge Deployment: %v", err) + } + checksum := deployment.Spec.Template.Annotations[edgeTLSChecksumAnnotation] + if checksum == "" { + t.Fatal("TLS edge pod template lacks certificate checksum") + } + + return checksum + } + + firstChecksum := assertMirror("certificate-v1") + source.Data[corev1.TLSCertKey] = []byte("certificate-v2") + if err := env.Client.Update(t.Context(), source); err != nil { + t.Fatalf("update source TLS Secret: %v", err) + } + if secondChecksum := assertMirror("certificate-v2"); secondChecksum == firstChecksum { + t.Fatalf("TLS edge checksum did not change after certificate rotation: %q", secondChecksum) + } + + endpoint.Spec.TLS = unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSExternal, + } + if err := env.Client.Update(t.Context(), endpoint); err != nil { + t.Fatalf("update endpoint TLS mode: %v", err) + } + if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { + t.Fatalf("reconcile external TLS endpoint: %v", err) + } + err := env.Client.Get(t.Context(), client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeTLSSecretName(endpoint.Name)}, &corev1.Secret{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("mirrored TLS Secret still exists: %v", err) + } +} + +func TestTLSSecretChangeEnqueuesReferencingSite(t *testing.T) { + scheme := testScheme(t) + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-https"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: "rack-a", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Mode: unboundedv1alpha3.NetbootEndpointTLSSecret, + SecretRef: &unboundedv1alpha3.NamespacedSecretReference{Namespace: "certificates", Name: "boot-example-com"}, + }, + }, + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(endpoint).Build() + + requests := requestsForTLSSecret(t.Context(), kubeClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Namespace: "certificates", + Name: "boot-example-com", + }}) + if len(requests) != 1 || requests[0].Name != "rack-a" { + t.Fatalf("requests = %#v, want rack-a", requests) + } +} + func TestReconcileEndpointEdgesDeletesStaleManagedWorkloads(t *testing.T) { scheme := testScheme(t) site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} From ebffa62fb1b28a65c2c4ac382a176d18732eeebe Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:28:19 +0000 Subject: [PATCH 22/43] metalman: publish managed endpoint readiness --- deploy/unbounded-operator/02-rbac.yaml.tmpl | 3 + .../operator/components/metalman/metalman.go | 47 ++++++++++ .../components/metalman/metalman_test.go | 94 ++++++++++++++++++- 3 files changed, 143 insertions(+), 1 deletion(-) diff --git a/deploy/unbounded-operator/02-rbac.yaml.tmpl b/deploy/unbounded-operator/02-rbac.yaml.tmpl index e84ebf48c..0157e7f1d 100644 --- a/deploy/unbounded-operator/02-rbac.yaml.tmpl +++ b/deploy/unbounded-operator/02-rbac.yaml.tmpl @@ -19,6 +19,9 @@ rules: - apiGroups: ["unbounded-cloud.io"] resources: ["netbootendpoints"] verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints/status"] + verbs: ["get", "patch", "update"] # Legacy net-group Sites are read during migration and translated into the # machina-group Sites above; their finalizers are cleared (update/patch) so the # CRD can be deleted once drained. diff --git a/internal/operator/components/metalman/metalman.go b/internal/operator/components/metalman/metalman.go index c209c2520..9b8d05244 100644 --- a/internal/operator/components/metalman/metalman.go +++ b/internal/operator/components/metalman/metalman.go @@ -15,10 +15,12 @@ import ( corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -402,6 +404,13 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou if err := env.ApplyObject(ctx, deployment); err != nil { return err } + liveDeployment := &appsv1.Deployment{} + if err := env.Client.Get(ctx, client.ObjectKeyFromObject(deployment), liveDeployment); err != nil { + return fmt.Errorf("get managed edge Deployment %s: %w", deployment.Name, err) + } + if err := reconcileManagedEndpointStatus(ctx, env.Client, endpoint, liveDeployment); err != nil { + return fmt.Errorf("update NetbootEndpoint %s status: %w", endpoint.Name, err) + } desiredDeployments[deployment.Name] = struct{}{} } if service != nil { @@ -452,6 +461,44 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou return nil } +func reconcileManagedEndpointStatus( + ctx context.Context, + kubeClient client.Client, + endpoint *unboundedv1alpha3.NetbootEndpoint, + deployment *appsv1.Deployment, +) error { + if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeExternalL2 { + return nil + } + if deployment == nil { + return fmt.Errorf("managed endpoint has no Deployment") + } + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + current := &unboundedv1alpha3.NetbootEndpoint{} + if err := kubeClient.Get(ctx, client.ObjectKey{Name: endpoint.Name}, current); err != nil { + return err + } + base := current.DeepCopy() + current.Status.ObservedGeneration = current.Generation + condition := metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "EdgeUnavailable", + Message: "managed edge Deployment is not available", + ObservedGeneration: current.Generation, + } + if deployment.Status.ObservedGeneration >= deployment.Generation && deployment.Status.AvailableReplicas > 0 { + condition.Status = metav1.ConditionTrue + condition.Reason = "EdgeAvailable" + condition.Message = "managed edge Deployment is available" + } + apimeta.SetStatusCondition(¤t.Status.Conditions, condition) + + return kubeClient.Status().Patch(ctx, current, client.MergeFrom(base)) + }) +} + func tlsSecretChecksum(secret *corev1.Secret) string { digest := sha256.New() _, _ = digest.Write(secret.Data[corev1.TLSCertKey]) diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index ea4e01bad..0f124a7fb 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -124,6 +124,7 @@ func TestOperatorRBACCanReconcileNetbootEndpoints(t *testing.T) { manifest := string(data) for _, required := range []string{ "resources: [\"netbootendpoints\"]", + "resources: [\"netbootendpoints/status\"]", "resources: [\"poddisruptionbudgets\"]", } { if !strings.Contains(manifest, required) { @@ -132,6 +133,94 @@ func TestOperatorRBACCanReconcileNetbootEndpoints(t *testing.T) { } } +func TestReconcileManagedEndpointStatusFromDeploymentAvailability(t *testing.T) { + scheme := testScheme(t) + now := metav1.Now() + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-http", Generation: 3}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: "rack-a", + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + }, + Status: unboundedv1alpha3.NetbootEndpointStatus{Claim: &unboundedv1alpha3.NetbootEndpointClaim{ + HolderIdentity: "edge-pod-1", + RenewedAt: now, + }}, + } + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: EdgeName(endpoint.Name), Namespace: component.DefaultNamespace, Generation: 7}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 7, AvailableReplicas: 1}, + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(endpoint).WithObjects(endpoint, deployment).Build() + + if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, endpoint, deployment); err != nil { + t.Fatalf("reconcileManagedEndpointStatus: %v", err) + } + updated := &unboundedv1alpha3.NetbootEndpoint{} + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { + t.Fatalf("get endpoint: %v", err) + } + if updated.Status.ObservedGeneration != endpoint.Generation { + t.Fatalf("observed generation = %d, want %d", updated.Status.ObservedGeneration, endpoint.Generation) + } + ready := findCondition(updated.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != "EdgeAvailable" { + t.Fatalf("Ready condition = %#v", ready) + } + if updated.Status.Claim == nil || updated.Status.Claim.HolderIdentity != "edge-pod-1" || updated.Status.Claim.RenewedAt.Time.Unix() != now.Time.Unix() { + t.Fatalf("claim was not preserved: %#v", updated.Status.Claim) + } + + deployment.Status.AvailableReplicas = 0 + if err := kubeClient.Status().Update(t.Context(), deployment); err != nil { + t.Fatalf("update Deployment status: %v", err) + } + if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, updated, deployment); err != nil { + t.Fatalf("reconcile unavailable endpoint: %v", err) + } + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { + t.Fatalf("get unavailable endpoint: %v", err) + } + ready = findCondition(updated.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "EdgeUnavailable" { + t.Fatalf("unavailable Ready condition = %#v", ready) + } +} + +func TestReconcileManagedEndpointStatusLeavesExternalClaimUntouched(t *testing.T) { + scheme := testScheme(t) + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "admin-laptop", Generation: 2}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{SiteRef: "rack-a", Type: unboundedv1alpha3.NetbootEndpointTypeExternalL2}, + Status: unboundedv1alpha3.NetbootEndpointStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "ExternalEdgeReady"}}, + }, + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(endpoint).WithObjects(endpoint).Build() + + if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, endpoint, nil); err != nil { + t.Fatalf("reconcileManagedEndpointStatus: %v", err) + } + updated := &unboundedv1alpha3.NetbootEndpoint{} + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { + t.Fatal(err) + } + if updated.Status.ObservedGeneration != 1 || updated.Status.Conditions[0].Reason != "ExternalEdgeReady" { + t.Fatalf("external endpoint status changed: %#v", updated.Status) + } +} + +func findCondition(conditions []metav1.Condition, conditionType string) *metav1.Condition { + for i := range conditions { + if conditions[i].Type == conditionType { + return &conditions[i] + } + } + + return nil +} + func TestDeployment(t *testing.T) { enabled := true site := &unboundedv1alpha3.Site{ @@ -470,7 +559,7 @@ func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { }, } env := &component.Env{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(source, endpoint).Build(), + Client: fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(endpoint).WithObjects(source, endpoint).Build(), Scheme: scheme, Namespace: component.DefaultNamespace, Config: component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}, } @@ -513,6 +602,9 @@ func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { t.Fatalf("TLS edge checksum did not change after certificate rotation: %q", secondChecksum) } + if err := env.Client.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, endpoint); err != nil { + t.Fatalf("refresh endpoint: %v", err) + } endpoint.Spec.TLS = unboundedv1alpha3.NetbootEndpointTLS{ Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, Mode: unboundedv1alpha3.NetbootEndpointTLSExternal, From c36cd5291d10fefa148de6a98ddf63d42bacba2d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:33:33 +0000 Subject: [PATCH 23/43] metalman: remove obsolete Site tuning --- api/machina/v1alpha3/site_types.go | 10 --- api/machina/v1alpha3/site_types_test.go | 33 ++++++-- api/machina/v1alpha3/zz_generated.deepcopy.go | 10 --- .../machina/crd/unbounded-cloud.io_sites.yaml | 11 --- e2e/operator/reaper_e2e_test.go | 4 +- internal/operator/migrate.go | 82 +------------------ internal/operator/migrate_gates_test.go | 12 +-- internal/operator/migrate_test.go | 60 +++++--------- 8 files changed, 56 insertions(+), 166 deletions(-) diff --git a/api/machina/v1alpha3/site_types.go b/api/machina/v1alpha3/site_types.go index a07907167..e368f4e7d 100644 --- a/api/machina/v1alpha3/site_types.go +++ b/api/machina/v1alpha3/site_types.go @@ -155,16 +155,6 @@ type MachinaComponentSpec struct { // MetalmanComponentSpec configures Metalman for a site. type MetalmanComponentSpec struct { SiteComponentSpec `json:",inline"` - - // DHCPAutoInterface lets Metalman choose the DHCP interface automatically. - // +optional - DHCPAutoInterface *bool `json:"dhcpAutoInterface,omitempty"` - - // Replicas is the desired number of Metalman replicas. Defaults to 1 when - // omitted. - // +kubebuilder:validation:Minimum=0 - // +optional - Replicas *int32 `json:"replicas,omitempty"` } // StorageComponentSpec configures unbounded-storage for a site. Storage daemon diff --git a/api/machina/v1alpha3/site_types_test.go b/api/machina/v1alpha3/site_types_test.go index 6cbf7a22b..07c6318d8 100644 --- a/api/machina/v1alpha3/site_types_test.go +++ b/api/machina/v1alpha3/site_types_test.go @@ -4,15 +4,40 @@ package v1alpha3 import ( + "os" "testing" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" ) +func TestMetalmanComponentSchemaExposesOnlyEnablement(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("../../../deploy/machina/crd/unbounded-cloud.io_sites.yaml") + if err != nil { + t.Fatalf("read Site CRD: %v", err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("parse Site CRD: %v", err) + } + metalman := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"].Properties["components"].Properties["metalman"] + if _, ok := metalman.Properties["enabled"]; !ok { + t.Error("Metalman component schema lacks enabled") + } + for _, field := range []string{"dhcpAutoInterface", "replicas"} { + if _, ok := metalman.Properties[field]; ok { + t.Errorf("Metalman component schema still exposes %q", field) + } + } +} + func TestSiteResourceAndAddToScheme(t *testing.T) { gr := Resource("sites") if gr.Group != GroupVersion.Group || gr.Resource != "sites" { @@ -40,7 +65,6 @@ func TestDeepCopySiteAndList(t *testing.T) { enabled := true priority := int32(10) detectMultiplier := int32(3) - replicas := int32(3) receive := intstr.FromString("300ms") transmit := intstr.FromInt(400) @@ -72,8 +96,6 @@ func TestDeepCopySiteAndList(t *testing.T) { }, Metalman: &MetalmanComponentSpec{ SiteComponentSpec: SiteComponentSpec{Enabled: &enabled}, - DHCPAutoInterface: &enabled, - Replicas: &replicas, }, }, }, @@ -98,7 +120,6 @@ func TestDeepCopySiteAndList(t *testing.T) { site.Spec.NodeCidrs[0] = "10.99.0.0/16" site.Spec.PodCidrAssignments[0].CidrBlocks[0] = "10.250.0.0/16" site.Spec.HealthCheckSettings.DetectMultiplier = ptrInt32(9) - site.Spec.Components.Metalman.Replicas = ptrInt32(5) site.Status.Conditions[0].Status = metav1.ConditionFalse if copied.Spec.NodeCidrs[0] != "10.0.0.0/16" { @@ -113,10 +134,6 @@ func TestDeepCopySiteAndList(t *testing.T) { t.Fatalf("expected deep-copied health check settings to be isolated") } - if copied.Spec.Components.Metalman.Replicas == nil || *copied.Spec.Components.Metalman.Replicas != 3 { - t.Fatalf("expected deep-copied Metalman replicas to be isolated") - } - if copied.Status.Conditions[0].Status != metav1.ConditionTrue { t.Fatalf("expected deep-copied component condition to be isolated") } diff --git a/api/machina/v1alpha3/zz_generated.deepcopy.go b/api/machina/v1alpha3/zz_generated.deepcopy.go index 0630c6e96..b0a0b54b4 100644 --- a/api/machina/v1alpha3/zz_generated.deepcopy.go +++ b/api/machina/v1alpha3/zz_generated.deepcopy.go @@ -1123,16 +1123,6 @@ func (in *MachineStatus) DeepCopy() *MachineStatus { func (in *MetalmanComponentSpec) DeepCopyInto(out *MetalmanComponentSpec) { *out = *in in.SiteComponentSpec.DeepCopyInto(&out.SiteComponentSpec) - if in.DHCPAutoInterface != nil { - in, out := &in.DHCPAutoInterface, &out.DHCPAutoInterface - *out = new(bool) - **out = **in - } - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetalmanComponentSpec. diff --git a/deploy/machina/crd/unbounded-cloud.io_sites.yaml b/deploy/machina/crd/unbounded-cloud.io_sites.yaml index d584feddb..a78e80ed8 100644 --- a/deploy/machina/crd/unbounded-cloud.io_sites.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_sites.yaml @@ -108,20 +108,9 @@ spec: description: Metalman configures the Metalman PXE controller for this site. properties: - dhcpAutoInterface: - description: DHCPAutoInterface lets Metalman choose the DHCP - interface automatically. - type: boolean enabled: description: Enabled controls whether the component is reconciled. type: boolean - replicas: - description: |- - Replicas is the desired number of Metalman replicas. Defaults to 1 when - omitted. - format: int32 - minimum: 0 - type: integer type: object storage: description: Storage configures the unbounded-storage supervisor diff --git a/e2e/operator/reaper_e2e_test.go b/e2e/operator/reaper_e2e_test.go index 975e95986..b9102f60c 100644 --- a/e2e/operator/reaper_e2e_test.go +++ b/e2e/operator/reaper_e2e_test.go @@ -462,8 +462,8 @@ func assertTranslatedSites(ctx context.Context, t *testing.T, cli client.Client, t.Fatalf("expected metalman enabled on edge site") } - if !nestedBool(edge, "spec", "components", "metalman", "dhcpAutoInterface") { - t.Fatalf("expected Metalman DHCP auto-interface mode preserved") + if _, found, err := unstructured.NestedBool(edge.Object, "spec", "components", "metalman", "dhcpAutoInterface"); err != nil || found { + t.Fatalf("removed Metalman DHCP auto-interface mode retained: found=%t err=%v", found, err) } repair := getMachinaSite(ctx, t, cli, fixture.siteName) diff --git a/internal/operator/migrate.go b/internal/operator/migrate.go index 402be2def..ffb10e6b4 100644 --- a/internal/operator/migrate.go +++ b/internal/operator/migrate.go @@ -542,18 +542,13 @@ func (r *LegacyReaper) detectComponents(ctx context.Context, siteName string) (m } } - metalman, dhcpAutoInterface, err := r.legacyMetalmanConfigForSite(ctx, siteName) + metalman, err := r.legacyMetalmanExistsForSite(ctx, siteName) if err != nil { return nil, err } if metalman { - config := map[string]any{"enabled": true} - if dhcpAutoInterface != nil { - config["dhcpAutoInterface"] = *dhcpAutoInterface - } - - components["metalman"] = config + components["metalman"] = map[string]any{"enabled": true} } return components, nil @@ -567,28 +562,13 @@ func (r *LegacyReaper) detectComponents(ctx context.Context, siteName string) (m // and the operator use) guards against the site label being carried under the // deprecated key on older clusters. func (r *LegacyReaper) legacyMetalmanExistsForSite(ctx context.Context, siteName string) (bool, error) { - found, _, err := r.legacyMetalmanConfigForSite(ctx, siteName) - - return found, err -} - -// legacyMetalmanConfigForSite finds the legacy per-site Metalman Deployment and -// preserves its --dhcp-auto-interface setting. Malformed or contradictory flag -// values block translation rather than silently changing DHCP behavior. -func (r *LegacyReaper) legacyMetalmanConfigForSite(ctx context.Context, siteName string) (bool, *bool, error) { reader := r.liveReader() - - var ( - dhcpAutoInterface *bool - effectiveValue *bool - ) - found := false for _, legacyNs := range r.LegacyNamespaces { var list appsv1.DeploymentList if err := reader.List(ctx, &list, client.InNamespace(legacyNs), client.MatchingLabels{"app": "unbounded-pxe"}); err != nil { - return false, nil, err + return false, err } for i := range list.Items { @@ -600,62 +580,6 @@ func (r *LegacyReaper) legacyMetalmanConfigForSite(ctx context.Context, siteName } found = true - - value, err := metalmanDHCPAutoInterface(d) - if err != nil { - return false, nil, err - } - - effective := false - if value != nil { - effective = *value - } - - if effectiveValue != nil && *effectiveValue != effective { - return false, nil, fmt.Errorf("legacy Metalman Deployments for Site %s have conflicting --dhcp-auto-interface values", siteName) - } - - matchedEffective := effective - effectiveValue = &matchedEffective - - if value != nil { - dhcpAutoInterface = value - } - } - } - - return found, dhcpAutoInterface, nil -} - -func metalmanDHCPAutoInterface(deploy *appsv1.Deployment) (*bool, error) { - var found *bool - - for _, container := range deploy.Spec.Template.Spec.Containers { - for _, arg := range container.Args { - var value bool - - switch { - case arg == "--dhcp-auto-interface": - value = true - case strings.HasPrefix(arg, "--dhcp-auto-interface="): - switch strings.TrimPrefix(arg, "--dhcp-auto-interface=") { - case "true": - value = true - case "false": - value = false - default: - return nil, fmt.Errorf("legacy Metalman Deployment %s/%s has invalid argument %q", deploy.Namespace, deploy.Name, arg) - } - default: - continue - } - - if found != nil && *found != value { - return nil, fmt.Errorf("legacy Metalman Deployment %s/%s has conflicting --dhcp-auto-interface arguments", deploy.Namespace, deploy.Name) - } - - matched := value - found = &matched } } diff --git a/internal/operator/migrate_gates_test.go b/internal/operator/migrate_gates_test.go index 9c1675eb4..c83f2a064 100644 --- a/internal/operator/migrate_gates_test.go +++ b/internal/operator/migrate_gates_test.go @@ -377,25 +377,25 @@ func TestLegacyMetalmanDetectionHardening(t *testing.T) { } }) - t.Run("conflicting matching Deployments fail closed", func(t *testing.T) { + t.Run("conflicting removed arguments do not block detection", func(t *testing.T) { byName := metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", "--dhcp-auto-interface") byLabel := metalmanDeploymentForSiteWithArgs(legacyNetNamespace, "edge", "--dhcp-auto-interface=false") byLabel.Name = "older-metalman-name" r := newReaper(t, byName, byLabel) - if _, _, err := r.legacyMetalmanConfigForSite(t.Context(), "edge"); err == nil { - t.Fatal("expected conflicting matching Metalman Deployments to fail closed") + if got, err := r.legacyMetalmanExistsForSite(t.Context(), "edge"); err != nil || !got { + t.Fatalf("legacyMetalmanExistsForSite = %t, err=%v; want true", got, err) } }) - t.Run("absent and enabled arguments conflict", func(t *testing.T) { + t.Run("absent and enabled removed arguments do not conflict", func(t *testing.T) { withoutFlag := metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", "serve-pxe") withFlag := metalmanDeploymentForSiteWithArgs(legacyNetNamespace, "edge", "--dhcp-auto-interface") withFlag.Name = "older-metalman-name" r := newReaper(t, withoutFlag, withFlag) - if _, _, err := r.legacyMetalmanConfigForSite(t.Context(), "edge"); err == nil { - t.Fatal("expected absent and enabled Metalman arguments to conflict") + if got, err := r.legacyMetalmanExistsForSite(t.Context(), "edge"); err != nil || !got { + t.Fatalf("legacyMetalmanExistsForSite = %t, err=%v; want true", got, err) } }) } diff --git a/internal/operator/migrate_test.go b/internal/operator/migrate_test.go index cd71d013d..607f308ae 100644 --- a/internal/operator/migrate_test.go +++ b/internal/operator/migrate_test.go @@ -230,15 +230,14 @@ func TestTranslateSitesCreatesMachinaSite(t *testing.T) { } } -func TestTranslateSitesPreservesMetalmanDHCPAutoInterface(t *testing.T) { +func TestTranslateSitesDropsLegacyMetalmanDHCPAutoInterface(t *testing.T) { tests := []struct { name string arg string - want bool }{ - {name: "bare", arg: "--dhcp-auto-interface", want: true}, - {name: "explicit true", arg: "--dhcp-auto-interface=true", want: true}, - {name: "explicit false", arg: "--dhcp-auto-interface=false", want: false}, + {name: "bare", arg: "--dhcp-auto-interface"}, + {name: "explicit true", arg: "--dhcp-auto-interface=true"}, + {name: "explicit false", arg: "--dhcp-auto-interface=false"}, } for _, tt := range tests { @@ -259,53 +258,34 @@ func TestTranslateSitesPreservesMetalmanDHCPAutoInterface(t *testing.T) { t.Fatalf("get translated Site: %v", err) } - value, found, err := unstructured.NestedBool(got.Object, "spec", "components", "metalman", "dhcpAutoInterface") - if err != nil || !found || value != tt.want { - t.Fatalf("dhcpAutoInterface = %t, found=%t, err=%v; want %t", value, found, err, tt.want) + _, found, err := unstructured.NestedBool(got.Object, "spec", "components", "metalman", "dhcpAutoInterface") + if err != nil || found { + t.Fatalf("dhcpAutoInterface found=%t, err=%v; want removed", found, err) } }) } } -func TestMetalmanDHCPAutoInterface(t *testing.T) { +func TestLegacyMetalmanDetectionIgnoresRemovedDHCPFlag(t *testing.T) { tests := []struct { - name string - args []string - want *bool - wantErr bool + name string + args []string }{ {name: "absent", args: []string{"serve-pxe"}}, - {name: "bare", args: []string{"--dhcp-auto-interface"}, want: boolPtr(true)}, - {name: "explicit true", args: []string{"--dhcp-auto-interface=true"}, want: boolPtr(true)}, - {name: "explicit false", args: []string{"--dhcp-auto-interface=false"}, want: boolPtr(false)}, - {name: "repeated same value", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=true"}, want: boolPtr(true)}, - {name: "invalid", args: []string{"--dhcp-auto-interface=yes"}, wantErr: true}, - {name: "conflicting", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=false"}, wantErr: true}, + {name: "bare", args: []string{"--dhcp-auto-interface"}}, + {name: "explicit true", args: []string{"--dhcp-auto-interface=true"}}, + {name: "explicit false", args: []string{"--dhcp-auto-interface=false"}}, + {name: "repeated same value", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=true"}}, + {name: "invalid", args: []string{"--dhcp-auto-interface=yes"}}, + {name: "conflicting", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=false"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - deploy := metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", tt.args...) - - got, err := metalmanDHCPAutoInterface(deploy) - if (err != nil) != tt.wantErr { - t.Fatalf("metalmanDHCPAutoInterface error = %v, wantErr %t", err, tt.wantErr) - } - - if tt.wantErr { - return - } - - if got == nil || tt.want == nil { - if got != nil || tt.want != nil { - t.Fatalf("metalmanDHCPAutoInterface = %v, want %v", got, tt.want) - } - - return - } - - if *got != *tt.want { - t.Fatalf("metalmanDHCPAutoInterface = %t, want %t", *got, *tt.want) + r := newReaper(t, metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", tt.args...)) + got, err := r.legacyMetalmanExistsForSite(t.Context(), "edge") + if err != nil || !got { + t.Fatalf("legacyMetalmanExistsForSite = %t, err=%v; want true", got, err) } }) } From 3bc12111f3089b36988066f1af18be1be44b70a6 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:46:06 +0000 Subject: [PATCH 24/43] kubectl: add netboot bootstrap command --- cmd/kubectl-unbounded/app/cmd_site.go | 3 +- .../app/site_bootstrap_netboot.go | 67 +++++++++++++++++++ .../app/site_bootstrap_netboot_test.go | 38 +++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 cmd/kubectl-unbounded/app/site_bootstrap_netboot.go create mode 100644 cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go diff --git a/cmd/kubectl-unbounded/app/cmd_site.go b/cmd/kubectl-unbounded/app/cmd_site.go index 0a184aaa4..8ca5fd547 100644 --- a/cmd/kubectl-unbounded/app/cmd_site.go +++ b/cmd/kubectl-unbounded/app/cmd_site.go @@ -14,7 +14,8 @@ func siteCommandGroup() *cobra.Command { } cmd.AddCommand( - siteInitCommand()) + siteInitCommand(), + siteBootstrapNetbootCommand(nil)) return cmd } diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go new file mode 100644 index 000000000..84f400b70 --- /dev/null +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package app + +import ( + "context" + "time" + + "github.com/spf13/cobra" + + "github.com/Azure/unbounded/internal/unbounded" +) + +const defaultBootstrapNetbootTimeout = 30 * time.Minute + +type siteBootstrapNetbootHandler struct { + site string + machine string + interfaceName string + address string + endpointName string + httpPort int + kubeconfigPath string + namespace string + metalmanBinary string + timeout time.Duration + routedCIDRs []string +} + +func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Command { + if handler == nil { + handler = &siteBootstrapNetbootHandler{} + } + + cmd := &cobra.Command{ + Use: "bootstrap-netboot SITE", + Short: "Temporarily serve netboot from this machine until the first Site node is Ready", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + handler.site = args[0] + + return handler.execute(cmd.Context()) + }, + } + + cmd.Flags().StringVar(&handler.machine, "machine", "", "Machine whose Node readiness completes bootstrap") + cmd.Flags().StringVar(&handler.interfaceName, "interface", "", "Provisioning network interface for DHCP and TFTP") + cmd.Flags().StringVar(&handler.address, "address", "", "Address on the provisioning interface advertised to clients") + cmd.Flags().StringVar(&handler.endpointName, "endpoint-name", "", "Ephemeral NetbootEndpoint name (generated when omitted)") + cmd.Flags().IntVar(&handler.httpPort, "http-port", 8880, "Local HTTP artifact port") + cmd.Flags().StringVar(&handler.kubeconfigPath, "kubeconfig", "", "Path to kubeconfig file") + cmd.Flags().StringVar(&handler.namespace, "namespace", unbounded.SystemNamespace(), "Namespace containing Metalman workloads") + cmd.Flags().StringVar(&handler.metalmanBinary, "metalman-binary", "", "Path to the metalman binary") + cmd.Flags().DurationVar(&handler.timeout, "timeout", defaultBootstrapNetbootTimeout, "Maximum time to wait for the designated Node to become Ready") + cmd.Flags().StringSliceVar(&handler.routedCIDRs, "routed-cidr", nil, "CIDR routed through an ephemeral external gateway (repeatable)") + + _ = cmd.MarkFlagRequired("machine") + _ = cmd.MarkFlagRequired("interface") + _ = cmd.MarkFlagRequired("address") + + return cmd +} + +func (h *siteBootstrapNetbootHandler) execute(_ context.Context) error { + return nil +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go new file mode 100644 index 000000000..425bfe8a9 --- /dev/null +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package app + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSiteBootstrapNetbootCommandContract(t *testing.T) { + group := siteCommandGroup() + + cmd, _, err := group.Find([]string{"bootstrap-netboot"}) + require.NoError(t, err) + require.Equal(t, "bootstrap-netboot SITE", cmd.Use) + + for _, name := range []string{ + "machine", + "interface", + "address", + "endpoint-name", + "http-port", + "kubeconfig", + "namespace", + "metalman-binary", + "timeout", + "routed-cidr", + } { + require.NotNilf(t, cmd.Flags().Lookup(name), "missing --%s", name) + } + + for _, name := range []string{"machine", "interface", "address"} { + flag := cmd.Flags().Lookup(name) + require.Contains(t, flag.Annotations, "cobra_annotation_bash_completion_one_required_flag") + } +} From a84548a0c0ed76d52594d3c16884360f9f51bd7a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:47:56 +0000 Subject: [PATCH 25/43] kubectl: prepare bootstrap netboot resources --- .../app/site_bootstrap_netboot.go | 96 +++++++++++++++++++ .../app/site_bootstrap_netboot_test.go | 90 +++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index 84f400b70..fbcf581e2 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -5,10 +5,18 @@ package app import ( "context" + "fmt" + "net" + "net/url" + "strconv" "time" "github.com/spf13/cobra" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" "github.com/Azure/unbounded/internal/unbounded" ) @@ -26,6 +34,11 @@ type siteBootstrapNetbootHandler struct { metalmanBinary string timeout time.Duration routedCIDRs []string + resources client.Client +} + +type bootstrapNetbootState struct { + originalEndpointRef string } func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Command { @@ -65,3 +78,86 @@ func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Co func (h *siteBootstrapNetbootHandler) execute(_ context.Context) error { return nil } + +func (h *siteBootstrapNetbootHandler) prepareClusterResources(ctx context.Context) (bootstrapNetbootState, error) { + var site v1alpha3.Site + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.site}, &site); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("get Site %s: %w", h.site, err) + } + + enabled := true + if site.Spec.Components.Metalman == nil { + site.Spec.Components.Metalman = &v1alpha3.MetalmanComponentSpec{} + } + site.Spec.Components.Metalman.Enabled = &enabled + + if err := h.resources.Update(ctx, &site); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("enable Metalman for Site %s: %w", h.site, err) + } + + var machine v1alpha3.Machine + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.machine}, &machine); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("get Machine %s: %w", h.machine, err) + } + + if machine.Labels[v1alpha3.MachineSiteLabelKey] != h.site { + return bootstrapNetbootState{}, fmt.Errorf("Machine %s does not belong to Site %s", h.machine, h.site) + } + + netboot := machine.Spec.Netboot() + if netboot == nil { + return bootstrapNetbootState{}, fmt.Errorf("Machine %s has no netboot configuration", h.machine) + } + + state := bootstrapNetbootState{originalEndpointRef: netboot.EndpointRef} + externalURL := (&url.URL{ + Scheme: "http", + Host: net.JoinHostPort(h.address, strconv.Itoa(h.httpPort)), + }).String() + endpoint := &v1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: v1alpha3.NetbootEndpointSpec{ + SiteRef: h.site, + Type: v1alpha3.NetbootEndpointTypeExternalL2, + ExternalURL: externalURL, + TLS: v1alpha3.NetbootEndpointTLS{ + Trust: v1alpha3.NetbootEndpointTrustTrustedLAN, + Mode: v1alpha3.NetbootEndpointTLSDisabled, + }, + }, + } + + if err := h.resources.Create(ctx, endpoint); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("create NetbootEndpoint %s: %w", h.endpointName, err) + } + + netboot.EndpointRef = h.endpointName + if err := h.resources.Update(ctx, &machine); err != nil { + _ = h.resources.Delete(ctx, endpoint) + + return bootstrapNetbootState{}, fmt.Errorf("select bootstrap endpoint for Machine %s: %w", h.machine, err) + } + + return state, nil +} + +func (h *siteBootstrapNetbootHandler) restoreClusterResources(ctx context.Context, state bootstrapNetbootState) error { + var machine v1alpha3.Machine + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.machine}, &machine); err != nil { + return fmt.Errorf("get Machine %s for cleanup: %w", h.machine, err) + } + + if netboot := machine.Spec.Netboot(); netboot != nil && netboot.EndpointRef == h.endpointName { + netboot.EndpointRef = state.originalEndpointRef + if err := h.resources.Update(ctx, &machine); err != nil { + return fmt.Errorf("restore Machine %s endpoint: %w", h.machine, err) + } + } + + endpoint := &v1alpha3.NetbootEndpoint{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + if err := h.resources.Delete(ctx, endpoint); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete NetbootEndpoint %s: %w", h.endpointName, err) + } + + return nil +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index 425bfe8a9..825f588fd 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -4,9 +4,15 @@ package app import ( + "context" "testing" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" ) func TestSiteBootstrapNetbootCommandContract(t *testing.T) { @@ -36,3 +42,87 @@ func TestSiteBootstrapNetbootCommandContract(t *testing.T) { require.Contains(t, flag.Annotations, "cobra_annotation_bash_completion_one_required_flag") } } + +func TestBootstrapNetbootPreparesAndRestoresClusterResources(t *testing.T) { + ctx := context.Background() + site := &v1alpha3.Site{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}, + Spec: v1alpha3.SiteSpec{ + Components: v1alpha3.SiteComponents{}, + }, + } + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-node", + Labels: map[string]string{v1alpha3.MachineSiteLabelKey: site.Name}, + }, + Spec: v1alpha3.MachineSpec{Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{ + EndpointRef: "permanent-edge", + }}}, + } + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).WithObjects(site, machine).Build() + h := &siteBootstrapNetbootHandler{ + site: site.Name, + machine: machine.Name, + address: "192.0.2.10", + httpPort: 8880, + endpointName: "bootstrap-first-node", + resources: resources, + } + + state, err := h.prepareClusterResources(ctx) + require.NoError(t, err) + require.Equal(t, "permanent-edge", state.originalEndpointRef) + + var gotSite v1alpha3.Site + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: site.Name}, &gotSite)) + require.NotNil(t, gotSite.Spec.Components.Metalman) + require.NotNil(t, gotSite.Spec.Components.Metalman.Enabled) + require.True(t, *gotSite.Spec.Components.Metalman.Enabled) + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, h.endpointName, gotMachine.Spec.Netboot().EndpointRef) + + var endpoint v1alpha3.NetbootEndpoint + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) + require.Equal(t, v1alpha3.NetbootEndpointTypeExternalL2, endpoint.Spec.Type) + require.Equal(t, site.Name, endpoint.Spec.SiteRef) + require.Equal(t, "http://192.0.2.10:8880", endpoint.Spec.ExternalURL) + require.Equal(t, v1alpha3.NetbootEndpointTrustTrustedLAN, endpoint.Spec.TLS.Trust) + + require.NoError(t, h.restoreClusterResources(ctx, state)) + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: site.Name}, &gotSite)) + require.True(t, *gotSite.Spec.Components.Metalman.Enabled) +} + +func TestBootstrapNetbootEndpointCollisionDoesNotMutateMachine(t *testing.T) { + ctx := context.Background() + enabled := true + site := &v1alpha3.Site{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}, + Spec: v1alpha3.SiteSpec{Components: v1alpha3.SiteComponents{ + Metalman: &v1alpha3.MetalmanComponentSpec{SiteComponentSpec: v1alpha3.SiteComponentSpec{Enabled: &enabled}}, + }}, + } + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "first-node", Labels: map[string]string{v1alpha3.MachineSiteLabelKey: site.Name}}, + Spec: v1alpha3.MachineSpec{Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{EndpointRef: "permanent-edge"}}}, + } + existing := &v1alpha3.NetbootEndpoint{ObjectMeta: metav1.ObjectMeta{Name: "bootstrap-first-node"}} + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).WithObjects(site, machine, existing).Build() + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, address: "192.0.2.10", httpPort: 8880, + endpointName: existing.Name, resources: resources, + } + + _, err := h.prepareClusterResources(ctx) + require.Error(t, err) + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) +} From b90d48aebee345d09fc1dee54424646bee69080c Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:49:15 +0000 Subject: [PATCH 26/43] kubectl: wait for bootstrap control plane --- .../app/site_bootstrap_netboot.go | 51 ++++++++++++++- .../app/site_bootstrap_netboot_test.go | 63 +++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index fbcf581e2..b4b94b51c 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -14,13 +14,17 @@ import ( "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" "github.com/Azure/unbounded/internal/unbounded" ) -const defaultBootstrapNetbootTimeout = 30 * time.Minute +const ( + defaultBootstrapNetbootTimeout = 30 * time.Minute + defaultBootstrapPollInterval = time.Second +) type siteBootstrapNetbootHandler struct { site string @@ -35,6 +39,8 @@ type siteBootstrapNetbootHandler struct { timeout time.Duration routedCIDRs []string resources client.Client + kubeClient kubernetes.Interface + pollInterval time.Duration } type bootstrapNetbootState struct { @@ -161,3 +167,46 @@ func (h *siteBootstrapNetbootHandler) restoreClusterResources(ctx context.Contex return nil } + +func (h *siteBootstrapNetbootHandler) metalmanDeploymentsReady(ctx context.Context) (bool, error) { + for _, name := range []string{"metalman-controller-" + h.site, "metalman-server-" + h.site} { + deployment, err := h.kubeClient.AppsV1().Deployments(h.namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get Deployment %s/%s: %w", h.namespace, name, err) + } + if !deploymentRolloutComplete(deployment) { + return false, nil + } + } + + return true, nil +} + +func (h *siteBootstrapNetbootHandler) waitForMetalman(ctx context.Context) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + ready, err := h.metalmanDeploymentsReady(ctx) + if err != nil { + return err + } + if ready { + return nil + } + + select { + case <-ctx.Done(): + return fmt.Errorf("wait for Metalman controller and server: %w", ctx.Err()) + case <-ticker.C: + } + } +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index 825f588fd..d4c6f8b67 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -6,9 +6,13 @@ package app import ( "context" "testing" + "time" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/fake" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -126,3 +130,62 @@ func TestBootstrapNetbootEndpointCollisionDoesNotMutateMachine(t *testing.T) { require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) } + +func TestBootstrapNetbootMetalmanReadinessRequiresCurrentControllerAndServers(t *testing.T) { + controller := readyDeployment("metalman-controller-rack-a", 1) + server := readyDeployment("metalman-server-rack-a", 2) + server.Status.ObservedGeneration-- + kubeClient := fake.NewSimpleClientset(controller, server) + h := &siteBootstrapNetbootHandler{site: "rack-a", namespace: "unbounded-system", kubeClient: kubeClient} + + ready, err := h.metalmanDeploymentsReady(context.Background()) + require.NoError(t, err) + require.False(t, ready) + + server.Status.ObservedGeneration = server.Generation + _, err = kubeClient.AppsV1().Deployments(h.namespace).UpdateStatus(context.Background(), server, metav1.UpdateOptions{}) + require.NoError(t, err) + + ready, err = h.metalmanDeploymentsReady(context.Background()) + require.NoError(t, err) + require.True(t, ready) +} + +func TestBootstrapNetbootWaitsUntilMetalmanRolloutCompletes(t *testing.T) { + controller := readyDeployment("metalman-controller-rack-a", 1) + server := readyDeployment("metalman-server-rack-a", 2) + server.Status.AvailableReplicas = 1 + kubeClient := fake.NewSimpleClientset(controller, server) + h := &siteBootstrapNetbootHandler{ + site: "rack-a", namespace: "unbounded-system", kubeClient: kubeClient, + pollInterval: time.Millisecond, + } + + go func() { + time.Sleep(5 * time.Millisecond) + server.Status.AvailableReplicas = 2 + _, _ = kubeClient.AppsV1().Deployments(h.namespace).UpdateStatus(context.Background(), server, metav1.UpdateOptions{}) + }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, h.waitForMetalman(ctx)) +} + +func readyDeployment(name string, replicas int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "unbounded-system", Generation: 2}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, + }}, + }, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, + Replicas: replicas, + UpdatedReplicas: replicas, + AvailableReplicas: replicas, + }, + } +} From 1fc2e73d7b5a90fdc1dc8e3e329757f6c995601a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:50:08 +0000 Subject: [PATCH 27/43] kubectl: constrain bootstrap edge process --- .../app/site_bootstrap_netboot.go | 16 ++++++++++ .../app/site_bootstrap_netboot_test.go | 30 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index b4b94b51c..d887dddfd 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -210,3 +210,19 @@ func (h *siteBootstrapNetbootHandler) waitForMetalman(ctx context.Context) error } } } + +func (h *siteBootstrapNetbootHandler) edgeArguments(backendURL, tokenFile string) []string { + return []string{ + "edge", + "--backend-url=" + backendURL, + "--endpoint=" + h.endpointName, + "--bind-address=" + h.address, + "--http-port=" + strconv.Itoa(h.httpPort), + "--dhcp-enabled", + "--dhcp-interface=" + h.interfaceName, + "--dhcp-server-ip=" + h.address, + "--edge-token-file=" + tokenFile, + "--tftp-enabled", + "--tftp-bind-address=" + h.address, + } +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index d4c6f8b67..f4fe81d87 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -189,3 +189,33 @@ func readyDeployment(name string, replicas int32) *appsv1.Deployment { }, } } + +func TestBootstrapNetbootEdgeArgumentsContainOnlyDataPlaneConfiguration(t *testing.T) { + h := &siteBootstrapNetbootHandler{ + endpointName: "bootstrap-first-node", + interfaceName: "eno1", + address: "192.0.2.10", + httpPort: 8880, + } + + args := h.edgeArguments("http://127.0.0.1:32123", "/run/token") + require.Equal(t, []string{ + "edge", + "--backend-url=http://127.0.0.1:32123", + "--endpoint=bootstrap-first-node", + "--bind-address=192.0.2.10", + "--http-port=8880", + "--dhcp-enabled", + "--dhcp-interface=eno1", + "--dhcp-server-ip=192.0.2.10", + "--edge-token-file=/run/token", + "--tftp-enabled", + "--tftp-bind-address=192.0.2.10", + }, args) + + for _, arg := range args { + require.NotContains(t, arg, "--site") + require.NotContains(t, arg, "--cache-dir") + require.NotContains(t, arg, "leader-elect") + } +} From 901655db7ec4d020f8c5fd918f2e27e41466d5e0 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:51:53 +0000 Subject: [PATCH 28/43] kubectl: gate bootstrap handoff on node readiness --- .../app/site_bootstrap_netboot.go | 85 +++++++++++++++++++ .../app/site_bootstrap_netboot_test.go | 63 ++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index d887dddfd..85ebbfc11 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -12,6 +12,7 @@ import ( "time" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -226,3 +227,87 @@ func (h *siteBootstrapNetbootHandler) edgeArguments(backendURL, tokenFile string "--tftp-bind-address=" + h.address, } } + +func (h *siteBootstrapNetbootHandler) claimEndpoint(ctx context.Context, identity string) error { + var endpoint v1alpha3.NetbootEndpoint + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint); err != nil { + return fmt.Errorf("get NetbootEndpoint %s: %w", h.endpointName, err) + } + + now := metav1.Now() + endpoint.Status.ObservedGeneration = endpoint.Generation + endpoint.Status.Claim = &v1alpha3.NetbootEndpointClaim{ + HolderIdentity: identity, + RenewedAt: now, + } + metaSetStatusCondition(&endpoint.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + ObservedGeneration: endpoint.Generation, + Reason: "ExternalEdgeReady", + Message: "administrator bootstrap edge is ready", + LastTransitionTime: now, + }) + + if err := h.resources.Status().Update(ctx, &endpoint); err != nil { + return fmt.Errorf("claim NetbootEndpoint %s: %w", h.endpointName, err) + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) waitForNodeReady(ctx context.Context, nodeName string) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + node, err := h.kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("get designated Node %s: %w", nodeName, err) + } + if err == nil && nodeReady(node) { + return nil + } + + select { + case <-ctx.Done(): + return fmt.Errorf("wait for designated Node %s to become Ready: %w", nodeName, ctx.Err()) + case <-ticker.C: + } + } +} + +func nodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + + return false +} + +func designatedNodeName(machine *v1alpha3.Machine) string { + if machine.Spec.Kubernetes != nil && machine.Spec.Kubernetes.NodeRef != nil && machine.Spec.Kubernetes.NodeRef.Name != "" { + return machine.Spec.Kubernetes.NodeRef.Name + } + + return machine.Name +} + +func metaSetStatusCondition(conditions *[]metav1.Condition, condition metav1.Condition) { + for i := range *conditions { + if (*conditions)[i].Type == condition.Type { + (*conditions)[i] = condition + + return + } + } + + *conditions = append(*conditions, condition) +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index f4fe81d87..21307f2de 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" @@ -219,3 +220,65 @@ func TestBootstrapNetbootEdgeArgumentsContainOnlyDataPlaneConfiguration(t *testi require.NotContains(t, arg, "leader-elect") } } + +func TestBootstrapNetbootClaimsEndpointAndWaitsForDesignatedNode(t *testing.T) { + ctx := context.Background() + endpoint := &v1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "bootstrap-first-node", Generation: 3}, + Spec: v1alpha3.NetbootEndpointSpec{SiteRef: "rack-a"}, + } + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).WithStatusSubresource(endpoint).WithObjects(endpoint).Build() + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{v1alpha3.MachineSiteLabelKey: "rack-a"}, + }} + kubeClient := fake.NewSimpleClientset(node) + h := &siteBootstrapNetbootHandler{ + site: "rack-a", endpointName: endpoint.Name, resources: resources, kubeClient: kubeClient, + pollInterval: time.Millisecond, + } + + require.NoError(t, h.claimEndpoint(ctx, "bootstrap/123")) + var claimed v1alpha3.NetbootEndpoint + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: endpoint.Name}, &claimed)) + require.Equal(t, endpoint.Generation, claimed.Status.ObservedGeneration) + require.Equal(t, "bootstrap/123", claimed.Status.Claim.HolderIdentity) + require.Equal(t, metav1.ConditionTrue, claimed.Status.Conditions[0].Status) + + waitCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + ready := make(chan error, 1) + go func() { ready <- h.waitForNodeReady(waitCtx, node.Name) }() + + time.Sleep(5 * time.Millisecond) + other := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "other", Labels: map[string]string{v1alpha3.MachineSiteLabelKey: "rack-a"}}, + Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}, + } + _, err := kubeClient.CoreV1().Nodes().Create(ctx, other, metav1.CreateOptions{}) + require.NoError(t, err) + + select { + case err := <-ready: + require.Failf(t, "wait returned for wrong Node", "error: %v", err) + case <-time.After(5 * time.Millisecond): + } + + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + _, err = kubeClient.CoreV1().Nodes().UpdateStatus(ctx, node, metav1.UpdateOptions{}) + require.NoError(t, err) + require.NoError(t, <-ready) +} + +func TestBootstrapNetbootDesignatedNodeNameUsesMachineNodeRef(t *testing.T) { + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "machine-name"}, + Spec: v1alpha3.MachineSpec{Kubernetes: &v1alpha3.KubernetesSpec{ + NodeRef: &v1alpha3.LocalObjectReference{Name: "node-name"}, + }}, + } + + require.Equal(t, "node-name", designatedNodeName(machine)) + machine.Spec.Kubernetes.NodeRef = nil + require.Equal(t, "machine-name", designatedNodeName(machine)) +} From a8b67e0ced2cfbdae59397cf8ecc6a87eedde260 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:55:57 +0000 Subject: [PATCH 29/43] kubectl: reconnect bootstrap server forwarding --- .../app/site_bootstrap_netboot.go | 237 ++++++++++++++++++ .../app/site_bootstrap_netboot_test.go | 136 ++++++++++ 2 files changed, 373 insertions(+) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index 85ebbfc11..92f1f3f9e 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -6,9 +6,13 @@ package app import ( "context" "fmt" + "io" "net" + "net/http" "net/url" + "sort" "strconv" + "sync" "time" "github.com/spf13/cobra" @@ -16,6 +20,9 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" @@ -48,6 +55,35 @@ type bootstrapNetbootState struct { originalEndpointRef string } +type bootstrapPortForwardAttempt interface { + Done() <-chan error + Stop() +} + +type bootstrapPortForwardStarter func( + ctx context.Context, + podName string, + localPort int, + remotePort int, +) (bootstrapPortForwardAttempt, error) + +type bootstrapPortForward struct { + url string + cancel context.CancelFunc + done chan struct{} +} + +func (f *bootstrapPortForward) URL() string { + return f.url +} + +func (f *bootstrapPortForward) Close() error { + f.cancel() + <-f.done + + return nil +} + func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Command { if handler == nil { handler = &siteBootstrapNetbootHandler{} @@ -311,3 +347,204 @@ func metaSetStatusCondition(conditions *[]metav1.Condition, condition metav1.Con *conditions = append(*conditions, condition) } + +func newBootstrapPortForward( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + deploymentName string, + localPort int, + remotePort int, + start bootstrapPortForwardStarter, + retryInterval time.Duration, +) (*bootstrapPortForward, error) { + if retryInterval <= 0 { + retryInterval = defaultBootstrapPollInterval + } + + forwardCtx, cancel := context.WithCancel(ctx) + podName, err := readyDeploymentPod(forwardCtx, kubeClient, namespace, deploymentName, "") + if err != nil { + cancel() + + return nil, err + } + + attempt, err := start(forwardCtx, podName, localPort, remotePort) + if err != nil { + cancel() + + return nil, fmt.Errorf("start port-forward to Pod %s: %w", podName, err) + } + + forward := &bootstrapPortForward{ + url: "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(localPort)), + cancel: cancel, + done: make(chan struct{}), + } + + go func() { + defer close(forward.done) + + current := attempt + for { + select { + case <-forwardCtx.Done(): + current.Stop() + <-current.Done() + + return + case <-current.Done(): + } + + for { + select { + case <-forwardCtx.Done(): + current.Stop() + + return + case <-time.After(retryInterval): + } + + podName, err = readyDeploymentPod(forwardCtx, kubeClient, namespace, deploymentName, podName) + if err != nil { + continue + } + + current, err = start(forwardCtx, podName, localPort, remotePort) + if err == nil { + break + } + } + } + }() + + return forward, nil +} + +func readyDeploymentPod( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + deploymentName string, + excludedPod string, +) (string, error) { + deployment, err := kubeClient.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("get Deployment %s/%s for port-forward: %w", namespace, deploymentName, err) + } + + selector := metav1.FormatLabelSelector(deployment.Spec.Selector) + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return "", fmt.Errorf("list Pods for Deployment %s/%s: %w", namespace, deploymentName, err) + } + + ready := make([]string, 0, len(pods.Items)) + excludedReady := false + for i := range pods.Items { + pod := &pods.Items[i] + if pod.DeletionTimestamp == nil && podReady(pod) { + if pod.Name == excludedPod { + excludedReady = true + + continue + } + ready = append(ready, pod.Name) + } + } + if len(ready) == 0 && excludedReady { + ready = append(ready, excludedPod) + } + if len(ready) == 0 { + return "", fmt.Errorf("Deployment %s/%s has no Ready Pods", namespace, deploymentName) + } + + sort.Strings(ready) + + return ready[0], nil +} + +func podReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + + return false +} + +type spdyBootstrapPortForwardAttempt struct { + stop chan struct{} + done chan error + once sync.Once +} + +func (a *spdyBootstrapPortForwardAttempt) Done() <-chan error { + return a.done +} + +func (a *spdyBootstrapPortForwardAttempt) Stop() { + a.once.Do(func() { close(a.stop) }) +} + +func newSPDYBootstrapPortForwardStarter( + kubeClient kubernetes.Interface, + config *rest.Config, + namespace string, +) bootstrapPortForwardStarter { + return func( + ctx context.Context, + podName string, + localPort int, + remotePort int, + ) (bootstrapPortForwardAttempt, error) { + targetURL := kubeClient.CoreV1().RESTClient().Post(). + Resource("pods"). + Namespace(namespace). + Name(podName). + SubResource("portforward"). + URL() + transport, upgrader, err := spdy.RoundTripperFor(config) + if err != nil { + return nil, fmt.Errorf("create SPDY transport: %w", err) + } + + stop := make(chan struct{}) + ready := make(chan struct{}) + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, targetURL) + forwarder, err := portforward.NewOnAddresses( + dialer, + []string{"127.0.0.1"}, + []string{fmt.Sprintf("%d:%d", localPort, remotePort)}, + stop, + ready, + io.Discard, + io.Discard, + ) + if err != nil { + return nil, fmt.Errorf("create port-forward: %w", err) + } + + attempt := &spdyBootstrapPortForwardAttempt{stop: stop, done: make(chan error, 1)} + go func() { attempt.done <- forwarder.ForwardPorts() }() + + select { + case <-ctx.Done(): + attempt.Stop() + + return nil, ctx.Err() + case err := <-attempt.done: + attempt.Stop() + + return nil, fmt.Errorf("port-forward exited before becoming ready: %w", err) + case <-ready: + return attempt, nil + case <-time.After(30 * time.Second): + attempt.Stop() + + return nil, fmt.Errorf("port-forward to Pod %s timed out", podName) + } + } +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index 21307f2de..cf9669c85 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -5,6 +5,9 @@ package app import ( "context" + "errors" + "fmt" + "sync" "testing" "time" @@ -282,3 +285,136 @@ func TestBootstrapNetbootDesignatedNodeNameUsesMachineNodeRef(t *testing.T) { machine.Spec.Kubernetes.NodeRef = nil require.Equal(t, "machine-name", designatedNodeName(machine)) } + +func TestBootstrapPortForwardReconnectsToReadyServerPod(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + deployment := readyDeployment("metalman-server-rack-a", 2) + deployment.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"app": "metalman-server"}} + unready := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "server-a", Namespace: deployment.Namespace, Labels: deployment.Spec.Selector.MatchLabels}, + } + ready := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "server-b", Namespace: deployment.Namespace, Labels: deployment.Spec.Selector.MatchLabels}, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, Status: corev1.ConditionTrue, + }}}, + } + replacement := ready.DeepCopy() + replacement.Name = "server-c" + kubeClient := fake.NewSimpleClientset(deployment, unready, ready, replacement) + starter := &fakeBootstrapPortForwardStarter{started: make(chan string, 4)} + + forward, err := newBootstrapPortForward( + ctx, + kubeClient, + deployment.Namespace, + deployment.Name, + 32123, + 8880, + starter.start, + time.Millisecond, + ) + require.NoError(t, err) + require.Equal(t, "http://127.0.0.1:32123", forward.URL()) + require.Equal(t, "server-b", <-starter.started) + + starter.fail("server-b", errors.New("connection lost")) + select { + case podName := <-starter.started: + require.Equal(t, "server-c", podName) + case <-time.After(time.Second): + require.Fail(t, "port-forward did not reconnect") + } + + require.NoError(t, forward.Close()) + require.Equal(t, []int{32123, 32123}, starter.ports()) +} + +func TestBootstrapPortForwardClosesWhileWaitingToReconnect(t *testing.T) { + deployment := readyDeployment("metalman-server-rack-a", 1) + deployment.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"app": "metalman-server"}} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "server-a", Namespace: deployment.Namespace, Labels: deployment.Spec.Selector.MatchLabels}, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, Status: corev1.ConditionTrue, + }}}, + } + starter := &fakeBootstrapPortForwardStarter{started: make(chan string, 2)} + forward, err := newBootstrapPortForward( + context.Background(), + fake.NewSimpleClientset(deployment, pod), + deployment.Namespace, + deployment.Name, + 32123, + 8880, + starter.start, + time.Hour, + ) + require.NoError(t, err) + require.Equal(t, pod.Name, <-starter.started) + starter.fail(pod.Name, errors.New("connection lost")) + time.Sleep(time.Millisecond) + + closed := make(chan error, 1) + go func() { closed <- forward.Close() }() + select { + case err := <-closed: + require.NoError(t, err) + case <-time.After(time.Second): + require.Fail(t, "port-forward close blocked during reconnect") + } +} + +type fakeBootstrapPortForwardStarter struct { + mu sync.Mutex + attempts map[string]*fakeBootstrapPortForwardAttempt + local []int + started chan string +} + +func (f *fakeBootstrapPortForwardStarter) start(_ context.Context, podName string, localPort, remotePort int) (bootstrapPortForwardAttempt, error) { + if remotePort != 8880 { + return nil, fmt.Errorf("unexpected remote port %d", remotePort) + } + + attempt := &fakeBootstrapPortForwardAttempt{done: make(chan error, 1)} + f.mu.Lock() + if f.attempts == nil { + f.attempts = map[string]*fakeBootstrapPortForwardAttempt{} + } + f.attempts[podName] = attempt + f.local = append(f.local, localPort) + f.mu.Unlock() + f.started <- podName + + return attempt, nil +} + +func (f *fakeBootstrapPortForwardStarter) fail(podName string, err error) { + f.mu.Lock() + attempt := f.attempts[podName] + f.mu.Unlock() + attempt.done <- err +} + +func (f *fakeBootstrapPortForwardStarter) ports() []int { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]int(nil), f.local...) +} + +type fakeBootstrapPortForwardAttempt struct { + done chan error + once sync.Once +} + +func (f *fakeBootstrapPortForwardAttempt) Done() <-chan error { + return f.done +} + +func (f *fakeBootstrapPortForwardAttempt) Stop() { + f.once.Do(func() { close(f.done) }) +} From 2c886a4d40ab5d1026a5d57d54cb8d1edc055705 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 18:57:55 +0000 Subject: [PATCH 30/43] kubectl: rotate bootstrap edge credentials --- .../app/site_bootstrap_netboot.go | 109 ++++++++++++++++++ .../app/site_bootstrap_netboot_test.go | 44 +++++++ 2 files changed, 153 insertions(+) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index 92f1f3f9e..0ad4b45c8 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -10,12 +10,15 @@ import ( "net" "net/http" "net/url" + "os" + "path/filepath" "sort" "strconv" "sync" "time" "github.com/spf13/cobra" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,6 +76,28 @@ type bootstrapPortForward struct { done chan struct{} } +type bootstrapEdgeToken struct { + path string + cancel context.CancelFunc + done chan struct{} + once sync.Once +} + +func (t *bootstrapEdgeToken) Path() string { + return t.path +} + +func (t *bootstrapEdgeToken) Close() error { + var err error + t.once.Do(func() { + t.cancel() + <-t.done + err = os.RemoveAll(filepath.Dir(t.path)) + }) + + return err +} + func (f *bootstrapPortForward) URL() string { return f.url } @@ -548,3 +573,87 @@ func newSPDYBootstrapPortForwardStarter( } } } + +func newBootstrapEdgeToken( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + tempRoot string, + refreshInterval time.Duration, +) (*bootstrapEdgeToken, error) { + directory, err := os.MkdirTemp(tempRoot, "unbounded-netboot-") + if err != nil { + return nil, fmt.Errorf("create edge token directory: %w", err) + } + + path := filepath.Join(directory, "edge-token") + if err := refreshBootstrapEdgeToken(ctx, kubeClient, namespace, path); err != nil { + _ = os.RemoveAll(directory) + + return nil, err + } + + if refreshInterval <= 0 { + refreshInterval = 20 * time.Minute + } + tokenCtx, cancel := context.WithCancel(ctx) + credential := &bootstrapEdgeToken{ + path: path, + cancel: cancel, + done: make(chan struct{}), + } + + go func() { + defer close(credential.done) + + ticker := time.NewTicker(refreshInterval) + defer ticker.Stop() + + for { + select { + case <-tokenCtx.Done(): + return + case <-ticker.C: + _ = refreshBootstrapEdgeToken(tokenCtx, kubeClient, namespace, path) + } + } + }() + + return credential, nil +} + +func refreshBootstrapEdgeToken( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + path string, +) error { + expirationSeconds := int64(time.Hour / time.Second) + response, err := kubeClient.CoreV1().ServiceAccounts(namespace).CreateToken( + ctx, + "metalman-edge", + &authenticationv1.TokenRequest{Spec: authenticationv1.TokenRequestSpec{ + Audiences: []string{"metalman-edge"}, + ExpirationSeconds: &expirationSeconds, + }}, + metav1.CreateOptions{}, + ) + if err != nil { + return fmt.Errorf("request metalman-edge token: %w", err) + } + if response.Status.Token == "" { + return fmt.Errorf("request metalman-edge token: API returned an empty token") + } + + temporaryPath := path + ".new" + if err := os.WriteFile(temporaryPath, []byte(response.Status.Token), 0o600); err != nil { + return fmt.Errorf("write metalman-edge token: %w", err) + } + if err := os.Rename(temporaryPath, path); err != nil { + _ = os.Remove(temporaryPath) + + return fmt.Errorf("replace metalman-edge token: %w", err) + } + + return nil +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index cf9669c85..6464e767c 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -7,16 +7,21 @@ import ( "context" "errors" "fmt" + "os" + "path/filepath" "sync" "testing" "time" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -367,6 +372,45 @@ func TestBootstrapPortForwardClosesWhileWaitingToReconnect(t *testing.T) { } } +func TestBootstrapEdgeTokenUsesAudienceAndRotatesSecureFile(t *testing.T) { + ctx := context.Background() + kubeClient := fake.NewSimpleClientset() + var mu sync.Mutex + requests := 0 + kubeClient.PrependReactor("create", "serviceaccounts", func(action clienttesting.Action) (bool, runtime.Object, error) { + create := action.(clienttesting.CreateAction) + request := create.GetObject().(*authenticationv1.TokenRequest) + require.Equal(t, []string{"metalman-edge"}, request.Spec.Audiences) + require.Equal(t, int64(3600), *request.Spec.ExpirationSeconds) + + mu.Lock() + requests++ + token := fmt.Sprintf("token-%d", requests) + mu.Unlock() + + return true, &authenticationv1.TokenRequest{ + Status: authenticationv1.TokenRequestStatus{Token: token}, + }, nil + }) + + credential, err := newBootstrapEdgeToken(ctx, kubeClient, "unbounded-system", t.TempDir(), time.Millisecond) + require.NoError(t, err) + path := credential.Path() + require.Equal(t, "edge-token", filepath.Base(path)) + + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + require.Eventually(t, func() bool { + contents, readErr := os.ReadFile(path) + return readErr == nil && string(contents) != "token-1" + }, time.Second, time.Millisecond) + + require.NoError(t, credential.Close()) + _, err = os.Stat(filepath.Dir(path)) + require.ErrorIs(t, err, os.ErrNotExist) +} + type fakeBootstrapPortForwardStarter struct { mu sync.Mutex attempts map[string]*fakeBootstrapPortForwardAttempt From 266eb16ea3054b6aa8ad7da8de5636dfc7325e82 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:01:32 +0000 Subject: [PATCH 31/43] kubectl: supervise bootstrap edge process --- .../app/site_bootstrap_netboot.go | 138 ++++++++++++++++++ .../app/site_bootstrap_netboot_test.go | 72 +++++++++ 2 files changed, 210 insertions(+) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index 0ad4b45c8..db2f9ef77 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -5,16 +5,19 @@ package app import ( "context" + "errors" "fmt" "io" "net" "net/http" "net/url" "os" + "os/exec" "path/filepath" "sort" "strconv" "sync" + "syscall" "time" "github.com/spf13/cobra" @@ -83,6 +86,58 @@ type bootstrapEdgeToken struct { once sync.Once } +type bootstrapEdgeProcess interface { + Done() <-chan struct{} + Err() error + Stop(ctx context.Context) error +} + +type commandBootstrapEdgeProcess struct { + cmd *exec.Cmd + done chan struct{} + stopOnce sync.Once + mu sync.Mutex + err error +} + +func (p *commandBootstrapEdgeProcess) Done() <-chan struct{} { + return p.done +} + +func (p *commandBootstrapEdgeProcess) Err() error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.err +} + +func (p *commandBootstrapEdgeProcess) Stop(ctx context.Context) error { + var signalErr error + p.stopOnce.Do(func() { + signalErr = p.cmd.Process.Signal(syscall.SIGTERM) + }) + if signalErr != nil && !errors.Is(signalErr, os.ErrProcessDone) { + return fmt.Errorf("stop Metalman edge: %w", signalErr) + } + + select { + case <-p.done: + err := p.Err() + if err != nil && !isSignalExit(err) { + return fmt.Errorf("wait for Metalman edge: %w", err) + } + + return nil + case <-ctx.Done(): + if err := p.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { + return fmt.Errorf("kill Metalman edge: %w", err) + } + <-p.done + + return ctx.Err() + } +} + func (t *bootstrapEdgeToken) Path() string { return t.path } @@ -289,6 +344,89 @@ func (h *siteBootstrapNetbootHandler) edgeArguments(backendURL, tokenFile string } } +func (h *siteBootstrapNetbootHandler) resolveMetalmanBinary(lookPath func(string) (string, error)) (string, error) { + if h.metalmanBinary != "" { + info, err := os.Stat(h.metalmanBinary) + if err != nil { + return "", fmt.Errorf("Metalman binary %q is not accessible: %w", h.metalmanBinary, err) + } + if info.IsDir() || info.Mode()&0o111 == 0 { + return "", fmt.Errorf("Metalman binary %q is not executable", h.metalmanBinary) + } + + return h.metalmanBinary, nil + } + + path, err := lookPath("metalman") + if err != nil { + return "", fmt.Errorf("find metalman executable: %w; install it or set --metalman-binary", err) + } + + return path, nil +} + +func (h *siteBootstrapNetbootHandler) waitForEdgeReady( + ctx context.Context, + process bootstrapEdgeProcess, + dial func(context.Context, string) error, +) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + address := net.JoinHostPort(h.address, strconv.Itoa(h.httpPort)) + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + if err := dial(ctx, address); err == nil { + return nil + } + + select { + case <-process.Done(): + if err := process.Err(); err != nil { + return fmt.Errorf("Metalman edge exited before becoming ready: %w", err) + } + + return errors.New("Metalman edge exited before becoming ready") + case <-ctx.Done(): + return fmt.Errorf("wait for Metalman edge listener: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func startBootstrapEdgeProcess(binary string, args []string, stdout, stderr io.Writer) (bootstrapEdgeProcess, error) { + cmd := exec.Command(binary, args...) + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start Metalman edge: %w", err) + } + + process := &commandBootstrapEdgeProcess{cmd: cmd, done: make(chan struct{})} + go func() { + err := cmd.Wait() + process.mu.Lock() + process.err = err + process.mu.Unlock() + close(process.done) + }() + + return process, nil +} + +func isSignalExit(err error) bool { + var exitError *exec.ExitError + if !errors.As(err, &exitError) || exitError.ProcessState == nil { + return false + } + status, ok := exitError.ProcessState.Sys().(syscall.WaitStatus) + + return ok && status.Signaled() +} + func (h *siteBootstrapNetbootHandler) claimEndpoint(ctx context.Context, identity string) error { var endpoint v1alpha3.NetbootEndpoint if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint); err != nil { diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index 6464e767c..43be4569a 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "os" + "os/exec" "path/filepath" "sync" "testing" @@ -411,6 +412,77 @@ func TestBootstrapEdgeTokenUsesAudienceAndRotatesSecureFile(t *testing.T) { require.ErrorIs(t, err, os.ErrNotExist) } +func TestBootstrapWaitsForEdgeListenerAndReportsEarlyExit(t *testing.T) { + h := &siteBootstrapNetbootHandler{address: "192.0.2.10", httpPort: 8880, pollInterval: time.Millisecond} + process := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + attempts := 0 + dial := func(_ context.Context, address string) error { + require.Equal(t, "192.0.2.10:8880", address) + attempts++ + if attempts < 2 { + return errors.New("not listening") + } + + return nil + } + + require.NoError(t, h.waitForEdgeReady(context.Background(), process, dial)) + require.Equal(t, 2, attempts) + + failed := &fakeBootstrapEdgeProcess{done: make(chan struct{}), err: errors.New("bind: address already in use")} + close(failed.done) + err := h.waitForEdgeReady(context.Background(), failed, func(context.Context, string) error { + return errors.New("not listening") + }) + require.ErrorContains(t, err, "edge exited before becoming ready") + require.ErrorContains(t, err, "address already in use") +} + +func TestBootstrapPreflightResolvesMetalmanBeforeClusterMutation(t *testing.T) { + explicitPath := filepath.Join(t.TempDir(), "metalman") + require.NoError(t, os.WriteFile(explicitPath, []byte("binary"), 0o700)) + h := &siteBootstrapNetbootHandler{metalmanBinary: explicitPath} + lookedUp := "" + path, err := h.resolveMetalmanBinary(func(name string) (string, error) { + lookedUp = name + + return name, nil + }) + require.NoError(t, err) + require.Equal(t, explicitPath, path) + require.Empty(t, lookedUp) + + h.metalmanBinary = "" + path, err = h.resolveMetalmanBinary(func(name string) (string, error) { + lookedUp = name + + return "/usr/local/bin/metalman", nil + }) + require.NoError(t, err) + require.Equal(t, "metalman", lookedUp) + require.Equal(t, "/usr/local/bin/metalman", path) + + _, err = h.resolveMetalmanBinary(func(string) (string, error) { + return "", exec.ErrNotFound + }) + require.ErrorContains(t, err, "--metalman-binary") +} + +type fakeBootstrapEdgeProcess struct { + done chan struct{} + err error +} + +func (f *fakeBootstrapEdgeProcess) Done() <-chan struct{} { + return f.done +} + +func (f *fakeBootstrapEdgeProcess) Err() error { return f.err } + +func (f *fakeBootstrapEdgeProcess) Stop(context.Context) error { + return nil +} + type fakeBootstrapPortForwardStarter struct { mu sync.Mutex attempts map[string]*fakeBootstrapPortForwardAttempt From b30f82ba9d9dbae4fa65bf08eaa14766084f0e82 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:06:23 +0000 Subject: [PATCH 32/43] kubectl: orchestrate netboot bootstrap lifecycle --- .../app/site_bootstrap_netboot.go | 234 +++++++++++++++++- .../app/site_bootstrap_netboot_test.go | 157 +++++++++++- 2 files changed, 388 insertions(+), 3 deletions(-) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index db2f9ef77..02292d125 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/kube" "github.com/Azure/unbounded/internal/unbounded" ) @@ -54,7 +55,19 @@ type siteBootstrapNetbootHandler struct { routedCIDRs []string resources client.Client kubeClient kubernetes.Interface + restConfig *rest.Config pollInterval time.Duration + dependencies bootstrapNetbootDependencies +} + +type bootstrapNetbootDependencies struct { + resolveBinary func() (string, error) + preflightNetwork func() error + localPort func() (int, error) + portForward func(context.Context, int) (*bootstrapPortForward, error) + edgeToken func(context.Context) (*bootstrapEdgeToken, error) + startEdge func(string, []string) (bootstrapEdgeProcess, error) + dialEdge func(context.Context, string) error } type bootstrapNetbootState struct { @@ -198,10 +211,204 @@ func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Co return cmd } -func (h *siteBootstrapNetbootHandler) execute(_ context.Context) error { +func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error) { + if h.timeout <= 0 { + h.timeout = defaultBootstrapNetbootTimeout + } + if h.endpointName == "" { + h.endpointName = "bootstrap-" + h.machine + } + + if err := h.initializeDependencies(); err != nil { + return err + } + binary, err := h.dependencies.resolveBinary() + if err != nil { + return err + } + if err := h.dependencies.preflightNetwork(); err != nil { + return err + } + if err := h.initializeClients(); err != nil { + return err + } + h.initializeRuntimeDependencies() + + runCtx, cancel := context.WithTimeout(ctx, h.timeout) + defer cancel() + + state, err := h.prepareClusterResources(runCtx) + if err != nil { + return err + } + resourcesPrepared := true + var ( + forward *bootstrapPortForward + token *bootstrapEdgeToken + process bootstrapEdgeProcess + ) + defer func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cleanupCancel() + + if resourcesPrepared { + retErr = errors.Join(retErr, h.restoreClusterResources(cleanupCtx, state)) + } + if process != nil { + retErr = errors.Join(retErr, process.Stop(cleanupCtx)) + } + if forward != nil { + retErr = errors.Join(retErr, forward.Close()) + } + if token != nil { + retErr = errors.Join(retErr, token.Close()) + } + }() + + if err := h.waitForMetalman(runCtx); err != nil { + return err + } + localPort, err := h.dependencies.localPort() + if err != nil { + return fmt.Errorf("select local port for Metalman server: %w", err) + } + forward, err = h.dependencies.portForward(runCtx, localPort) + if err != nil { + return err + } + token, err = h.dependencies.edgeToken(runCtx) + if err != nil { + return err + } + process, err = h.dependencies.startEdge(binary, h.edgeArguments(forward.URL(), token.Path())) + if err != nil { + return err + } + if err := h.waitForEdgeReady(runCtx, process, h.dependencies.dialEdge); err != nil { + return err + } + if err := h.claimEndpoint(runCtx, "kubectl-unbounded/"+h.endpointName); err != nil { + return err + } + + var machine v1alpha3.Machine + if err := h.resources.Get(runCtx, client.ObjectKey{Name: h.machine}, &machine); err != nil { + return fmt.Errorf("get designated Machine %s: %w", h.machine, err) + } + + return h.waitForNodeReadyAndEdge(runCtx, designatedNodeName(&machine), process) +} + +func (h *siteBootstrapNetbootHandler) initializeDependencies() error { + if h.dependencies.resolveBinary == nil { + h.dependencies.resolveBinary = func() (string, error) { + return h.resolveMetalmanBinary(exec.LookPath) + } + } + if h.dependencies.preflightNetwork == nil { + h.dependencies.preflightNetwork = h.validateProvisioningInterface + } + if h.dependencies.localPort == nil { + h.dependencies.localPort = availableLoopbackPort + } + if h.dependencies.startEdge == nil { + h.dependencies.startEdge = func(binary string, args []string) (bootstrapEdgeProcess, error) { + return startBootstrapEdgeProcess(binary, args, os.Stdout, os.Stderr) + } + } + if h.dependencies.dialEdge == nil { + h.dependencies.dialEdge = dialBootstrapEdge + } + return nil } +func (h *siteBootstrapNetbootHandler) initializeClients() error { + if h.resources != nil && h.kubeClient != nil && h.restConfig != nil { + return nil + } + + kubeClient, config, err := kube.ClientAndConfigFromFile(getKubeconfigPath(h.kubeconfigPath)) + if err != nil { + return fmt.Errorf("create Kubernetes client for netboot bootstrap: %w", err) + } + resources, err := client.New(config, client.Options{Scheme: buildScheme()}) + if err != nil { + return fmt.Errorf("create resource client for netboot bootstrap: %w", err) + } + h.kubeClient = kubeClient + h.resources = resources + h.restConfig = config + + return nil +} + +func (h *siteBootstrapNetbootHandler) initializeRuntimeDependencies() { + if h.dependencies.portForward == nil { + starter := newSPDYBootstrapPortForwardStarter(h.kubeClient, h.restConfig, h.namespace) + h.dependencies.portForward = func(ctx context.Context, localPort int) (*bootstrapPortForward, error) { + return newBootstrapPortForward( + ctx, + h.kubeClient, + h.namespace, + "metalman-server-"+h.site, + localPort, + 8880, + starter, + h.pollInterval, + ) + } + } + if h.dependencies.edgeToken == nil { + h.dependencies.edgeToken = func(ctx context.Context) (*bootstrapEdgeToken, error) { + return newBootstrapEdgeToken(ctx, h.kubeClient, h.namespace, "", 0) + } + } +} + +func (h *siteBootstrapNetbootHandler) validateProvisioningInterface() error { + interfaceInfo, err := net.InterfaceByName(h.interfaceName) + if err != nil { + return fmt.Errorf("find provisioning interface %s: %w", h.interfaceName, err) + } + wanted := net.ParseIP(h.address) + if wanted == nil || wanted.To4() == nil { + return fmt.Errorf("provisioning address %q must be an IPv4 address", h.address) + } + addresses, err := interfaceInfo.Addrs() + if err != nil { + return fmt.Errorf("list addresses on provisioning interface %s: %w", h.interfaceName, err) + } + for _, address := range addresses { + ip, _, parseErr := net.ParseCIDR(address.String()) + if parseErr == nil && ip.Equal(wanted) { + return nil + } + } + + return fmt.Errorf("provisioning interface %s does not own address %s", h.interfaceName, h.address) +} + +func availableLoopbackPort() (int, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer listener.Close() + + return listener.Addr().(*net.TCPAddr).Port, nil +} + +func dialBootstrapEdge(ctx context.Context, address string) error { + dialer := net.Dialer{} + connection, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return err + } + + return connection.Close() +} + func (h *siteBootstrapNetbootHandler) prepareClusterResources(ctx context.Context) (bootstrapNetbootState, error) { var site v1alpha3.Site if err := h.resources.Get(ctx, client.ObjectKey{Name: h.site}, &site); err != nil { @@ -481,6 +688,31 @@ func (h *siteBootstrapNetbootHandler) waitForNodeReady(ctx context.Context, node } } +func (h *siteBootstrapNetbootHandler) waitForNodeReadyAndEdge( + ctx context.Context, + nodeName string, + process bootstrapEdgeProcess, +) error { + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + + ready := make(chan error, 1) + go func() { ready <- h.waitForNodeReady(waitCtx, nodeName) }() + + select { + case err := <-ready: + return err + case <-process.Done(): + if err := process.Err(); err != nil { + return fmt.Errorf("Metalman edge exited before designated Node %s became Ready: %w", nodeName, err) + } + + return fmt.Errorf("Metalman edge exited before designated Node %s became Ready", nodeName) + case <-ctx.Done(): + return fmt.Errorf("wait for designated Node %s while Metalman edge is running: %w", nodeName, ctx.Err()) + } +} + func nodeReady(node *corev1.Node) bool { for _, condition := range node.Status.Conditions { if condition.Type == corev1.NodeReady { diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index 43be4569a..324f3b9c6 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" clienttesting "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -468,9 +469,159 @@ func TestBootstrapPreflightResolvesMetalmanBeforeClusterMutation(t *testing.T) { require.ErrorContains(t, err, "--metalman-binary") } +func TestBootstrapNetbootExecuteCleansUpAfterDesignatedNodeReady(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, true) + process := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + forward := testBootstrapPortForward(ctx) + token := testBootstrapEdgeToken(t, ctx) + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, pollInterval: time.Millisecond, + dependencies: bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + localPort: func() (int, error) { return 32123, nil }, + portForward: func(context.Context, int) (*bootstrapPortForward, error) { return forward, nil }, + edgeToken: func(context.Context) (*bootstrapEdgeToken, error) { return token, nil }, + startEdge: func(binary string, args []string) (bootstrapEdgeProcess, error) { + require.Equal(t, "/usr/bin/metalman", binary) + require.Contains(t, args, "--backend-url=http://127.0.0.1:32123") + + return process, nil + }, + dialEdge: func(context.Context, string) error { return nil }, + }, + } + + require.NoError(t, h.execute(ctx)) + require.True(t, process.stopped) + require.Equal(t, "bootstrap-first-node", h.endpointName) + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + var endpoint v1alpha3.NetbootEndpoint + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) + _, err := os.Stat(filepath.Dir(token.Path())) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestBootstrapNetbootExecuteReportsEarlyEdgeExitAndRollsBack(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, false) + process := &fakeBootstrapEdgeProcess{done: make(chan struct{}), err: errors.New("bind: address already in use")} + close(process.done) + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, pollInterval: time.Millisecond, + dependencies: bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + localPort: func() (int, error) { return 32123, nil }, + portForward: func(ctx context.Context, _ int) (*bootstrapPortForward, error) { + return testBootstrapPortForward(ctx), nil + }, + edgeToken: func(ctx context.Context) (*bootstrapEdgeToken, error) { + return testBootstrapEdgeToken(t, ctx), nil + }, + startEdge: func(string, []string) (bootstrapEdgeProcess, error) { return process, nil }, + dialEdge: func(context.Context, string) error { return errors.New("not listening") }, + }, + } + + err := h.execute(ctx) + require.ErrorContains(t, err, "edge exited before becoming ready") + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + var endpoint v1alpha3.NetbootEndpoint + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) +} + +func TestBootstrapNetbootExecuteRunsPreflightBeforeClusterMutation(t *testing.T) { + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, true) + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, + dependencies: bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "", errors.New("metalman missing") }, + preflightNetwork: func() error { require.Fail(t, "network preflight should not follow binary failure"); return nil }, + }, + } + + err := h.execute(context.Background()) + require.ErrorContains(t, err, "metalman missing") + + var gotSite v1alpha3.Site + require.NoError(t, resources.Get(context.Background(), client.ObjectKey{Name: site.Name}, &gotSite)) + require.Nil(t, gotSite.Spec.Components.Metalman) +} + +func bootstrapLifecycleFixture( + t *testing.T, + nodeReady bool, +) (*v1alpha3.Site, *v1alpha3.Machine, client.Client, *fake.Clientset) { + t.Helper() + + site := &v1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}} + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-node", Labels: map[string]string{v1alpha3.MachineSiteLabelKey: site.Name}, + }, + Spec: v1alpha3.MachineSpec{Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{ + EndpointRef: "permanent-edge", + }}}, + } + resources := fakeclient.NewClientBuilder(). + WithScheme(buildScheme()). + WithStatusSubresource(&v1alpha3.NetbootEndpoint{}). + WithObjects(site, machine). + Build() + controller := readyDeployment("metalman-controller-rack-a", 1) + server := readyDeployment("metalman-server-rack-a", 2) + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}} + if nodeReady { + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + } + + return site, machine, resources, fake.NewSimpleClientset(controller, server, node) +} + +func testBootstrapPortForward(ctx context.Context) *bootstrapPortForward { + forwardCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + <-forwardCtx.Done() + close(done) + }() + + return &bootstrapPortForward{url: "http://127.0.0.1:32123", cancel: cancel, done: done} +} + +func testBootstrapEdgeToken(t *testing.T, ctx context.Context) *bootstrapEdgeToken { + t.Helper() + directory := t.TempDir() + path := filepath.Join(directory, "edge-token") + require.NoError(t, os.WriteFile(path, []byte("token"), 0o600)) + tokenCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + <-tokenCtx.Done() + close(done) + }() + + return &bootstrapEdgeToken{path: path, cancel: cancel, done: done} +} + type fakeBootstrapEdgeProcess struct { - done chan struct{} - err error + done chan struct{} + err error + stopped bool } func (f *fakeBootstrapEdgeProcess) Done() <-chan struct{} { @@ -480,6 +631,8 @@ func (f *fakeBootstrapEdgeProcess) Done() <-chan struct{} { func (f *fakeBootstrapEdgeProcess) Err() error { return f.err } func (f *fakeBootstrapEdgeProcess) Stop(context.Context) error { + f.stopped = true + return nil } From f37a47c59b7c9c75c6a9fa4b76779b2d2574ff14 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:10:13 +0000 Subject: [PATCH 33/43] net: support synthetic external gateways --- .../site_watch_reconcile.go | 23 +++++++++++---- .../site_watch_reconcile_test.go | 23 +++++++++++++++ deploy/net/node/03-daemonset.yaml.tmpl | 7 +++++ deploy/net/node/daemonset_test.go | 29 +++++++++++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 deploy/net/node/daemonset_test.go diff --git a/cmd/unbounded-net-node/site_watch_reconcile.go b/cmd/unbounded-net-node/site_watch_reconcile.go index 5ba575087..09df92309 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile.go +++ b/cmd/unbounded-net-node/site_watch_reconcile.go @@ -803,13 +803,24 @@ func isGatewayNodeFromCRDs(gatewayPoolInformer cache.SharedIndexInformer, myPubK return false } +func netbootMembershipFromCRDs( + sliceInformer, gatewayPoolInformer cache.SharedIndexInformer, + myPubKey string, +) (string, bool) { + if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + return siteName, true + } + + return "", isGatewayNodeFromCRDs(gatewayPoolInformer, myPubKey) +} + // waitForSiteMembership waits for this node to appear in a SiteNodeSlice or GatewayPool. // This ensures the site controller has processed this node before we continue. // Returns the site name once found, or empty string for gateway nodes without a site. func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInformer cache.SharedIndexInformer, myPubKey string) (string, error) { // Check immediately first - siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey) - if siteName != "" { + siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey) + if found { klog.Infof("Node found in site %q", siteName) return siteName, nil } @@ -822,7 +833,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform // Add event handlers to detect when we appear in a slice or pool sliceHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: @@ -830,7 +841,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform } }, UpdateFunc: func(oldObj, newObj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: @@ -841,7 +852,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform poolHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: @@ -849,7 +860,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform } }, UpdateFunc: func(oldObj, newObj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: diff --git a/cmd/unbounded-net-node/site_watch_reconcile_test.go b/cmd/unbounded-net-node/site_watch_reconcile_test.go index 8c56511c7..75314f4ea 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile_test.go +++ b/cmd/unbounded-net-node/site_watch_reconcile_test.go @@ -106,6 +106,29 @@ func TestWaitForSiteMembershipImmediate(t *testing.T) { } } +func TestWaitForSiteMembershipAcceptsSiteLessGateway(t *testing.T) { + sliceInformer := newTestInformer() + gatewayPoolInformer := newTestInformer() + pool := &unboundednetv1alpha1.GatewayPool{ + ObjectMeta: metav1.ObjectMeta{Name: "external"}, + Spec: unboundednetv1alpha1.GatewayPoolSpec{NodeSelector: map[string]string{"external": "true"}}, + Status: unboundednetv1alpha1.GatewayPoolStatus{Nodes: []unboundednetv1alpha1.GatewayNodeInfo{ + {Name: "admin-gateway", WireGuardPublicKey: "pub-external"}, + }}, + } + if err := gatewayPoolInformer.GetStore().Add(toUnstructuredSiteWatch(t, pool)); err != nil { + t.Fatalf("add pool failed: %v", err) + } + + got, err := waitForSiteMembership(context.Background(), sliceInformer, gatewayPoolInformer, "pub-external") + if err != nil { + t.Fatalf("waitForSiteMembership returned error: %v", err) + } + if got != "" { + t.Fatalf("site-less gateway membership = %q, want empty", got) + } +} + // TestGetManageCniPluginFromCRDs tests GetManageCniPluginFromCRDs. func TestGetManageCniPluginFromCRDs(t *testing.T) { siteInformer := newTestInformer() diff --git a/deploy/net/node/03-daemonset.yaml.tmpl b/deploy/net/node/03-daemonset.yaml.tmpl index 71adf66b7..0bb5e49af 100644 --- a/deploy/net/node/03-daemonset.yaml.tmpl +++ b/deploy/net/node/03-daemonset.yaml.tmpl @@ -35,6 +35,13 @@ spec: # Tolerate all taints - CNI must run on every node tolerations: - operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: net.unbounded-cloud.io/external-node + operator: DoesNotExist # Prioritize scheduling - CNI is critical for pod networking priorityClassName: system-node-critical initContainers: diff --git a/deploy/net/node/daemonset_test.go b/deploy/net/node/daemonset_test.go new file mode 100644 index 000000000..78f3e9818 --- /dev/null +++ b/deploy/net/node/daemonset_test.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package node + +import ( + "os" + "strings" + "testing" +) + +func TestDaemonSetExcludesExternalSyntheticNodes(t *testing.T) { + contents, err := os.ReadFile("03-daemonset.yaml.tmpl") + if err != nil { + t.Fatalf("read DaemonSet template: %v", err) + } + + template := string(contents) + for _, expected := range []string{ + "nodeAffinity:", + "requiredDuringSchedulingIgnoredDuringExecution:", + "net.unbounded-cloud.io/external-node", + "operator: DoesNotExist", + } { + if !strings.Contains(template, expected) { + t.Fatalf("DaemonSet template missing %q", expected) + } + } +} From a47b17fcc6f323c2529d0a58fd421132e5796e44 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:15:10 +0000 Subject: [PATCH 34/43] kubectl: prepare bootstrap gateway resources --- cmd/kubectl-unbounded/app/machine_ops.go | 2 + .../app/site_bootstrap_netboot.go | 102 ++++++++++++++++++ .../app/site_bootstrap_netboot_test.go | 59 ++++++++++ 3 files changed, 163 insertions(+) diff --git a/cmd/kubectl-unbounded/app/machine_ops.go b/cmd/kubectl-unbounded/app/machine_ops.go index a6e4fd012..1937303d2 100644 --- a/cmd/kubectl-unbounded/app/machine_ops.go +++ b/cmd/kubectl-unbounded/app/machine_ops.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" ) // ANSI color/style codes for terminal output. @@ -32,6 +33,7 @@ func buildScheme() *runtime.Scheme { s := runtime.NewScheme() utilruntime.Must(corev1.AddToScheme(s)) utilruntime.Must(v1alpha3.AddToScheme(s)) + utilruntime.Must(netv1alpha1.AddToScheme(s)) return s } diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index 02292d125..023707ebb 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" "github.com/Azure/unbounded/internal/kube" "github.com/Azure/unbounded/internal/unbounded" ) @@ -492,6 +493,107 @@ func (h *siteBootstrapNetbootHandler) restoreClusterResources(ctx context.Contex return nil } +func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Context) (retErr error) { + for _, routedCIDR := range h.routedCIDRs { + if _, _, err := net.ParseCIDR(routedCIDR); err != nil { + return fmt.Errorf("invalid routed CIDR %q: %w", routedCIDR, err) + } + } + + protocol := netv1alpha1.TunnelProtocolWireGuard + enabled := true + selector := map[string]string{"net.unbounded-cloud.io/bootstrap-gateway": h.endpointName} + pool := &netv1alpha1.GatewayPool{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: netv1alpha1.GatewayPoolSpec{ + Type: "External", + NodeSelector: selector, + RoutedCidrs: append([]string(nil), h.routedCIDRs...), + TunnelProtocol: &protocol, + }, + } + if err := h.resources.Create(ctx, pool); err != nil { + return fmt.Errorf("create bootstrap GatewayPool %s: %w", h.endpointName, err) + } + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, h.cleanupGatewayResources(context.WithoutCancel(ctx))) + } + }() + + assignment := &netv1alpha1.SiteGatewayPoolAssignment{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: netv1alpha1.SiteGatewayPoolAssignmentSpec{ + Enabled: &enabled, + Sites: []string{h.site}, + GatewayPools: []string{h.endpointName}, + TunnelProtocol: &protocol, + }, + } + if err := h.resources.Create(ctx, assignment); err != nil { + return fmt.Errorf("create bootstrap SiteGatewayPoolAssignment %s: %w", h.endpointName, err) + } + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: h.endpointName, + Labels: map[string]string{ + "net.unbounded-cloud.io/bootstrap-gateway": h.endpointName, + "net.unbounded-cloud.io/external-node": "true", + }, + }, + Spec: corev1.NodeSpec{ + Unschedulable: true, + Taints: []corev1.Taint{{ + Key: "net.unbounded-cloud.io/gateway-node", Value: "true", Effect: corev1.TaintEffectNoSchedule, + }}, + }, + } + createdNode, err := h.kubeClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create bootstrap gateway Node %s: %w", h.endpointName, err) + } + createdNode.Status.Addresses = []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: h.address}, + {Type: corev1.NodeExternalIP, Address: h.address}, + } + if _, err := h.kubeClient.CoreV1().Nodes().UpdateStatus(ctx, createdNode, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("set bootstrap gateway Node %s addresses: %w", h.endpointName, err) + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) cleanupGatewayResources(ctx context.Context) error { + assignment := &netv1alpha1.SiteGatewayPoolAssignment{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + assignmentErr := h.resources.Delete(ctx, assignment) + if apierrors.IsNotFound(assignmentErr) { + assignmentErr = nil + } + if assignmentErr != nil { + assignmentErr = fmt.Errorf("delete bootstrap SiteGatewayPoolAssignment %s: %w", h.endpointName, assignmentErr) + } + + nodeErr := h.kubeClient.CoreV1().Nodes().Delete(ctx, h.endpointName, metav1.DeleteOptions{}) + if apierrors.IsNotFound(nodeErr) { + nodeErr = nil + } + if nodeErr != nil { + nodeErr = fmt.Errorf("delete bootstrap gateway Node %s: %w", h.endpointName, nodeErr) + } + + pool := &netv1alpha1.GatewayPool{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + poolErr := h.resources.Delete(ctx, pool) + if apierrors.IsNotFound(poolErr) { + poolErr = nil + } + if poolErr != nil { + poolErr = fmt.Errorf("delete bootstrap GatewayPool %s: %w", h.endpointName, poolErr) + } + + return errors.Join(assignmentErr, nodeErr, poolErr) +} + func (h *siteBootstrapNetbootHandler) metalmanDeploymentsReady(ctx context.Context) (bool, error) { for _, name := range []string{"metalman-controller-" + h.site, "metalman-server-" + h.site} { deployment, err := h.kubeClient.AppsV1().Deployments(h.namespace).Get(ctx, name, metav1.GetOptions{}) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index 324f3b9c6..ee18930f3 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -28,6 +28,7 @@ import ( fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" ) func TestSiteBootstrapNetbootCommandContract(t *testing.T) { @@ -562,6 +563,64 @@ func TestBootstrapNetbootExecuteRunsPreflightBeforeClusterMutation(t *testing.T) require.Nil(t, gotSite.Spec.Components.Metalman) } +func TestBootstrapNetbootPreparesAndCleansExternalGatewayResources(t *testing.T) { + ctx := context.Background() + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).Build() + kubeClient := fake.NewSimpleClientset() + h := &siteBootstrapNetbootHandler{ + site: "rack-a", endpointName: "bootstrap-first-node", address: "192.0.2.10", + routedCIDRs: []string{"10.40.0.0/16", "10.50.0.0/24"}, resources: resources, kubeClient: kubeClient, + } + + require.NoError(t, h.prepareGatewayResources(ctx)) + var pool netv1alpha1.GatewayPool + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) + require.Equal(t, "External", pool.Spec.Type) + require.Equal(t, h.routedCIDRs, pool.Spec.RoutedCidrs) + require.Equal(t, map[string]string{"net.unbounded-cloud.io/bootstrap-gateway": h.endpointName}, pool.Spec.NodeSelector) + require.Equal(t, netv1alpha1.TunnelProtocolWireGuard, *pool.Spec.TunnelProtocol) + + var assignment netv1alpha1.SiteGatewayPoolAssignment + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &assignment)) + require.Equal(t, []string{h.site}, assignment.Spec.Sites) + require.Equal(t, []string{h.endpointName}, assignment.Spec.GatewayPools) + require.True(t, *assignment.Spec.Enabled) + require.Equal(t, netv1alpha1.TunnelProtocolWireGuard, *assignment.Spec.TunnelProtocol) + + node, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.NoError(t, err) + require.True(t, node.Spec.Unschedulable) + require.Equal(t, h.endpointName, node.Labels["net.unbounded-cloud.io/bootstrap-gateway"]) + require.Equal(t, "true", node.Labels["net.unbounded-cloud.io/external-node"]) + require.Contains(t, node.Spec.Taints, corev1.Taint{ + Key: "net.unbounded-cloud.io/gateway-node", Value: "true", Effect: corev1.TaintEffectNoSchedule, + }) + require.ElementsMatch(t, []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: h.address}, + {Type: corev1.NodeExternalIP, Address: h.address}, + }, node.Status.Addresses) + + require.NoError(t, h.cleanupGatewayResources(ctx)) + _, err = kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.Error(t, err) + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &assignment)) + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) +} + +func TestBootstrapNetbootRejectsInvalidRoutedCIDRBeforeCreatingGateway(t *testing.T) { + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).Build() + h := &siteBootstrapNetbootHandler{ + endpointName: "bootstrap-first-node", routedCIDRs: []string{"not-a-cidr"}, resources: resources, + kubeClient: fake.NewSimpleClientset(), + } + + err := h.prepareGatewayResources(context.Background()) + require.ErrorContains(t, err, "invalid routed CIDR") + var pools netv1alpha1.GatewayPoolList + require.NoError(t, resources.List(context.Background(), &pools)) + require.Empty(t, pools.Items) +} + func bootstrapLifecycleFixture( t *testing.T, nodeReady bool, From 68d272bde59ec26d8fb9dd0702d429c091afaedd Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:17:39 +0000 Subject: [PATCH 35/43] net: skip CNI for site-less gateways --- cmd/unbounded-net-node/main.go | 2 +- cmd/unbounded-net-node/site_watch_reconcile.go | 8 ++++++++ cmd/unbounded-net-node/site_watch_reconcile_test.go | 11 +++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd/unbounded-net-node/main.go b/cmd/unbounded-net-node/main.go index 3e79a9f30..3b78478f4 100644 --- a/cmd/unbounded-net-node/main.go +++ b/cmd/unbounded-net-node/main.go @@ -769,7 +769,7 @@ func run(cfg *config) error { } // Check if this node's site has manageCniPlugin enabled using the informer cache - manageCniPlugin := getManageCniPluginFromCRDs(siteInformer, mySiteName) + manageCniPlugin := manageCNIForMembership(siteInformer, mySiteName, isGatewayNode) var nodePodCIDRs []string if manageCniPlugin { diff --git a/cmd/unbounded-net-node/site_watch_reconcile.go b/cmd/unbounded-net-node/site_watch_reconcile.go index 09df92309..22be3810b 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile.go +++ b/cmd/unbounded-net-node/site_watch_reconcile.go @@ -932,6 +932,14 @@ func getManageCniPluginFromCRDs(siteInformer cache.SharedIndexInformer, siteName return true // Site not found, default to true } +func manageCNIForMembership(siteInformer cache.SharedIndexInformer, siteName string, isGatewayNode bool) bool { + if isGatewayNode && siteName == "" { + return false + } + + return getManageCniPluginFromCRDs(siteInformer, siteName) +} + var configureWireGuardFunc = configureWireGuard // updateWireGuardFromSlices reads Site and SiteNodeSlices from the informer caches and configures WireGuard diff --git a/cmd/unbounded-net-node/site_watch_reconcile_test.go b/cmd/unbounded-net-node/site_watch_reconcile_test.go index 75314f4ea..4eab11fa5 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile_test.go +++ b/cmd/unbounded-net-node/site_watch_reconcile_test.go @@ -167,3 +167,14 @@ func TestGetManageCniPluginFromCRDs(t *testing.T) { t.Fatalf("expected missing site default to true") } } + +func TestSiteLessGatewayDisablesCNIManagement(t *testing.T) { + siteInformer := newTestInformer() + + if manageCNIForMembership(siteInformer, "", true) { + t.Fatal("site-less gateway must not wait for PodCIDRs or write CNI configuration") + } + if !manageCNIForMembership(siteInformer, "", false) { + t.Fatal("ordinary node without resolved membership must retain the default CNI policy") + } +} From 66af6e15c2e2298829fcf1046c82214366debb02 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:22:15 +0000 Subject: [PATCH 36/43] net: define external gateway runner options --- internal/net/nodeagent/external.go | 59 +++++++++++++++++++++++++ internal/net/nodeagent/external_test.go | 46 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 internal/net/nodeagent/external.go create mode 100644 internal/net/nodeagent/external_test.go diff --git a/internal/net/nodeagent/external.go b/internal/net/nodeagent/external.go new file mode 100644 index 000000000..0b584c6a0 --- /dev/null +++ b/internal/net/nodeagent/external.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodeagent + +import ( + "fmt" + "path/filepath" + "time" + + "k8s.io/client-go/rest" +) + +// ExternalGatewayOptions identifies a temporary external gateway dataplane. +type ExternalGatewayOptions struct { + NodeName string + RuntimeDir string + RESTConfig *rest.Config +} + +type config struct { + NodeName string + CNIConfDir string + WireGuardDir string + StatusPushEnabled bool + StatusWSEnabled bool + KubeProxyHealthInterval time.Duration + HealthPort int + RouteTableID int + RemoveConfigurationOnShutdown bool + PreferredPublicEncap string + RESTConfig *rest.Config +} + +func externalGatewayConfig(options ExternalGatewayOptions) (*config, error) { + if options.NodeName == "" { + return nil, fmt.Errorf("external gateway node name is required") + } + if options.RuntimeDir == "" { + return nil, fmt.Errorf("external gateway runtime directory is required") + } + if !filepath.IsAbs(options.RuntimeDir) { + return nil, fmt.Errorf("external gateway runtime directory must be absolute") + } + + return &config{ + NodeName: options.NodeName, + CNIConfDir: filepath.Join(options.RuntimeDir, "cni"), + WireGuardDir: filepath.Join(options.RuntimeDir, "wireguard"), + StatusPushEnabled: false, + StatusWSEnabled: false, + KubeProxyHealthInterval: 0, + HealthPort: 0, + RouteTableID: 254, + RemoveConfigurationOnShutdown: true, + PreferredPublicEncap: "WireGuard", + RESTConfig: options.RESTConfig, + }, nil +} diff --git a/internal/net/nodeagent/external_test.go b/internal/net/nodeagent/external_test.go new file mode 100644 index 000000000..79602d897 --- /dev/null +++ b/internal/net/nodeagent/external_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodeagent + +import ( + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/client-go/rest" +) + +func TestExternalGatewayConfigUsesSafeStandaloneDefaults(t *testing.T) { + restConfig := &rest.Config{Host: "https://api.example.test"} + cfg, err := externalGatewayConfig(ExternalGatewayOptions{ + NodeName: "bootstrap-gateway", + RuntimeDir: "/run/unbounded-netboot/bootstrap-gateway", + RESTConfig: restConfig, + }) + require.NoError(t, err) + require.Equal(t, "bootstrap-gateway", cfg.NodeName) + require.Equal(t, "/run/unbounded-netboot/bootstrap-gateway/wireguard", cfg.WireGuardDir) + require.Equal(t, "/run/unbounded-netboot/bootstrap-gateway/cni", cfg.CNIConfDir) + require.Same(t, restConfig, cfg.RESTConfig) + require.False(t, cfg.StatusPushEnabled) + require.False(t, cfg.StatusWSEnabled) + require.Zero(t, cfg.KubeProxyHealthInterval) + require.Zero(t, cfg.HealthPort) + require.Equal(t, 254, cfg.RouteTableID) + require.True(t, cfg.RemoveConfigurationOnShutdown) + require.Equal(t, "WireGuard", cfg.PreferredPublicEncap) +} + +func TestExternalGatewayConfigRequiresIdentityAndRuntimeDirectory(t *testing.T) { + _, err := externalGatewayConfig(ExternalGatewayOptions{RuntimeDir: "/run/unbounded-netboot/gateway"}) + require.ErrorContains(t, err, "node name") + + _, err = externalGatewayConfig(ExternalGatewayOptions{NodeName: "gateway"}) + require.ErrorContains(t, err, "runtime directory") + + _, err = externalGatewayConfig(ExternalGatewayOptions{ + NodeName: "gateway", + RuntimeDir: "relative/path", + }) + require.ErrorContains(t, err, "absolute") +} From c6e1e6d643744adaac8022c69697a922d5962bc5 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:26:43 +0000 Subject: [PATCH 37/43] net: extract reusable node agent --- cmd/unbounded-net-node/main.go | 790 +---------------- .../net/nodeagent}/bootstrap_helpers.go | 2 +- .../net/nodeagent}/bootstrap_helpers_test.go | 2 +- .../net/nodeagent}/bpf_status.go | 2 +- .../net/nodeagent}/encapsulation.go | 2 +- .../net/nodeagent}/encapsulation_test.go | 2 +- internal/net/nodeagent/external.go | 61 +- .../net/nodeagent}/gateway_routes.go | 2 +- .../net/nodeagent}/gateway_routes_test.go | 2 +- .../net/nodeagent}/kube_proxy_monitor.go | 2 +- .../net/nodeagent}/kube_proxy_monitor_test.go | 2 +- .../net/nodeagent}/link_stats_monitor.go | 2 +- .../net/nodeagent}/link_stats_monitor_test.go | 2 +- .../net/nodeagent}/main_config_test.go | 2 +- .../net/nodeagent}/main_test.go | 2 +- .../net/nodeagent}/main_update_test.go | 2 +- .../net/nodeagent}/metrics.go | 2 +- internal/net/nodeagent/node.go | 795 ++++++++++++++++++ .../net/nodeagent}/node_config_validation.go | 2 +- .../nodeagent}/node_config_validation_test.go | 2 +- .../net/nodeagent}/node_types.go | 2 +- .../net/nodeagent}/peer_healthcheck.go | 2 +- .../net/nodeagent}/reconciliation_helpers.go | 2 +- .../reconciliation_helpers_bfd_test.go | 2 +- .../reconciliation_helpers_more_test.go | 2 +- .../nodeagent}/reconciliation_helpers_test.go | 2 +- .../net/nodeagent}/route_annotations.go | 2 +- .../net/nodeagent}/runtime_utils.go | 2 +- .../net/nodeagent}/runtime_utils_test.go | 2 +- .../net/nodeagent}/site_watch_reconcile.go | 2 +- .../nodeagent}/site_watch_reconcile_test.go | 2 +- .../net/nodeagent}/status_proto.go | 2 +- .../net/nodeagent}/status_proto_test.go | 2 +- .../net/nodeagent}/status_server.go | 2 +- .../net/nodeagent}/status_server_http_test.go | 2 +- .../net/nodeagent}/status_server_test.go | 2 +- .../net/nodeagent}/tunnel_config.go | 2 +- .../nodeagent}/tunnel_interface_validation.go | 2 +- .../tunnel_interface_validation_test.go | 2 +- .../net/nodeagent}/wireguard_config.go | 2 +- .../net/nodeagent}/wireguard_config_test.go | 2 +- 41 files changed, 872 insertions(+), 850 deletions(-) rename {cmd/unbounded-net-node => internal/net/nodeagent}/bootstrap_helpers.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/bootstrap_helpers_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/bpf_status.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/encapsulation.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/encapsulation_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/gateway_routes.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/gateway_routes_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/kube_proxy_monitor.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/kube_proxy_monitor_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/link_stats_monitor.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/link_stats_monitor_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/main_config_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/main_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/main_update_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/metrics.go (99%) create mode 100644 internal/net/nodeagent/node.go rename {cmd/unbounded-net-node => internal/net/nodeagent}/node_config_validation.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/node_config_validation_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/node_types.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/peer_healthcheck.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/reconciliation_helpers.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/reconciliation_helpers_bfd_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/reconciliation_helpers_more_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/reconciliation_helpers_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/route_annotations.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/runtime_utils.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/runtime_utils_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/site_watch_reconcile.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/site_watch_reconcile_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/status_proto.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/status_proto_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/status_server.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/status_server_http_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/status_server_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/tunnel_config.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/tunnel_interface_validation.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/tunnel_interface_validation_test.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/wireguard_config.go (99%) rename {cmd/unbounded-net-node => internal/net/nodeagent}/wireguard_config_test.go (80%) diff --git a/cmd/unbounded-net-node/main.go b/cmd/unbounded-net-node/main.go index 3b78478f4..11de9a9b6 100644 --- a/cmd/unbounded-net-node/main.go +++ b/cmd/unbounded-net-node/main.go @@ -5,798 +5,20 @@ package main import ( "context" - "flag" - "fmt" "os" "os/signal" "syscall" - "time" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/dynamic/dynamicinformer" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - - unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" - configpkg "github.com/Azure/unbounded/internal/net/config" - "github.com/Azure/unbounded/internal/net/metrics" - unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" - "github.com/Azure/unbounded/internal/version" -) - -// CNIConfig represents the CNI configuration file structure -type CNIConfig struct { - CNIVersion string `json:"cniVersion"` - Name string `json:"name"` - Plugins []PluginConf `json:"plugins"` -} - -// PluginConf represents a CNI plugin configuration -type PluginConf struct { - Type string `json:"type"` - Bridge string `json:"bridge,omitempty"` - IsGateway bool `json:"isGateway,omitempty"` - IsDefaultGW bool `json:"isDefaultGateway,omitempty"` - ForceAddress bool `json:"forceAddress,omitempty"` - IPMasq bool `json:"ipMasq,omitempty"` - HairpinMode bool `json:"hairpinMode,omitempty"` - MTU int `json:"mtu,omitempty"` - IPAM *IPAMConfig `json:"ipam,omitempty"` - Capabilities *Caps `json:"capabilities,omitempty"` -} - -// IPAMConfig represents the IPAM configuration -type IPAMConfig struct { - Type string `json:"type"` - Ranges [][]IPRange `json:"ranges,omitempty"` -} - -// IPRange represents an IP range for IPAM -type IPRange struct { - Subnet string `json:"subnet"` -} - -// Caps represents plugin capabilities -type Caps struct { - PortMappings bool `json:"portMappings,omitempty"` -} - -type config struct { - ConfigFile string - KubeconfigPath string - ApiserverURL string // Override Kubernetes API server URL (empty = use default) - NodeName string - CNIConfDir string - CNIConfFile string - BridgeName string - WireGuardDir string - WireGuardPort int - EnablePolicyRouting bool - MTU int - HealthPort int - InformerResyncPeriod time.Duration - StatusPushEnabled bool // Whether to push status to controller - StatusPushURL string // Controller URL for status push - StatusPushInterval time.Duration // Interval between status pushes to controller - StatusPushAPIServerInterval time.Duration // Interval between status pushes via aggregated API server - StatusPushDelta bool // Whether periodic HTTP pushes use deltas - StatusWSEnabled bool // Whether websocket push is enabled - StatusWSURL string // Controller websocket URL for status push - StatusWSAPIServerMode string // API server websocket mode: never, fallback, preferred - StatusWSAPIServerURL string // API server websocket URL for status push fallback - StatusWSAPIServerStartupDelay time.Duration // Delay before API server fallback is allowed after startup - StatusWSKeepaliveInterval time.Duration // Interval between websocket keepalive pings (0 disables keepalive) - StatusWSKeepaliveFailureCount int // Sequential websocket keepalive ping failures before reconnect - RemoveConfigurationOnShutdown bool // Remove all managed configuration (WireGuard, routes, masquerade, etc.) on shutdown - RemoveWireGuardOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown - CleanupNetlinkOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown - RemoveMasqueradeOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown - HealthCheckPort int // UDP port for health check probes (default 9997) - BaseMetric int // Base metric for programmed routes (default 1) - RouteTableID int // Route table ID for managed routes (default 252) - CriticalDeltaEvery time.Duration // Maximum critical delta publish frequency; changes are queued up to this interval for batching - StatsDeltaEvery time.Duration // Maximum statistics delta publish frequency; changes are queued up to this interval for batching - FullSyncEvery time.Duration // Forced full status sync interval; ensures controller has complete status periodically - GenevePort int // GENEVE UDP destination port (default 6081) - GeneveVNI int // GENEVE Virtual Network Identifier (default 1) - GeneveInterfaceName string // GENEVE shared tunnel interface name (default geneve0) - VXLANInterfaceName string // VXLAN shared tunnel interface name (default vxlan0) - IPIPInterfaceName string // IPIP shared tunnel interface name (default ipip0) - WireGuardInterfacePrefix string // Prefix for per-port WireGuard interfaces (default "wg"; per-peer name is ) - VXLANPort int // VXLAN UDP destination port (default 4789) - VXLANSrcPortLow int // VXLAN UDP source port range low (default 47891) - VXLANSrcPortHigh int // VXLAN UDP source port range high (default 47922) - PreferredPrivateEncap string // Preferred encap for private/internal networks (GENEVE, IPIP, VXLAN, WireGuard) - PreferredPublicEncap string // Preferred encap for public/external networks (WireGuard, IPIP, GENEVE, VXLAN) - HealthFlapMaxBackoff time.Duration // Maximum backoff duration for health check flap dampening - KubeProxyHealthInterval time.Duration // Interval between kube-proxy health checks (0 to disable) - NetlinkResyncPeriod time.Duration // Interval between full netlink cache resyncs - TunnelDataplaneMapSize int // Maximum LPM trie entries for eBPF tunnel map (default 16384) - TunnelIPFamily string // Tunnel underlay IP family: "IPv4" (default) or "IPv6" -} - -var siteGVR = schema.GroupVersionResource{ - Group: unboundedv1alpha3.GroupVersion.Group, - Version: unboundedv1alpha3.GroupVersion.Version, - Resource: "sites", -} - -var siteNodeSliceGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "sitenodeslices", -} - -var gatewayPoolGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "gatewaypools", -} - -var gatewayNodeGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "gatewaypoolnodes", -} - -var sitePeeringGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "sitepeerings", -} - -var siteGatewayPoolAssignmentGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "sitegatewaypoolassignments", -} - -var gatewayPoolPeeringGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "gatewaypoolpeerings", -} - -const ( - // WireGuard public key annotation on the node - WireGuardPubKeyAnnotation = "net.unbounded-cloud.io/wg-pubkey" - - // TunnelMTUAnnotation is the maximum tunnel MTU this node - // can support, based on its default-route interface MTU minus encapsulation - // overhead. The controller uses this to validate that the configured MTU - // does not exceed what any node in the cluster can handle. - TunnelMTUAnnotation = "net.unbounded-cloud.io/tunnel-mtu" - - // Gateway node taint key - prevents regular workloads from running on gateway nodes - // since they don't have regular pod CIDR routing - GatewayNodeTaintKey = "net.unbounded-cloud.io/gateway-node" - - // gatewayNodeHeartbeatInterval controls how frequently the node agent refreshes - // GatewayNode.status.lastUpdated. Route staleness is derived from this cadence. - gatewayNodeHeartbeatInterval = 10 * time.Second + "github.com/Azure/unbounded/internal/net/nodeagent" ) func main() { - // Initialize klog flags - klog.InitFlags(nil) - - // Add klog flags to pflag - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - - cfg := &config{ - ConfigFile: "/etc/unbounded-net/config.yaml", - CNIConfDir: "/etc/cni/net.d", - CNIConfFile: "10-unbounded.conflist", - BridgeName: "cbr0", - WireGuardDir: "/etc/wireguard", - WireGuardPort: 51820, - EnablePolicyRouting: false, - MTU: 1280, // default WireGuard MTU (IPv6 minimum) - HealthPort: 9998, - InformerResyncPeriod: 600 * time.Second, - StatusPushEnabled: true, // Enabled by default - StatusPushInterval: 10 * time.Second, // Default 10s push interval - StatusPushAPIServerInterval: 30 * time.Second, - StatusPushDelta: true, - StatusWSEnabled: true, - StatusWSAPIServerMode: statusWSAPIServerModeFallback, - StatusWSAPIServerStartupDelay: 60 * time.Second, - StatusWSKeepaliveInterval: 10 * time.Second, - StatusWSKeepaliveFailureCount: 2, - CriticalDeltaEvery: 1 * time.Second, - StatsDeltaEvery: 15 * time.Second, - FullSyncEvery: 2 * time.Minute, - GenevePort: 6081, - GeneveVNI: 1, - GeneveInterfaceName: "geneve0", - VXLANInterfaceName: "vxlan0", - IPIPInterfaceName: "ipip0", - WireGuardInterfacePrefix: "wg", - VXLANPort: 4789, - VXLANSrcPortLow: 47891, - VXLANSrcPortHigh: 47922, - PreferredPrivateEncap: "GENEVE", - PreferredPublicEncap: "WireGuard", - NetlinkResyncPeriod: 300 * time.Second, - TunnelDataplaneMapSize: 16384, - TunnelIPFamily: "IPv4", - } - - rootCmd := &cobra.Command{ - Use: "unbounded-net-node", - Short: "CNI configuration agent for unbounded-net", - Long: `unbounded-net-node runs on each node as a DaemonSet and configures CNI -networking by writing a CNI configuration file based on the node's podCIDRs. - -It watches the node object in Kubernetes and waits for podCIDRs to be assigned -by the unbounded-net-controller. Once assigned, it writes a CNI configuration -file that sets up pod networking using the bridge plugin with host-local IPAM. - -It also generates WireGuard keys for the node and stores them in /etc/wireguard, -then annotates the node with the public key.`, - Version: version.Version + " (commit: " + version.GitCommit + ")", - SilenceUsage: true, - PreRunE: func(cmd *cobra.Command, args []string) error { - return applyNodeRuntimeConfig(cmd, cfg) - }, - RunE: func(cmd *cobra.Command, args []string) error { - return run(cfg) - }, - } - - // Add flags - flags := rootCmd.Flags() - - // Change version flag from -v to -V to avoid conflict with klog's -v flag - rootCmd.Flags().BoolP("version", "V", false, "Print version information") - rootCmd.SetVersionTemplate(`{{printf "%s\n" .Version}}`) - - // General flags - flags.StringVar(&cfg.ConfigFile, "config-file", "/etc/unbounded-net/config.yaml", "Path to runtime YAML config file") - flags.StringVar(&cfg.KubeconfigPath, "kubeconfig", "", "Path to kubeconfig file (uses in-cluster config if not specified)") - flags.StringVar(&cfg.ApiserverURL, "apiserver-url", "", "Override Kubernetes API server URL (empty = use default from kubeconfig or in-cluster config)") - flags.StringVar(&cfg.NodeName, "node-name", os.Getenv("NODE_NAME"), "Name of this node (defaults to NODE_NAME env var)") - flags.IntVar(&cfg.HealthPort, "health-port", 9998, "Port for health check HTTP server (0 to disable)") - flags.DurationVar(&cfg.InformerResyncPeriod, "informer-resync-period", 600*time.Second, "Resync period for Kubernetes informers") - - // CNI configuration flags - flags.StringVar(&cfg.CNIConfDir, "cni-conf-dir", "/etc/cni/net.d", "Directory to write CNI configuration") - flags.StringVar(&cfg.CNIConfFile, "cni-conf-file", "10-unbounded.conflist", "Name of the CNI configuration file") - flags.StringVar(&cfg.BridgeName, "bridge-name", "cbr0", "Name of the bridge interface") - flags.IntVar(&cfg.MTU, "mtu", 1280, "MTU for WireGuard and bridge interfaces (default 1280, the IPv6 minimum)") - - // WireGuard configuration flags - flags.StringVar(&cfg.WireGuardDir, "wireguard-dir", "/etc/wireguard", "Directory to store WireGuard keys") - flags.IntVar(&cfg.WireGuardPort, "wireguard-port", 51820, "WireGuard listen port") - flags.BoolVar(&cfg.EnablePolicyRouting, "enable-policy-routing", false, "Enable policy-based routing on gateway interfaces (deprecated, UNBOUNDED-FORWARD chain rules replace PBR)") - - // Tunnel-interface configuration flags. All three shared tunnel device - // names must be non-empty, distinct, and must not collide with - // "unbounded0" (the agent's eBPF dummy device). Kernel interface names - // are limited to 15 bytes. - flags.IntVar(&cfg.GenevePort, "geneve-port", 6081, "GENEVE UDP destination port") - flags.IntVar(&cfg.GeneveVNI, "geneve-vni", 1, "GENEVE Virtual Network Identifier") - flags.StringVar(&cfg.GeneveInterfaceName, "geneve-interface", "geneve0", "Shared flow-based GENEVE interface name") - flags.StringVar(&cfg.VXLANInterfaceName, "vxlan-interface", "vxlan0", "Shared flow-based VXLAN interface name") - flags.StringVar(&cfg.IPIPInterfaceName, "ipip-interface", "ipip0", "Shared flow-based IPIP interface name") - flags.StringVar(&cfg.WireGuardInterfacePrefix, "wireguard-interface-prefix", "wg", "Prefix for per-port WireGuard interfaces; runtime name is ") - flags.IntVar(&cfg.VXLANPort, "vxlan-port", 4789, "VXLAN UDP destination port") - flags.IntVar(&cfg.VXLANSrcPortLow, "vxlan-src-port-low", 47891, "VXLAN UDP source port range low (narrow range reduces VM flow count in cloud platforms)") - flags.IntVar(&cfg.VXLANSrcPortHigh, "vxlan-src-port-high", 47922, "VXLAN UDP source port range high (narrow range reduces VM flow count in cloud platforms)") - flags.StringVar(&cfg.PreferredPrivateEncap, "preferred-private-encap", "GENEVE", "Preferred encapsulation for private networks (GENEVE, IPIP, VXLAN, WireGuard)") - flags.StringVar(&cfg.PreferredPublicEncap, "preferred-public-encap", "WireGuard", "Preferred encapsulation for public networks (WireGuard, IPIP, GENEVE, VXLAN)") - - // Status push flags - flags.BoolVar(&cfg.StatusPushEnabled, "status-push-enabled", true, "Enable pushing node status to controller") - flags.StringVar(&cfg.StatusPushURL, "status-push-url", "", "Controller URL for status push (default: use UNBOUNDED_NET_CONTROLLER_SERVICE_HOST/PORT)") - flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 60*time.Second, "Interval between status pushes to controller") - flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 60*time.Second, "Interval between status pushes via aggregated API server") - flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "Enable delta mode for periodic HTTP status push") - flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "Enable websocket status push to controller") - flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "Controller websocket URL for status push (default: ws://service/status/nodews)") - flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "API server websocket mode: never, fallback, preferred") - flags.StringVar(&cfg.StatusWSAPIServerURL, "status-ws-apiserver-url", "", "API server websocket URL for status push fallback (default: wss://$(KUBERNETES_SERVICE_HOST)/apis/status.net.unbounded-cloud.io/v1alpha1/status/nodews)") - flags.DurationVar(&cfg.StatusWSAPIServerStartupDelay, "status-ws-apiserver-startup-delay", 60*time.Second, "Delay before API server websocket/push fallback is allowed after startup (0 to disable delay)") - flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings (0 to disable)") - flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before reconnect") - flags.BoolVar(&cfg.RemoveConfigurationOnShutdown, "remove-configuration-on-shutdown", false, "Remove all managed configuration (WireGuard, routes, masquerade, tunnel interfaces) on shutdown") - flags.BoolVar(&cfg.RemoveWireGuardOnShutdown, "shutdown-remove-wireguard-configuration", false, "Remove WireGuard interfaces/configuration on shutdown (deprecated: use --remove-configuration-on-shutdown)") - flags.BoolVar(&cfg.CleanupNetlinkOnShutdown, "shutdown-cleanup-netlink", false, "Remove managed netlink routes/policy rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") - flags.BoolVar(&cfg.RemoveMasqueradeOnShutdown, "shutdown-remove-masquerade-rules", false, "Remove managed masquerade rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") - flags.IntVar(&cfg.HealthCheckPort, "healthcheck-port", 9997, "UDP port for health check probes") - flags.IntVar(&cfg.BaseMetric, "base-metric", 1, "Base metric for programmed routes") - flags.IntVar(&cfg.RouteTableID, "route-table-id", 252, "Route table ID for managed routes (default 252, set to 254 for main table)") - flags.DurationVar(&cfg.CriticalDeltaEvery, "status-critical-interval", 15*time.Second, "Maximum critical delta publish frequency; changed fields are queued up to this interval for batching") - flags.DurationVar(&cfg.StatsDeltaEvery, "status-stats-interval", 60*time.Second, "Maximum statistics delta publish frequency; changed fields are queued up to this interval for batching") - flags.DurationVar(&cfg.FullSyncEvery, "status-full-sync-interval", 2*time.Minute, "Forced full status sync interval; ensures controller has complete status periodically") - flags.DurationVar(&cfg.HealthFlapMaxBackoff, "health-flap-max-backoff", 120*time.Second, "Maximum backoff duration for health check flap dampening") - flags.DurationVar(&cfg.KubeProxyHealthInterval, "kube-proxy-health-interval", 30*time.Second, "Interval between kube-proxy health checks (0 to disable)") - flags.DurationVar(&cfg.NetlinkResyncPeriod, "netlink-resync-period", 300*time.Second, "Interval between full netlink cache resyncs") - flags.IntVar(&cfg.TunnelDataplaneMapSize, "tunnel-dataplane-map-size", 16384, "Maximum LPM trie entries for eBPF tunnel map") - flags.StringVar(&cfg.TunnelIPFamily, "tunnel-ip-family", "IPv4", "Tunnel underlay IP family: IPv4 (default) or IPv6") - - if err := rootCmd.Execute(); err != nil { - os.Exit(1) - } -} - -func applyNodeRuntimeConfig(cmd *cobra.Command, cfg *config) error { - runtimeCfg, err := configpkg.LoadRuntimeConfig(cfg.ConfigFile) - if err != nil { - return err - } - - flags := cmd.Flags() - nodeCfg := runtimeCfg.Node - - if !flags.Changed("informer-resync-period") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.InformerResyncPeriod, "node.informerResyncPeriod"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.InformerResyncPeriod = d - } - } - - if !flags.Changed("node-name") && nodeCfg.NodeName != "" { - cfg.NodeName = nodeCfg.NodeName - } - - if !flags.Changed("cni-conf-dir") && nodeCfg.CNIConfDir != "" { - cfg.CNIConfDir = nodeCfg.CNIConfDir - } - - if !flags.Changed("cni-conf-file") && nodeCfg.CNIConfFile != "" { - cfg.CNIConfFile = nodeCfg.CNIConfFile - } - - if !flags.Changed("bridge-name") && nodeCfg.BridgeName != "" { - cfg.BridgeName = nodeCfg.BridgeName - } - - if !flags.Changed("wireguard-dir") && nodeCfg.WireGuardDir != "" { - cfg.WireGuardDir = nodeCfg.WireGuardDir - } - - if !flags.Changed("wireguard-port") && nodeCfg.WireGuardPort != nil { - cfg.WireGuardPort = *nodeCfg.WireGuardPort - } - - if !flags.Changed("enable-policy-routing") && nodeCfg.EnablePolicyRouting != nil { //nolint:staticcheck // intentional use of deprecated field for backward compat - cfg.EnablePolicyRouting = *nodeCfg.EnablePolicyRouting //nolint:staticcheck // intentional use of deprecated field - } - - if !flags.Changed("mtu") && nodeCfg.MTU != nil { - cfg.MTU = *nodeCfg.MTU - } - - if !flags.Changed("health-port") && nodeCfg.HealthPort != nil { - cfg.HealthPort = *nodeCfg.HealthPort - } - - if !flags.Changed("status-push-enabled") && nodeCfg.StatusPushEnabled != nil { - cfg.StatusPushEnabled = *nodeCfg.StatusPushEnabled - } - - if !flags.Changed("status-push-url") && nodeCfg.StatusPushURL != "" { - cfg.StatusPushURL = nodeCfg.StatusPushURL - } - - if !flags.Changed("status-push-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushInterval, "node.statusPushInterval"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.StatusPushInterval = d - } - } - - if !flags.Changed("status-push-apiserver-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushAPIServerInterval, "node.statusPushApiserverInterval"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.StatusPushAPIServerInterval = d - } - } - - if !flags.Changed("status-push-delta") && nodeCfg.StatusPushDelta != nil { - cfg.StatusPushDelta = *nodeCfg.StatusPushDelta - } - - if !flags.Changed("status-ws-enabled") && nodeCfg.StatusWSEnabled != nil { - cfg.StatusWSEnabled = *nodeCfg.StatusWSEnabled - } - - if !flags.Changed("status-ws-url") && nodeCfg.StatusWSURL != "" { - cfg.StatusWSURL = nodeCfg.StatusWSURL - } - - if !flags.Changed("status-ws-apiserver-mode") && nodeCfg.StatusWSAPIServerMode != "" { - cfg.StatusWSAPIServerMode = nodeCfg.StatusWSAPIServerMode - } - - if !flags.Changed("status-ws-apiserver-url") && nodeCfg.StatusWSAPIServerURL != "" { - cfg.StatusWSAPIServerURL = nodeCfg.StatusWSAPIServerURL - } - - if !flags.Changed("status-ws-apiserver-startup-delay") && nodeCfg.StatusWSAPIServerStartupDelay != "" { - d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSAPIServerStartupDelay, "node.statusWebsocketApiserverStartupDelay") - if parseErr != nil { - return parseErr - } - - if d < 0 { - return fmt.Errorf("node.statusWebsocketApiserverStartupDelay must be >= 0") - } - - cfg.StatusWSAPIServerStartupDelay = d - } - - if !flags.Changed("status-ws-keepalive-interval") && nodeCfg.StatusWSKeepaliveInterval != "" { - d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSKeepaliveInterval, "node.statusWebsocketKeepaliveInterval") - if parseErr != nil { - return parseErr - } - - cfg.StatusWSKeepaliveInterval = d - } - - if !flags.Changed("status-ws-keepalive-failure-count") && nodeCfg.StatusWSKeepaliveFailCount != nil { - cfg.StatusWSKeepaliveFailureCount = *nodeCfg.StatusWSKeepaliveFailCount - } - // New consolidated shutdown cleanup flag. - if !flags.Changed("remove-configuration-on-shutdown") && nodeCfg.RemoveConfigurationOnShutdown != nil { - cfg.RemoveConfigurationOnShutdown = *nodeCfg.RemoveConfigurationOnShutdown - } - // Deprecated individual shutdown cleanup flags (kept for backward compatibility). - if !flags.Changed("shutdown-remove-wireguard-configuration") && nodeCfg.ShutdownRemoveWireGuardConfiguration != nil { - cfg.RemoveWireGuardOnShutdown = *nodeCfg.ShutdownRemoveWireGuardConfiguration - } - - if !flags.Changed("shutdown-cleanup-netlink") && nodeCfg.ShutdownRemoveIPRoutes != nil { - cfg.CleanupNetlinkOnShutdown = *nodeCfg.ShutdownRemoveIPRoutes - } - - if !flags.Changed("shutdown-remove-masquerade-rules") && nodeCfg.ShutdownRemoveMasqueradeRules != nil { - cfg.RemoveMasqueradeOnShutdown = *nodeCfg.ShutdownRemoveMasqueradeRules - } - // If any deprecated flag is true, activate the consolidated flag. - if cfg.RemoveWireGuardOnShutdown || cfg.CleanupNetlinkOnShutdown || cfg.RemoveMasqueradeOnShutdown { - cfg.RemoveConfigurationOnShutdown = true - } - - if _, err := parseStatusWSAPIServerMode(cfg.StatusWSAPIServerMode); err != nil { - return err - } - - if !flags.Changed("status-critical-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.CriticalDeltaEvery, "node.criticalDeltaEvery"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.CriticalDeltaEvery = d - } - } - - if !flags.Changed("status-stats-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatsDeltaEvery, "node.statsDeltaEvery"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.StatsDeltaEvery = d - } - } - - if !flags.Changed("status-full-sync-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.FullSyncEvery, "node.fullSyncEvery"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.FullSyncEvery = d - } - } - - if cfg.StatusWSKeepaliveFailureCount < 1 { - return fmt.Errorf("node.statusWsKeepaliveFailureCount must be >= 1") - } - - // Apply preferred encapsulation from config file if not set via CLI. - if !flags.Changed("preferred-private-encap") && nodeCfg.PreferredPrivateNetworkEncapsulation != "" { - cfg.PreferredPrivateEncap = nodeCfg.PreferredPrivateNetworkEncapsulation - } - - if !flags.Changed("preferred-public-encap") && nodeCfg.PreferredPublicNetworkEncapsulation != "" { - cfg.PreferredPublicEncap = nodeCfg.PreferredPublicNetworkEncapsulation - } - - if !flags.Changed("health-flap-max-backoff") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.HealthFlapMaxBackoff, "node.healthFlapMaxBackoff"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.HealthFlapMaxBackoff = d - } - } - - if !flags.Changed("route-table-id") && nodeCfg.RouteTableID != nil { - cfg.RouteTableID = *nodeCfg.RouteTableID - } - - if !flags.Changed("kube-proxy-health-interval") && nodeCfg.KubeProxyHealthInterval != "" { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.KubeProxyHealthInterval, "node.kubeProxyHealthInterval"); parseErr != nil { - return parseErr - } else { - cfg.KubeProxyHealthInterval = d - } - } - - if !flags.Changed("netlink-resync-period") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.NetlinkResyncPeriod, "node.netlinkResyncPeriod"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.NetlinkResyncPeriod = d - } - } - - if !flags.Changed("tunnel-dataplane-map-size") && nodeCfg.TunnelDataplaneMapSize != nil { - cfg.TunnelDataplaneMapSize = *nodeCfg.TunnelDataplaneMapSize - } - - if !flags.Changed("tunnel-ip-family") && nodeCfg.TunnelIPFamily != "" { - cfg.TunnelIPFamily = nodeCfg.TunnelIPFamily - } - - if !flags.Changed("vxlan-src-port-low") && nodeCfg.VXLANSrcPortLow != nil { - cfg.VXLANSrcPortLow = *nodeCfg.VXLANSrcPortLow - } - - if !flags.Changed("vxlan-src-port-high") && nodeCfg.VXLANSrcPortHigh != nil { - cfg.VXLANSrcPortHigh = *nodeCfg.VXLANSrcPortHigh - } - - if !flags.Changed("geneve-interface") && nodeCfg.GeneveInterfaceName != "" { - cfg.GeneveInterfaceName = nodeCfg.GeneveInterfaceName - } - - if !flags.Changed("vxlan-interface") && nodeCfg.VXLANInterfaceName != "" { - cfg.VXLANInterfaceName = nodeCfg.VXLANInterfaceName - } - - if !flags.Changed("ipip-interface") && nodeCfg.IPIPInterfaceName != "" { - cfg.IPIPInterfaceName = nodeCfg.IPIPInterfaceName - } - - if !flags.Changed("wireguard-interface-prefix") && nodeCfg.WireGuardInterfacePrefix != "" { - cfg.WireGuardInterfacePrefix = nodeCfg.WireGuardInterfacePrefix - } - - // Validate and default tunnelIPFamily - switch cfg.TunnelIPFamily { - case "IPv4", "IPv6": - // valid - case "": - cfg.TunnelIPFamily = "IPv4" - default: - return fmt.Errorf("invalid tunnel-ip-family %q: must be 'IPv4' or 'IPv6'", cfg.TunnelIPFamily) - } - - if err := validateTunnelInterfaceNames(cfg.GeneveInterfaceName, cfg.VXLANInterfaceName, cfg.IPIPInterfaceName); err != nil { - return err - } - - if err := validateWireGuardInterfacePrefix(cfg.WireGuardInterfacePrefix); err != nil { - return err - } - - // Normalize MTU: treat 0 as 1280 (the IPv6 minimum, safe for all links). - if cfg.MTU == 0 { - cfg.MTU = 1280 - } - - // Apply common config. - if !flags.Changed("apiserver-url") && runtimeCfg.Common.ApiserverURL != "" { - cfg.ApiserverURL = runtimeCfg.Common.ApiserverURL - } - - return nil -} - -func run(cfg *config) error { - klog.Infof("unbounded-net-node version=%s commit=%s built=%s", version.Version, version.GitCommit, version.BuildTime) - - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() - // Handle shutdown signals - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sigCh - klog.Info("Received shutdown signal") - cancel() - }() - - // Validate node name - if cfg.NodeName == "" { - klog.Fatal("Node name is required. Set NODE_NAME environment variable or use --node-name flag") - } - - klog.Infof("Running on node: %s", cfg.NodeName) - - if cfg.EnablePolicyRouting { - klog.Info("Policy-based routing on gateway interfaces is enabled") - } else { - klog.Info("Policy-based routing on gateway interfaces is disabled") - } - - // Build Kubernetes client - var ( - restConfig *rest.Config - err error - ) - - if cfg.KubeconfigPath != "" { - restConfig, err = clientcmd.BuildConfigFromFlags(cfg.ApiserverURL, cfg.KubeconfigPath) - } else { - restConfig, err = rest.InClusterConfig() - if err == nil && cfg.ApiserverURL != "" { - restConfig.Host = cfg.ApiserverURL - } - } - - if err != nil { - klog.Fatalf("Failed to build kubeconfig: %v", err) - } - - if cfg.ApiserverURL != "" { - klog.Infof("Using API server URL override: %s", cfg.ApiserverURL) - } - - // Wire client-go metrics into Prometheus before creating clients. - metrics.RegisterClientGoMetrics() - - clientset, err := kubernetes.NewForConfig(restConfig) - if err != nil { - klog.Fatalf("Failed to create Kubernetes client: %v", err) - } - - dynamicClient, err := dynamic.NewForConfig(restConfig) - if err != nil { - klog.Fatalf("Failed to create dynamic Kubernetes client: %v", err) - } - - // Generate WireGuard keys and annotate node - pubKey, err := ensureWireGuardKeys(cfg) - if err != nil { - klog.Fatalf("Failed to ensure WireGuard keys: %v", err) - } - - klog.Infof("WireGuard public key: %s", pubKey) - - if err := annotateNodeWithPubKey(ctx, clientset, cfg.NodeName, pubKey); err != nil { - klog.Fatalf("Failed to annotate node with WireGuard public key: %v", err) - } - - klog.Info("Node annotated with WireGuard public key") - - // Detect and annotate the node's maximum tunnel MTU so the controller - // can validate that the configured MTU is compatible across all nodes. - if detectedMTU := unboundednetnetlink.DetectDefaultRouteMTU(); detectedMTU > 0 { - wgMTU := detectedMTU - unboundednetnetlink.WireGuardMTUOverhead - if err := annotateNodeWithMTU(ctx, clientset, cfg.NodeName, wgMTU); err != nil { - klog.Warningf("Failed to annotate node with tunnel MTU: %v", err) - } else { - klog.Infof("Node annotated with tunnel MTU %d (detected default route MTU %d - %d overhead)", wgMTU, detectedMTU, unboundednetnetlink.WireGuardMTUOverhead) - } - } - - // Watch the config file for dynamic log level changes. - go configpkg.WatchConfigLogLevel(ctx, cfg.ConfigFile) - - // Warn if public network traffic will be sent unencrypted. - if cfg.PreferredPublicEncap != "" && cfg.PreferredPublicEncap != "WireGuard" { - klog.Warningf("WARNING: preferredPublicNetworkEncapsulation is set to %q -- traffic over public networks will be sent UNENCRYPTED", cfg.PreferredPublicEncap) - } - - // Create informers early - before any CRD-based lookups - // This allows us to use the informer cache for all CRD operations - informerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, cfg.InformerResyncPeriod) - sliceInformer := informerFactory.ForResource(siteNodeSliceGVR).Informer() - siteInformer := informerFactory.ForResource(siteGVR).Informer() - gatewayPoolInformer := informerFactory.ForResource(gatewayPoolGVR).Informer() - gatewayNodeInformer := informerFactory.ForResource(gatewayNodeGVR).Informer() - sitePeeringInformer := informerFactory.ForResource(sitePeeringGVR).Informer() - assignmentInformer := informerFactory.ForResource(siteGatewayPoolAssignmentGVR).Informer() - poolPeeringInformer := informerFactory.ForResource(gatewayPoolPeeringGVR).Informer() - - // Start the informers - informerFactory.Start(ctx.Done()) - - // Wait for caches to sync - klog.Info("Waiting for informer caches to sync") - - if !cache.WaitForCacheSync(ctx.Done(), sliceInformer.HasSynced, siteInformer.HasSynced, gatewayPoolInformer.HasSynced, gatewayNodeInformer.HasSynced, sitePeeringInformer.HasSynced, assignmentInformer.HasSynced, poolPeeringInformer.HasSynced) { - return fmt.Errorf("failed to sync informer caches") - } - - klog.Info("Informer caches synced") - - // Start the netlink cache (read-only network state snapshot). - netlinkCache := unboundednetnetlink.NewNetlinkCache(cfg.NetlinkResyncPeriod) - if err := netlinkCache.Start(ctx); err != nil { - klog.Fatalf("Failed to start netlink cache: %v", err) - } - - // Track if CNI is configured for health checks - cniConfigured := false - - // Create shared health state for health server - healthState := &nodeHealthState{ - cniConfigured: &cniConfigured, - informersSynced: []cache.InformerSynced{ - sliceInformer.HasSynced, - siteInformer.HasSynced, - gatewayPoolInformer.HasSynced, - gatewayNodeInformer.HasSynced, - sitePeeringInformer.HasSynced, - assignmentInformer.HasSynced, - poolPeeringInformer.HasSynced, - }, - } - - // Start health server if enabled (readiness should not wait on site membership) - if cfg.HealthPort > 0 { - go startHealthServer(cfg.HealthPort, healthState) - } - - // Check if this node is a gateway node by checking the informer cache - isGatewayNode := isGatewayNodeFromCRDs(gatewayPoolInformer, pubKey) - if isGatewayNode { - klog.Info("Node is a gateway node (found in GatewayPool status)") - } - - // Wait for this node to appear in a SiteNodeSlice or GatewayPool - // This ensures the site controller has processed this node before we continue - mySiteName, err := waitForSiteMembership(ctx, sliceInformer, gatewayPoolInformer, pubKey) - if err != nil { - if err == context.Canceled { - return nil - } - - return err - } - - // Check if this node's site has manageCniPlugin enabled using the informer cache - manageCniPlugin := manageCNIForMembership(siteInformer, mySiteName, isGatewayNode) - - var nodePodCIDRs []string - if manageCniPlugin { - // Wait for podCIDRs and configure CNI - nodePodCIDRs, err = waitForPodCIDRsAndConfigure(ctx, clientset, cfg, &cniConfigured) - if err != nil { - if err == context.Canceled { - return nil - } - - return err - } - } else { - klog.Info("manageCniPlugin is false for this site - skipping CNI configuration") - // Still need to get the node's podCIDRs for WireGuard gateway IP calculation - node, err := clientset.CoreV1().Nodes().Get(ctx, cfg.NodeName, metav1.GetOptions{}) - if err != nil { - klog.Fatalf("Failed to get node: %v", err) - } - - nodePodCIDRs = node.Spec.PodCIDRs - // Mark CNI as "configured" since we're intentionally not managing it - cniConfigured = true + cmd := nodeagent.NewCommand() + cmd.SetContext(ctx) + if err := cmd.Execute(); err != nil { + os.Exit(1) } - - // After CNI is configured (or skipped), watch Site CRD for WireGuard peers - // Pass the node's podCIDRs so routes can be configured with preferred source IPs - // Also pass the informers so they can be reused (already synced) - return watchSiteAndConfigureWireGuard(ctx, clientset, dynamicClient, cfg, pubKey, nodePodCIDRs, manageCniPlugin, healthState, netlinkCache, siteInformer, sliceInformer, gatewayPoolInformer, gatewayNodeInformer, sitePeeringInformer, assignmentInformer, poolPeeringInformer) } diff --git a/cmd/unbounded-net-node/bootstrap_helpers.go b/internal/net/nodeagent/bootstrap_helpers.go similarity index 99% rename from cmd/unbounded-net-node/bootstrap_helpers.go rename to internal/net/nodeagent/bootstrap_helpers.go index fcd4af565..df5af285a 100644 --- a/cmd/unbounded-net-node/bootstrap_helpers.go +++ b/internal/net/nodeagent/bootstrap_helpers.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/bootstrap_helpers_test.go b/internal/net/nodeagent/bootstrap_helpers_test.go similarity index 99% rename from cmd/unbounded-net-node/bootstrap_helpers_test.go rename to internal/net/nodeagent/bootstrap_helpers_test.go index 0bc3a1e8c..b27fb0683 100644 --- a/cmd/unbounded-net-node/bootstrap_helpers_test.go +++ b/internal/net/nodeagent/bootstrap_helpers_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/bpf_status.go b/internal/net/nodeagent/bpf_status.go similarity index 99% rename from cmd/unbounded-net-node/bpf_status.go rename to internal/net/nodeagent/bpf_status.go index 0fae10870..27c2a5f10 100644 --- a/cmd/unbounded-net-node/bpf_status.go +++ b/internal/net/nodeagent/bpf_status.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "fmt" diff --git a/cmd/unbounded-net-node/encapsulation.go b/internal/net/nodeagent/encapsulation.go similarity index 99% rename from cmd/unbounded-net-node/encapsulation.go rename to internal/net/nodeagent/encapsulation.go index 1e6406b2c..4a8897fc0 100644 --- a/cmd/unbounded-net-node/encapsulation.go +++ b/internal/net/nodeagent/encapsulation.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" diff --git a/cmd/unbounded-net-node/encapsulation_test.go b/internal/net/nodeagent/encapsulation_test.go similarity index 99% rename from cmd/unbounded-net-node/encapsulation_test.go rename to internal/net/nodeagent/encapsulation_test.go index 5fc7b021d..eea9e5f82 100644 --- a/cmd/unbounded-net-node/encapsulation_test.go +++ b/internal/net/nodeagent/encapsulation_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "testing" diff --git a/internal/net/nodeagent/external.go b/internal/net/nodeagent/external.go index 0b584c6a0..ef3b18b3d 100644 --- a/internal/net/nodeagent/external.go +++ b/internal/net/nodeagent/external.go @@ -4,9 +4,10 @@ package nodeagent import ( + "context" "fmt" + "os" "path/filepath" - "time" "k8s.io/client-go/rest" ) @@ -18,20 +19,6 @@ type ExternalGatewayOptions struct { RESTConfig *rest.Config } -type config struct { - NodeName string - CNIConfDir string - WireGuardDir string - StatusPushEnabled bool - StatusWSEnabled bool - KubeProxyHealthInterval time.Duration - HealthPort int - RouteTableID int - RemoveConfigurationOnShutdown bool - PreferredPublicEncap string - RESTConfig *rest.Config -} - func externalGatewayConfig(options ExternalGatewayOptions) (*config, error) { if options.NodeName == "" { return nil, fmt.Errorf("external gateway node name is required") @@ -43,17 +30,35 @@ func externalGatewayConfig(options ExternalGatewayOptions) (*config, error) { return nil, fmt.Errorf("external gateway runtime directory must be absolute") } - return &config{ - NodeName: options.NodeName, - CNIConfDir: filepath.Join(options.RuntimeDir, "cni"), - WireGuardDir: filepath.Join(options.RuntimeDir, "wireguard"), - StatusPushEnabled: false, - StatusWSEnabled: false, - KubeProxyHealthInterval: 0, - HealthPort: 0, - RouteTableID: 254, - RemoveConfigurationOnShutdown: true, - PreferredPublicEncap: "WireGuard", - RESTConfig: options.RESTConfig, - }, nil + cfg := defaultConfig() + cfg.NodeName = options.NodeName + cfg.CNIConfDir = filepath.Join(options.RuntimeDir, "cni") + cfg.WireGuardDir = filepath.Join(options.RuntimeDir, "wireguard") + cfg.StatusPushEnabled = false + cfg.StatusWSEnabled = false + cfg.KubeProxyHealthInterval = 0 + cfg.HealthPort = 0 + cfg.RouteTableID = 254 + cfg.RemoveConfigurationOnShutdown = true + cfg.PreferredPublicEncap = "WireGuard" + cfg.RESTConfig = options.RESTConfig + + return cfg, nil +} + +// RunExternalGateway runs the normal node dataplane with external-gateway-safe +// defaults until ctx is cancelled. +func RunExternalGateway(ctx context.Context, options ExternalGatewayOptions) error { + cfg, err := externalGatewayConfig(options) + if err != nil { + return err + } + if err := os.MkdirAll(cfg.WireGuardDir, 0o700); err != nil { + return fmt.Errorf("create WireGuard runtime directory: %w", err) + } + if err := os.MkdirAll(cfg.CNIConfDir, 0o700); err != nil { + return fmt.Errorf("create CNI runtime directory: %w", err) + } + + return run(ctx, cfg) } diff --git a/cmd/unbounded-net-node/gateway_routes.go b/internal/net/nodeagent/gateway_routes.go similarity index 99% rename from cmd/unbounded-net-node/gateway_routes.go rename to internal/net/nodeagent/gateway_routes.go index 73ec34e6b..2692dc88a 100644 --- a/cmd/unbounded-net-node/gateway_routes.go +++ b/internal/net/nodeagent/gateway_routes.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/gateway_routes_test.go b/internal/net/nodeagent/gateway_routes_test.go similarity index 99% rename from cmd/unbounded-net-node/gateway_routes_test.go rename to internal/net/nodeagent/gateway_routes_test.go index 6e6a5aff3..fde76fcbd 100644 --- a/cmd/unbounded-net-node/gateway_routes_test.go +++ b/internal/net/nodeagent/gateway_routes_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/kube_proxy_monitor.go b/internal/net/nodeagent/kube_proxy_monitor.go similarity index 99% rename from cmd/unbounded-net-node/kube_proxy_monitor.go rename to internal/net/nodeagent/kube_proxy_monitor.go index 9b701e846..12b2117ca 100644 --- a/cmd/unbounded-net-node/kube_proxy_monitor.go +++ b/internal/net/nodeagent/kube_proxy_monitor.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/kube_proxy_monitor_test.go b/internal/net/nodeagent/kube_proxy_monitor_test.go similarity index 99% rename from cmd/unbounded-net-node/kube_proxy_monitor_test.go rename to internal/net/nodeagent/kube_proxy_monitor_test.go index 26742b721..5fff00985 100644 --- a/cmd/unbounded-net-node/kube_proxy_monitor_test.go +++ b/internal/net/nodeagent/kube_proxy_monitor_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/link_stats_monitor.go b/internal/net/nodeagent/link_stats_monitor.go similarity index 99% rename from cmd/unbounded-net-node/link_stats_monitor.go rename to internal/net/nodeagent/link_stats_monitor.go index 569d30309..42a16c85b 100644 --- a/cmd/unbounded-net-node/link_stats_monitor.go +++ b/internal/net/nodeagent/link_stats_monitor.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/link_stats_monitor_test.go b/internal/net/nodeagent/link_stats_monitor_test.go similarity index 99% rename from cmd/unbounded-net-node/link_stats_monitor_test.go rename to internal/net/nodeagent/link_stats_monitor_test.go index c2196af4b..b1f80d807 100644 --- a/cmd/unbounded-net-node/link_stats_monitor_test.go +++ b/internal/net/nodeagent/link_stats_monitor_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "testing" diff --git a/cmd/unbounded-net-node/main_config_test.go b/internal/net/nodeagent/main_config_test.go similarity index 99% rename from cmd/unbounded-net-node/main_config_test.go rename to internal/net/nodeagent/main_config_test.go index a3af27346..183140180 100644 --- a/cmd/unbounded-net-node/main_config_test.go +++ b/internal/net/nodeagent/main_config_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "os" diff --git a/cmd/unbounded-net-node/main_test.go b/internal/net/nodeagent/main_test.go similarity index 99% rename from cmd/unbounded-net-node/main_test.go rename to internal/net/nodeagent/main_test.go index 3eef81cf1..9685bf821 100644 --- a/cmd/unbounded-net-node/main_test.go +++ b/internal/net/nodeagent/main_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/main_update_test.go b/internal/net/nodeagent/main_update_test.go similarity index 99% rename from cmd/unbounded-net-node/main_update_test.go rename to internal/net/nodeagent/main_update_test.go index 68633f98d..22463cd86 100644 --- a/cmd/unbounded-net-node/main_update_test.go +++ b/internal/net/nodeagent/main_update_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/metrics.go b/internal/net/nodeagent/metrics.go similarity index 99% rename from cmd/unbounded-net-node/metrics.go rename to internal/net/nodeagent/metrics.go index bffd758ee..3265058a4 100644 --- a/cmd/unbounded-net-node/metrics.go +++ b/internal/net/nodeagent/metrics.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "github.com/prometheus/client_golang/prometheus" diff --git a/internal/net/nodeagent/node.go b/internal/net/nodeagent/node.go new file mode 100644 index 000000000..0875b3d4b --- /dev/null +++ b/internal/net/nodeagent/node.go @@ -0,0 +1,795 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodeagent + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + configpkg "github.com/Azure/unbounded/internal/net/config" + "github.com/Azure/unbounded/internal/net/metrics" + unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" + "github.com/Azure/unbounded/internal/version" +) + +// CNIConfig represents the CNI configuration file structure +type CNIConfig struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Plugins []PluginConf `json:"plugins"` +} + +// PluginConf represents a CNI plugin configuration +type PluginConf struct { + Type string `json:"type"` + Bridge string `json:"bridge,omitempty"` + IsGateway bool `json:"isGateway,omitempty"` + IsDefaultGW bool `json:"isDefaultGateway,omitempty"` + ForceAddress bool `json:"forceAddress,omitempty"` + IPMasq bool `json:"ipMasq,omitempty"` + HairpinMode bool `json:"hairpinMode,omitempty"` + MTU int `json:"mtu,omitempty"` + IPAM *IPAMConfig `json:"ipam,omitempty"` + Capabilities *Caps `json:"capabilities,omitempty"` +} + +// IPAMConfig represents the IPAM configuration +type IPAMConfig struct { + Type string `json:"type"` + Ranges [][]IPRange `json:"ranges,omitempty"` +} + +// IPRange represents an IP range for IPAM +type IPRange struct { + Subnet string `json:"subnet"` +} + +// Caps represents plugin capabilities +type Caps struct { + PortMappings bool `json:"portMappings,omitempty"` +} + +type config struct { + ConfigFile string + KubeconfigPath string + ApiserverURL string // Override Kubernetes API server URL (empty = use default) + NodeName string + CNIConfDir string + CNIConfFile string + BridgeName string + WireGuardDir string + WireGuardPort int + EnablePolicyRouting bool + MTU int + HealthPort int + InformerResyncPeriod time.Duration + StatusPushEnabled bool // Whether to push status to controller + StatusPushURL string // Controller URL for status push + StatusPushInterval time.Duration // Interval between status pushes to controller + StatusPushAPIServerInterval time.Duration // Interval between status pushes via aggregated API server + StatusPushDelta bool // Whether periodic HTTP pushes use deltas + StatusWSEnabled bool // Whether websocket push is enabled + StatusWSURL string // Controller websocket URL for status push + StatusWSAPIServerMode string // API server websocket mode: never, fallback, preferred + StatusWSAPIServerURL string // API server websocket URL for status push fallback + StatusWSAPIServerStartupDelay time.Duration // Delay before API server fallback is allowed after startup + StatusWSKeepaliveInterval time.Duration // Interval between websocket keepalive pings (0 disables keepalive) + StatusWSKeepaliveFailureCount int // Sequential websocket keepalive ping failures before reconnect + RemoveConfigurationOnShutdown bool // Remove all managed configuration (WireGuard, routes, masquerade, etc.) on shutdown + RemoveWireGuardOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown + CleanupNetlinkOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown + RemoveMasqueradeOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown + HealthCheckPort int // UDP port for health check probes (default 9997) + BaseMetric int // Base metric for programmed routes (default 1) + RouteTableID int // Route table ID for managed routes (default 252) + CriticalDeltaEvery time.Duration // Maximum critical delta publish frequency; changes are queued up to this interval for batching + StatsDeltaEvery time.Duration // Maximum statistics delta publish frequency; changes are queued up to this interval for batching + FullSyncEvery time.Duration // Forced full status sync interval; ensures controller has complete status periodically + GenevePort int // GENEVE UDP destination port (default 6081) + GeneveVNI int // GENEVE Virtual Network Identifier (default 1) + GeneveInterfaceName string // GENEVE shared tunnel interface name (default geneve0) + VXLANInterfaceName string // VXLAN shared tunnel interface name (default vxlan0) + IPIPInterfaceName string // IPIP shared tunnel interface name (default ipip0) + WireGuardInterfacePrefix string // Prefix for per-port WireGuard interfaces (default "wg"; per-peer name is ) + VXLANPort int // VXLAN UDP destination port (default 4789) + VXLANSrcPortLow int // VXLAN UDP source port range low (default 47891) + VXLANSrcPortHigh int // VXLAN UDP source port range high (default 47922) + PreferredPrivateEncap string // Preferred encap for private/internal networks (GENEVE, IPIP, VXLAN, WireGuard) + PreferredPublicEncap string // Preferred encap for public/external networks (WireGuard, IPIP, GENEVE, VXLAN) + HealthFlapMaxBackoff time.Duration // Maximum backoff duration for health check flap dampening + KubeProxyHealthInterval time.Duration // Interval between kube-proxy health checks (0 to disable) + NetlinkResyncPeriod time.Duration // Interval between full netlink cache resyncs + TunnelDataplaneMapSize int // Maximum LPM trie entries for eBPF tunnel map (default 16384) + TunnelIPFamily string // Tunnel underlay IP family: "IPv4" (default) or "IPv6" + RESTConfig *rest.Config // Explicit client configuration for embedded runners +} + +var siteGVR = schema.GroupVersionResource{ + Group: unboundedv1alpha3.GroupVersion.Group, + Version: unboundedv1alpha3.GroupVersion.Version, + Resource: "sites", +} + +var siteNodeSliceGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "sitenodeslices", +} + +var gatewayPoolGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "gatewaypools", +} + +var gatewayNodeGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "gatewaypoolnodes", +} + +var sitePeeringGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "sitepeerings", +} + +var siteGatewayPoolAssignmentGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "sitegatewaypoolassignments", +} + +var gatewayPoolPeeringGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "gatewaypoolpeerings", +} + +const ( + // WireGuard public key annotation on the node + WireGuardPubKeyAnnotation = "net.unbounded-cloud.io/wg-pubkey" + + // TunnelMTUAnnotation is the maximum tunnel MTU this node + // can support, based on its default-route interface MTU minus encapsulation + // overhead. The controller uses this to validate that the configured MTU + // does not exceed what any node in the cluster can handle. + TunnelMTUAnnotation = "net.unbounded-cloud.io/tunnel-mtu" + + // Gateway node taint key - prevents regular workloads from running on gateway nodes + // since they don't have regular pod CIDR routing + GatewayNodeTaintKey = "net.unbounded-cloud.io/gateway-node" + + // gatewayNodeHeartbeatInterval controls how frequently the node agent refreshes + // GatewayNode.status.lastUpdated. Route staleness is derived from this cadence. + gatewayNodeHeartbeatInterval = 10 * time.Second +) + +// NewCommand builds the unbounded-net node-agent command. +func NewCommand() *cobra.Command { + // Initialize klog flags + klog.InitFlags(nil) + + // Add klog flags to pflag + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + + cfg := defaultConfig() + if nodeName := os.Getenv("NODE_NAME"); nodeName != "" { + cfg.NodeName = nodeName + } + + rootCmd := &cobra.Command{ + Use: "unbounded-net-node", + Short: "CNI configuration agent for unbounded-net", + Long: `unbounded-net-node runs on each node as a DaemonSet and configures CNI +networking by writing a CNI configuration file based on the node's podCIDRs. + +It watches the node object in Kubernetes and waits for podCIDRs to be assigned +by the unbounded-net-controller. Once assigned, it writes a CNI configuration +file that sets up pod networking using the bridge plugin with host-local IPAM. + +It also generates WireGuard keys for the node and stores them in /etc/wireguard, +then annotates the node with the public key.`, + Version: version.Version + " (commit: " + version.GitCommit + ")", + SilenceUsage: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + return applyNodeRuntimeConfig(cmd, cfg) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return run(cmd.Context(), cfg) + }, + } + + // Add flags + flags := rootCmd.Flags() + + // Change version flag from -v to -V to avoid conflict with klog's -v flag + rootCmd.Flags().BoolP("version", "V", false, "Print version information") + rootCmd.SetVersionTemplate(`{{printf "%s\n" .Version}}`) + + // General flags + flags.StringVar(&cfg.ConfigFile, "config-file", "/etc/unbounded-net/config.yaml", "Path to runtime YAML config file") + flags.StringVar(&cfg.KubeconfigPath, "kubeconfig", "", "Path to kubeconfig file (uses in-cluster config if not specified)") + flags.StringVar(&cfg.ApiserverURL, "apiserver-url", "", "Override Kubernetes API server URL (empty = use default from kubeconfig or in-cluster config)") + flags.StringVar(&cfg.NodeName, "node-name", os.Getenv("NODE_NAME"), "Name of this node (defaults to NODE_NAME env var)") + flags.IntVar(&cfg.HealthPort, "health-port", 9998, "Port for health check HTTP server (0 to disable)") + flags.DurationVar(&cfg.InformerResyncPeriod, "informer-resync-period", 600*time.Second, "Resync period for Kubernetes informers") + + // CNI configuration flags + flags.StringVar(&cfg.CNIConfDir, "cni-conf-dir", "/etc/cni/net.d", "Directory to write CNI configuration") + flags.StringVar(&cfg.CNIConfFile, "cni-conf-file", "10-unbounded.conflist", "Name of the CNI configuration file") + flags.StringVar(&cfg.BridgeName, "bridge-name", "cbr0", "Name of the bridge interface") + flags.IntVar(&cfg.MTU, "mtu", 1280, "MTU for WireGuard and bridge interfaces (default 1280, the IPv6 minimum)") + + // WireGuard configuration flags + flags.StringVar(&cfg.WireGuardDir, "wireguard-dir", "/etc/wireguard", "Directory to store WireGuard keys") + flags.IntVar(&cfg.WireGuardPort, "wireguard-port", 51820, "WireGuard listen port") + flags.BoolVar(&cfg.EnablePolicyRouting, "enable-policy-routing", false, "Enable policy-based routing on gateway interfaces (deprecated, UNBOUNDED-FORWARD chain rules replace PBR)") + + // Tunnel-interface configuration flags. All three shared tunnel device + // names must be non-empty, distinct, and must not collide with + // "unbounded0" (the agent's eBPF dummy device). Kernel interface names + // are limited to 15 bytes. + flags.IntVar(&cfg.GenevePort, "geneve-port", 6081, "GENEVE UDP destination port") + flags.IntVar(&cfg.GeneveVNI, "geneve-vni", 1, "GENEVE Virtual Network Identifier") + flags.StringVar(&cfg.GeneveInterfaceName, "geneve-interface", "geneve0", "Shared flow-based GENEVE interface name") + flags.StringVar(&cfg.VXLANInterfaceName, "vxlan-interface", "vxlan0", "Shared flow-based VXLAN interface name") + flags.StringVar(&cfg.IPIPInterfaceName, "ipip-interface", "ipip0", "Shared flow-based IPIP interface name") + flags.StringVar(&cfg.WireGuardInterfacePrefix, "wireguard-interface-prefix", "wg", "Prefix for per-port WireGuard interfaces; runtime name is ") + flags.IntVar(&cfg.VXLANPort, "vxlan-port", 4789, "VXLAN UDP destination port") + flags.IntVar(&cfg.VXLANSrcPortLow, "vxlan-src-port-low", 47891, "VXLAN UDP source port range low (narrow range reduces VM flow count in cloud platforms)") + flags.IntVar(&cfg.VXLANSrcPortHigh, "vxlan-src-port-high", 47922, "VXLAN UDP source port range high (narrow range reduces VM flow count in cloud platforms)") + flags.StringVar(&cfg.PreferredPrivateEncap, "preferred-private-encap", "GENEVE", "Preferred encapsulation for private networks (GENEVE, IPIP, VXLAN, WireGuard)") + flags.StringVar(&cfg.PreferredPublicEncap, "preferred-public-encap", "WireGuard", "Preferred encapsulation for public networks (WireGuard, IPIP, GENEVE, VXLAN)") + + // Status push flags + flags.BoolVar(&cfg.StatusPushEnabled, "status-push-enabled", true, "Enable pushing node status to controller") + flags.StringVar(&cfg.StatusPushURL, "status-push-url", "", "Controller URL for status push (default: use UNBOUNDED_NET_CONTROLLER_SERVICE_HOST/PORT)") + flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 60*time.Second, "Interval between status pushes to controller") + flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 60*time.Second, "Interval between status pushes via aggregated API server") + flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "Enable delta mode for periodic HTTP status push") + flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "Enable websocket status push to controller") + flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "Controller websocket URL for status push (default: ws://service/status/nodews)") + flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "API server websocket mode: never, fallback, preferred") + flags.StringVar(&cfg.StatusWSAPIServerURL, "status-ws-apiserver-url", "", "API server websocket URL for status push fallback (default: wss://$(KUBERNETES_SERVICE_HOST)/apis/status.net.unbounded-cloud.io/v1alpha1/status/nodews)") + flags.DurationVar(&cfg.StatusWSAPIServerStartupDelay, "status-ws-apiserver-startup-delay", 60*time.Second, "Delay before API server websocket/push fallback is allowed after startup (0 to disable delay)") + flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings (0 to disable)") + flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before reconnect") + flags.BoolVar(&cfg.RemoveConfigurationOnShutdown, "remove-configuration-on-shutdown", false, "Remove all managed configuration (WireGuard, routes, masquerade, tunnel interfaces) on shutdown") + flags.BoolVar(&cfg.RemoveWireGuardOnShutdown, "shutdown-remove-wireguard-configuration", false, "Remove WireGuard interfaces/configuration on shutdown (deprecated: use --remove-configuration-on-shutdown)") + flags.BoolVar(&cfg.CleanupNetlinkOnShutdown, "shutdown-cleanup-netlink", false, "Remove managed netlink routes/policy rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") + flags.BoolVar(&cfg.RemoveMasqueradeOnShutdown, "shutdown-remove-masquerade-rules", false, "Remove managed masquerade rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") + flags.IntVar(&cfg.HealthCheckPort, "healthcheck-port", 9997, "UDP port for health check probes") + flags.IntVar(&cfg.BaseMetric, "base-metric", 1, "Base metric for programmed routes") + flags.IntVar(&cfg.RouteTableID, "route-table-id", 252, "Route table ID for managed routes (default 252, set to 254 for main table)") + flags.DurationVar(&cfg.CriticalDeltaEvery, "status-critical-interval", 15*time.Second, "Maximum critical delta publish frequency; changed fields are queued up to this interval for batching") + flags.DurationVar(&cfg.StatsDeltaEvery, "status-stats-interval", 60*time.Second, "Maximum statistics delta publish frequency; changed fields are queued up to this interval for batching") + flags.DurationVar(&cfg.FullSyncEvery, "status-full-sync-interval", 2*time.Minute, "Forced full status sync interval; ensures controller has complete status periodically") + flags.DurationVar(&cfg.HealthFlapMaxBackoff, "health-flap-max-backoff", 120*time.Second, "Maximum backoff duration for health check flap dampening") + flags.DurationVar(&cfg.KubeProxyHealthInterval, "kube-proxy-health-interval", 30*time.Second, "Interval between kube-proxy health checks (0 to disable)") + flags.DurationVar(&cfg.NetlinkResyncPeriod, "netlink-resync-period", 300*time.Second, "Interval between full netlink cache resyncs") + flags.IntVar(&cfg.TunnelDataplaneMapSize, "tunnel-dataplane-map-size", 16384, "Maximum LPM trie entries for eBPF tunnel map") + flags.StringVar(&cfg.TunnelIPFamily, "tunnel-ip-family", "IPv4", "Tunnel underlay IP family: IPv4 (default) or IPv6") + + return rootCmd +} + +func defaultConfig() *config { + return &config{ + ConfigFile: "/etc/unbounded-net/config.yaml", + CNIConfDir: "/etc/cni/net.d", + CNIConfFile: "10-unbounded.conflist", + BridgeName: "cbr0", + WireGuardDir: "/etc/wireguard", + WireGuardPort: 51820, + MTU: 1280, + HealthPort: 9998, + InformerResyncPeriod: 600 * time.Second, + StatusPushEnabled: true, + StatusPushInterval: 10 * time.Second, + StatusPushAPIServerInterval: 30 * time.Second, + StatusPushDelta: true, + StatusWSEnabled: true, + StatusWSAPIServerMode: statusWSAPIServerModeFallback, + StatusWSAPIServerStartupDelay: 60 * time.Second, + StatusWSKeepaliveInterval: 10 * time.Second, + StatusWSKeepaliveFailureCount: 2, + CriticalDeltaEvery: time.Second, + StatsDeltaEvery: 15 * time.Second, + FullSyncEvery: 2 * time.Minute, + GenevePort: 6081, + GeneveVNI: 1, + GeneveInterfaceName: "geneve0", + VXLANInterfaceName: "vxlan0", + IPIPInterfaceName: "ipip0", + WireGuardInterfacePrefix: "wg", + VXLANPort: 4789, + VXLANSrcPortLow: 47891, + VXLANSrcPortHigh: 47922, + PreferredPrivateEncap: "GENEVE", + PreferredPublicEncap: "WireGuard", + NetlinkResyncPeriod: 300 * time.Second, + TunnelDataplaneMapSize: 16384, + TunnelIPFamily: "IPv4", + } +} + +func applyNodeRuntimeConfig(cmd *cobra.Command, cfg *config) error { + runtimeCfg, err := configpkg.LoadRuntimeConfig(cfg.ConfigFile) + if err != nil { + return err + } + + flags := cmd.Flags() + nodeCfg := runtimeCfg.Node + + if !flags.Changed("informer-resync-period") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.InformerResyncPeriod, "node.informerResyncPeriod"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.InformerResyncPeriod = d + } + } + + if !flags.Changed("node-name") && nodeCfg.NodeName != "" { + cfg.NodeName = nodeCfg.NodeName + } + + if !flags.Changed("cni-conf-dir") && nodeCfg.CNIConfDir != "" { + cfg.CNIConfDir = nodeCfg.CNIConfDir + } + + if !flags.Changed("cni-conf-file") && nodeCfg.CNIConfFile != "" { + cfg.CNIConfFile = nodeCfg.CNIConfFile + } + + if !flags.Changed("bridge-name") && nodeCfg.BridgeName != "" { + cfg.BridgeName = nodeCfg.BridgeName + } + + if !flags.Changed("wireguard-dir") && nodeCfg.WireGuardDir != "" { + cfg.WireGuardDir = nodeCfg.WireGuardDir + } + + if !flags.Changed("wireguard-port") && nodeCfg.WireGuardPort != nil { + cfg.WireGuardPort = *nodeCfg.WireGuardPort + } + + if !flags.Changed("enable-policy-routing") && nodeCfg.EnablePolicyRouting != nil { //nolint:staticcheck // intentional use of deprecated field for backward compat + cfg.EnablePolicyRouting = *nodeCfg.EnablePolicyRouting //nolint:staticcheck // intentional use of deprecated field + } + + if !flags.Changed("mtu") && nodeCfg.MTU != nil { + cfg.MTU = *nodeCfg.MTU + } + + if !flags.Changed("health-port") && nodeCfg.HealthPort != nil { + cfg.HealthPort = *nodeCfg.HealthPort + } + + if !flags.Changed("status-push-enabled") && nodeCfg.StatusPushEnabled != nil { + cfg.StatusPushEnabled = *nodeCfg.StatusPushEnabled + } + + if !flags.Changed("status-push-url") && nodeCfg.StatusPushURL != "" { + cfg.StatusPushURL = nodeCfg.StatusPushURL + } + + if !flags.Changed("status-push-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushInterval, "node.statusPushInterval"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.StatusPushInterval = d + } + } + + if !flags.Changed("status-push-apiserver-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushAPIServerInterval, "node.statusPushApiserverInterval"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.StatusPushAPIServerInterval = d + } + } + + if !flags.Changed("status-push-delta") && nodeCfg.StatusPushDelta != nil { + cfg.StatusPushDelta = *nodeCfg.StatusPushDelta + } + + if !flags.Changed("status-ws-enabled") && nodeCfg.StatusWSEnabled != nil { + cfg.StatusWSEnabled = *nodeCfg.StatusWSEnabled + } + + if !flags.Changed("status-ws-url") && nodeCfg.StatusWSURL != "" { + cfg.StatusWSURL = nodeCfg.StatusWSURL + } + + if !flags.Changed("status-ws-apiserver-mode") && nodeCfg.StatusWSAPIServerMode != "" { + cfg.StatusWSAPIServerMode = nodeCfg.StatusWSAPIServerMode + } + + if !flags.Changed("status-ws-apiserver-url") && nodeCfg.StatusWSAPIServerURL != "" { + cfg.StatusWSAPIServerURL = nodeCfg.StatusWSAPIServerURL + } + + if !flags.Changed("status-ws-apiserver-startup-delay") && nodeCfg.StatusWSAPIServerStartupDelay != "" { + d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSAPIServerStartupDelay, "node.statusWebsocketApiserverStartupDelay") + if parseErr != nil { + return parseErr + } + + if d < 0 { + return fmt.Errorf("node.statusWebsocketApiserverStartupDelay must be >= 0") + } + + cfg.StatusWSAPIServerStartupDelay = d + } + + if !flags.Changed("status-ws-keepalive-interval") && nodeCfg.StatusWSKeepaliveInterval != "" { + d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSKeepaliveInterval, "node.statusWebsocketKeepaliveInterval") + if parseErr != nil { + return parseErr + } + + cfg.StatusWSKeepaliveInterval = d + } + + if !flags.Changed("status-ws-keepalive-failure-count") && nodeCfg.StatusWSKeepaliveFailCount != nil { + cfg.StatusWSKeepaliveFailureCount = *nodeCfg.StatusWSKeepaliveFailCount + } + // New consolidated shutdown cleanup flag. + if !flags.Changed("remove-configuration-on-shutdown") && nodeCfg.RemoveConfigurationOnShutdown != nil { + cfg.RemoveConfigurationOnShutdown = *nodeCfg.RemoveConfigurationOnShutdown + } + // Deprecated individual shutdown cleanup flags (kept for backward compatibility). + if !flags.Changed("shutdown-remove-wireguard-configuration") && nodeCfg.ShutdownRemoveWireGuardConfiguration != nil { + cfg.RemoveWireGuardOnShutdown = *nodeCfg.ShutdownRemoveWireGuardConfiguration + } + + if !flags.Changed("shutdown-cleanup-netlink") && nodeCfg.ShutdownRemoveIPRoutes != nil { + cfg.CleanupNetlinkOnShutdown = *nodeCfg.ShutdownRemoveIPRoutes + } + + if !flags.Changed("shutdown-remove-masquerade-rules") && nodeCfg.ShutdownRemoveMasqueradeRules != nil { + cfg.RemoveMasqueradeOnShutdown = *nodeCfg.ShutdownRemoveMasqueradeRules + } + // If any deprecated flag is true, activate the consolidated flag. + if cfg.RemoveWireGuardOnShutdown || cfg.CleanupNetlinkOnShutdown || cfg.RemoveMasqueradeOnShutdown { + cfg.RemoveConfigurationOnShutdown = true + } + + if _, err := parseStatusWSAPIServerMode(cfg.StatusWSAPIServerMode); err != nil { + return err + } + + if !flags.Changed("status-critical-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.CriticalDeltaEvery, "node.criticalDeltaEvery"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.CriticalDeltaEvery = d + } + } + + if !flags.Changed("status-stats-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatsDeltaEvery, "node.statsDeltaEvery"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.StatsDeltaEvery = d + } + } + + if !flags.Changed("status-full-sync-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.FullSyncEvery, "node.fullSyncEvery"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.FullSyncEvery = d + } + } + + if cfg.StatusWSKeepaliveFailureCount < 1 { + return fmt.Errorf("node.statusWsKeepaliveFailureCount must be >= 1") + } + + // Apply preferred encapsulation from config file if not set via CLI. + if !flags.Changed("preferred-private-encap") && nodeCfg.PreferredPrivateNetworkEncapsulation != "" { + cfg.PreferredPrivateEncap = nodeCfg.PreferredPrivateNetworkEncapsulation + } + + if !flags.Changed("preferred-public-encap") && nodeCfg.PreferredPublicNetworkEncapsulation != "" { + cfg.PreferredPublicEncap = nodeCfg.PreferredPublicNetworkEncapsulation + } + + if !flags.Changed("health-flap-max-backoff") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.HealthFlapMaxBackoff, "node.healthFlapMaxBackoff"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.HealthFlapMaxBackoff = d + } + } + + if !flags.Changed("route-table-id") && nodeCfg.RouteTableID != nil { + cfg.RouteTableID = *nodeCfg.RouteTableID + } + + if !flags.Changed("kube-proxy-health-interval") && nodeCfg.KubeProxyHealthInterval != "" { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.KubeProxyHealthInterval, "node.kubeProxyHealthInterval"); parseErr != nil { + return parseErr + } else { + cfg.KubeProxyHealthInterval = d + } + } + + if !flags.Changed("netlink-resync-period") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.NetlinkResyncPeriod, "node.netlinkResyncPeriod"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.NetlinkResyncPeriod = d + } + } + + if !flags.Changed("tunnel-dataplane-map-size") && nodeCfg.TunnelDataplaneMapSize != nil { + cfg.TunnelDataplaneMapSize = *nodeCfg.TunnelDataplaneMapSize + } + + if !flags.Changed("tunnel-ip-family") && nodeCfg.TunnelIPFamily != "" { + cfg.TunnelIPFamily = nodeCfg.TunnelIPFamily + } + + if !flags.Changed("vxlan-src-port-low") && nodeCfg.VXLANSrcPortLow != nil { + cfg.VXLANSrcPortLow = *nodeCfg.VXLANSrcPortLow + } + + if !flags.Changed("vxlan-src-port-high") && nodeCfg.VXLANSrcPortHigh != nil { + cfg.VXLANSrcPortHigh = *nodeCfg.VXLANSrcPortHigh + } + + if !flags.Changed("geneve-interface") && nodeCfg.GeneveInterfaceName != "" { + cfg.GeneveInterfaceName = nodeCfg.GeneveInterfaceName + } + + if !flags.Changed("vxlan-interface") && nodeCfg.VXLANInterfaceName != "" { + cfg.VXLANInterfaceName = nodeCfg.VXLANInterfaceName + } + + if !flags.Changed("ipip-interface") && nodeCfg.IPIPInterfaceName != "" { + cfg.IPIPInterfaceName = nodeCfg.IPIPInterfaceName + } + + if !flags.Changed("wireguard-interface-prefix") && nodeCfg.WireGuardInterfacePrefix != "" { + cfg.WireGuardInterfacePrefix = nodeCfg.WireGuardInterfacePrefix + } + + // Validate and default tunnelIPFamily + switch cfg.TunnelIPFamily { + case "IPv4", "IPv6": + // valid + case "": + cfg.TunnelIPFamily = "IPv4" + default: + return fmt.Errorf("invalid tunnel-ip-family %q: must be 'IPv4' or 'IPv6'", cfg.TunnelIPFamily) + } + + if err := validateTunnelInterfaceNames(cfg.GeneveInterfaceName, cfg.VXLANInterfaceName, cfg.IPIPInterfaceName); err != nil { + return err + } + + if err := validateWireGuardInterfacePrefix(cfg.WireGuardInterfacePrefix); err != nil { + return err + } + + // Normalize MTU: treat 0 as 1280 (the IPv6 minimum, safe for all links). + if cfg.MTU == 0 { + cfg.MTU = 1280 + } + + // Apply common config. + if !flags.Changed("apiserver-url") && runtimeCfg.Common.ApiserverURL != "" { + cfg.ApiserverURL = runtimeCfg.Common.ApiserverURL + } + + return nil +} + +func run(ctx context.Context, cfg *config) error { + klog.Infof("unbounded-net-node version=%s commit=%s built=%s", version.Version, version.GitCommit, version.BuildTime) + + // Validate node name + if cfg.NodeName == "" { + return fmt.Errorf("node name is required; set NODE_NAME or use --node-name") + } + + klog.Infof("Running on node: %s", cfg.NodeName) + + if cfg.EnablePolicyRouting { + klog.Info("Policy-based routing on gateway interfaces is enabled") + } else { + klog.Info("Policy-based routing on gateway interfaces is disabled") + } + + // Build Kubernetes client + var ( + restConfig *rest.Config + err error + ) + + if cfg.RESTConfig != nil { + restConfig = rest.CopyConfig(cfg.RESTConfig) + } else if cfg.KubeconfigPath != "" { + restConfig, err = clientcmd.BuildConfigFromFlags(cfg.ApiserverURL, cfg.KubeconfigPath) + } else { + restConfig, err = rest.InClusterConfig() + if err == nil && cfg.ApiserverURL != "" { + restConfig.Host = cfg.ApiserverURL + } + } + + if err != nil { + return fmt.Errorf("build kubeconfig: %w", err) + } + + if cfg.ApiserverURL != "" { + klog.Infof("Using API server URL override: %s", cfg.ApiserverURL) + } + + // Wire client-go metrics into Prometheus before creating clients. + metrics.RegisterClientGoMetrics() + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create Kubernetes client: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create dynamic Kubernetes client: %w", err) + } + + // Generate WireGuard keys and annotate node + pubKey, err := ensureWireGuardKeys(cfg) + if err != nil { + return fmt.Errorf("ensure WireGuard keys: %w", err) + } + + klog.Infof("WireGuard public key: %s", pubKey) + + if err := annotateNodeWithPubKey(ctx, clientset, cfg.NodeName, pubKey); err != nil { + return fmt.Errorf("annotate Node with WireGuard public key: %w", err) + } + + klog.Info("Node annotated with WireGuard public key") + + // Detect and annotate the node's maximum tunnel MTU so the controller + // can validate that the configured MTU is compatible across all nodes. + if detectedMTU := unboundednetnetlink.DetectDefaultRouteMTU(); detectedMTU > 0 { + wgMTU := detectedMTU - unboundednetnetlink.WireGuardMTUOverhead + if err := annotateNodeWithMTU(ctx, clientset, cfg.NodeName, wgMTU); err != nil { + klog.Warningf("Failed to annotate node with tunnel MTU: %v", err) + } else { + klog.Infof("Node annotated with tunnel MTU %d (detected default route MTU %d - %d overhead)", wgMTU, detectedMTU, unboundednetnetlink.WireGuardMTUOverhead) + } + } + + // Watch the config file for dynamic log level changes. + go configpkg.WatchConfigLogLevel(ctx, cfg.ConfigFile) + + // Warn if public network traffic will be sent unencrypted. + if cfg.PreferredPublicEncap != "" && cfg.PreferredPublicEncap != "WireGuard" { + klog.Warningf("WARNING: preferredPublicNetworkEncapsulation is set to %q -- traffic over public networks will be sent UNENCRYPTED", cfg.PreferredPublicEncap) + } + + // Create informers early - before any CRD-based lookups + // This allows us to use the informer cache for all CRD operations + informerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, cfg.InformerResyncPeriod) + sliceInformer := informerFactory.ForResource(siteNodeSliceGVR).Informer() + siteInformer := informerFactory.ForResource(siteGVR).Informer() + gatewayPoolInformer := informerFactory.ForResource(gatewayPoolGVR).Informer() + gatewayNodeInformer := informerFactory.ForResource(gatewayNodeGVR).Informer() + sitePeeringInformer := informerFactory.ForResource(sitePeeringGVR).Informer() + assignmentInformer := informerFactory.ForResource(siteGatewayPoolAssignmentGVR).Informer() + poolPeeringInformer := informerFactory.ForResource(gatewayPoolPeeringGVR).Informer() + + // Start the informers + informerFactory.Start(ctx.Done()) + + // Wait for caches to sync + klog.Info("Waiting for informer caches to sync") + + if !cache.WaitForCacheSync(ctx.Done(), sliceInformer.HasSynced, siteInformer.HasSynced, gatewayPoolInformer.HasSynced, gatewayNodeInformer.HasSynced, sitePeeringInformer.HasSynced, assignmentInformer.HasSynced, poolPeeringInformer.HasSynced) { + return fmt.Errorf("failed to sync informer caches") + } + + klog.Info("Informer caches synced") + + // Start the netlink cache (read-only network state snapshot). + netlinkCache := unboundednetnetlink.NewNetlinkCache(cfg.NetlinkResyncPeriod) + if err := netlinkCache.Start(ctx); err != nil { + return fmt.Errorf("start netlink cache: %w", err) + } + + // Track if CNI is configured for health checks + cniConfigured := false + + // Create shared health state for health server + healthState := &nodeHealthState{ + cniConfigured: &cniConfigured, + informersSynced: []cache.InformerSynced{ + sliceInformer.HasSynced, + siteInformer.HasSynced, + gatewayPoolInformer.HasSynced, + gatewayNodeInformer.HasSynced, + sitePeeringInformer.HasSynced, + assignmentInformer.HasSynced, + poolPeeringInformer.HasSynced, + }, + } + + // Start health server if enabled (readiness should not wait on site membership) + if cfg.HealthPort > 0 { + go startHealthServer(cfg.HealthPort, healthState) + } + + // Check if this node is a gateway node by checking the informer cache + isGatewayNode := isGatewayNodeFromCRDs(gatewayPoolInformer, pubKey) + if isGatewayNode { + klog.Info("Node is a gateway node (found in GatewayPool status)") + } + + // Wait for this node to appear in a SiteNodeSlice or GatewayPool + // This ensures the site controller has processed this node before we continue + mySiteName, err := waitForSiteMembership(ctx, sliceInformer, gatewayPoolInformer, pubKey) + if err != nil { + if err == context.Canceled { + return nil + } + + return err + } + + // Check if this node's site has manageCniPlugin enabled using the informer cache + manageCniPlugin := manageCNIForMembership(siteInformer, mySiteName, isGatewayNode) + + var nodePodCIDRs []string + if manageCniPlugin { + // Wait for podCIDRs and configure CNI + nodePodCIDRs, err = waitForPodCIDRsAndConfigure(ctx, clientset, cfg, &cniConfigured) + if err != nil { + if err == context.Canceled { + return nil + } + + return err + } + } else { + klog.Info("manageCniPlugin is false for this site - skipping CNI configuration") + // Still need to get the node's podCIDRs for WireGuard gateway IP calculation + node, err := clientset.CoreV1().Nodes().Get(ctx, cfg.NodeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get Node %s: %w", cfg.NodeName, err) + } + + nodePodCIDRs = node.Spec.PodCIDRs + // Mark CNI as "configured" since we're intentionally not managing it + cniConfigured = true + } + + // After CNI is configured (or skipped), watch Site CRD for WireGuard peers + // Pass the node's podCIDRs so routes can be configured with preferred source IPs + // Also pass the informers so they can be reused (already synced) + return watchSiteAndConfigureWireGuard(ctx, clientset, dynamicClient, cfg, pubKey, nodePodCIDRs, manageCniPlugin, healthState, netlinkCache, siteInformer, sliceInformer, gatewayPoolInformer, gatewayNodeInformer, sitePeeringInformer, assignmentInformer, poolPeeringInformer) +} diff --git a/cmd/unbounded-net-node/node_config_validation.go b/internal/net/nodeagent/node_config_validation.go similarity index 99% rename from cmd/unbounded-net-node/node_config_validation.go rename to internal/net/nodeagent/node_config_validation.go index eda654ded..0384ab7c8 100644 --- a/cmd/unbounded-net-node/node_config_validation.go +++ b/internal/net/nodeagent/node_config_validation.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "fmt" diff --git a/cmd/unbounded-net-node/node_config_validation_test.go b/internal/net/nodeagent/node_config_validation_test.go similarity index 99% rename from cmd/unbounded-net-node/node_config_validation_test.go rename to internal/net/nodeagent/node_config_validation_test.go index 061411d7e..298a7768d 100644 --- a/cmd/unbounded-net-node/node_config_validation_test.go +++ b/internal/net/nodeagent/node_config_validation_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "errors" diff --git a/cmd/unbounded-net-node/node_types.go b/internal/net/nodeagent/node_types.go similarity index 99% rename from cmd/unbounded-net-node/node_types.go rename to internal/net/nodeagent/node_types.go index d0d372713..f2bf9817a 100644 --- a/cmd/unbounded-net-node/node_types.go +++ b/internal/net/nodeagent/node_types.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "sync" diff --git a/cmd/unbounded-net-node/peer_healthcheck.go b/internal/net/nodeagent/peer_healthcheck.go similarity index 99% rename from cmd/unbounded-net-node/peer_healthcheck.go rename to internal/net/nodeagent/peer_healthcheck.go index 00bd26e9a..c3d86170b 100644 --- a/cmd/unbounded-net-node/peer_healthcheck.go +++ b/internal/net/nodeagent/peer_healthcheck.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "net" diff --git a/cmd/unbounded-net-node/reconciliation_helpers.go b/internal/net/nodeagent/reconciliation_helpers.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers.go rename to internal/net/nodeagent/reconciliation_helpers.go index f1ba45ef0..cab7a9c0f 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers.go +++ b/internal/net/nodeagent/reconciliation_helpers.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/reconciliation_helpers_bfd_test.go b/internal/net/nodeagent/reconciliation_helpers_bfd_test.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers_bfd_test.go rename to internal/net/nodeagent/reconciliation_helpers_bfd_test.go index 1e16ca18d..324daf465 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers_bfd_test.go +++ b/internal/net/nodeagent/reconciliation_helpers_bfd_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import "testing" diff --git a/cmd/unbounded-net-node/reconciliation_helpers_more_test.go b/internal/net/nodeagent/reconciliation_helpers_more_test.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers_more_test.go rename to internal/net/nodeagent/reconciliation_helpers_more_test.go index 46f722f24..d33a2e74c 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers_more_test.go +++ b/internal/net/nodeagent/reconciliation_helpers_more_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/reconciliation_helpers_test.go b/internal/net/nodeagent/reconciliation_helpers_test.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers_test.go rename to internal/net/nodeagent/reconciliation_helpers_test.go index 725a85849..651f2c518 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers_test.go +++ b/internal/net/nodeagent/reconciliation_helpers_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/route_annotations.go b/internal/net/nodeagent/route_annotations.go similarity index 99% rename from cmd/unbounded-net-node/route_annotations.go rename to internal/net/nodeagent/route_annotations.go index b47e9ffcf..eae63c617 100644 --- a/cmd/unbounded-net-node/route_annotations.go +++ b/internal/net/nodeagent/route_annotations.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "sort" diff --git a/cmd/unbounded-net-node/runtime_utils.go b/internal/net/nodeagent/runtime_utils.go similarity index 99% rename from cmd/unbounded-net-node/runtime_utils.go rename to internal/net/nodeagent/runtime_utils.go index f65692226..6e2b06503 100644 --- a/cmd/unbounded-net-node/runtime_utils.go +++ b/internal/net/nodeagent/runtime_utils.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "net" diff --git a/cmd/unbounded-net-node/runtime_utils_test.go b/internal/net/nodeagent/runtime_utils_test.go similarity index 99% rename from cmd/unbounded-net-node/runtime_utils_test.go rename to internal/net/nodeagent/runtime_utils_test.go index f1c5c4752..6f25fbc48 100644 --- a/cmd/unbounded-net-node/runtime_utils_test.go +++ b/internal/net/nodeagent/runtime_utils_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "slices" diff --git a/cmd/unbounded-net-node/site_watch_reconcile.go b/internal/net/nodeagent/site_watch_reconcile.go similarity index 99% rename from cmd/unbounded-net-node/site_watch_reconcile.go rename to internal/net/nodeagent/site_watch_reconcile.go index 22be3810b..ab5f42fe5 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile.go +++ b/internal/net/nodeagent/site_watch_reconcile.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/site_watch_reconcile_test.go b/internal/net/nodeagent/site_watch_reconcile_test.go similarity index 99% rename from cmd/unbounded-net-node/site_watch_reconcile_test.go rename to internal/net/nodeagent/site_watch_reconcile_test.go index 4eab11fa5..db6fc00b0 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile_test.go +++ b/internal/net/nodeagent/site_watch_reconcile_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/status_proto.go b/internal/net/nodeagent/status_proto.go similarity index 99% rename from cmd/unbounded-net-node/status_proto.go rename to internal/net/nodeagent/status_proto.go index 61dc259c0..9ba72b313 100644 --- a/cmd/unbounded-net-node/status_proto.go +++ b/internal/net/nodeagent/status_proto.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/status_proto_test.go b/internal/net/nodeagent/status_proto_test.go similarity index 99% rename from cmd/unbounded-net-node/status_proto_test.go rename to internal/net/nodeagent/status_proto_test.go index 4b4965f5e..01f256c5a 100644 --- a/cmd/unbounded-net-node/status_proto_test.go +++ b/internal/net/nodeagent/status_proto_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/status_server.go b/internal/net/nodeagent/status_server.go similarity index 99% rename from cmd/unbounded-net-node/status_server.go rename to internal/net/nodeagent/status_server.go index 10da70cf9..d75fc437c 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/internal/net/nodeagent/status_server.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "bytes" diff --git a/cmd/unbounded-net-node/status_server_http_test.go b/internal/net/nodeagent/status_server_http_test.go similarity index 99% rename from cmd/unbounded-net-node/status_server_http_test.go rename to internal/net/nodeagent/status_server_http_test.go index a19c87343..bcb211c67 100644 --- a/cmd/unbounded-net-node/status_server_http_test.go +++ b/internal/net/nodeagent/status_server_http_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/status_server_test.go b/internal/net/nodeagent/status_server_test.go similarity index 99% rename from cmd/unbounded-net-node/status_server_test.go rename to internal/net/nodeagent/status_server_test.go index 6bf2810c3..2f7d4ea2d 100644 --- a/cmd/unbounded-net-node/status_server_test.go +++ b/internal/net/nodeagent/status_server_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/tunnel_config.go b/internal/net/nodeagent/tunnel_config.go similarity index 99% rename from cmd/unbounded-net-node/tunnel_config.go rename to internal/net/nodeagent/tunnel_config.go index 45c5692a7..b624ac8f1 100644 --- a/cmd/unbounded-net-node/tunnel_config.go +++ b/internal/net/nodeagent/tunnel_config.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/tunnel_interface_validation.go b/internal/net/nodeagent/tunnel_interface_validation.go similarity index 99% rename from cmd/unbounded-net-node/tunnel_interface_validation.go rename to internal/net/nodeagent/tunnel_interface_validation.go index 78396783d..fe76897c1 100644 --- a/cmd/unbounded-net-node/tunnel_interface_validation.go +++ b/internal/net/nodeagent/tunnel_interface_validation.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "fmt" diff --git a/cmd/unbounded-net-node/tunnel_interface_validation_test.go b/internal/net/nodeagent/tunnel_interface_validation_test.go similarity index 99% rename from cmd/unbounded-net-node/tunnel_interface_validation_test.go rename to internal/net/nodeagent/tunnel_interface_validation_test.go index ed37a1ed0..7f6248e1a 100644 --- a/cmd/unbounded-net-node/tunnel_interface_validation_test.go +++ b/internal/net/nodeagent/tunnel_interface_validation_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "strings" diff --git a/cmd/unbounded-net-node/wireguard_config.go b/internal/net/nodeagent/wireguard_config.go similarity index 99% rename from cmd/unbounded-net-node/wireguard_config.go rename to internal/net/nodeagent/wireguard_config.go index fb14e9d5a..d63bc5326 100644 --- a/cmd/unbounded-net-node/wireguard_config.go +++ b/internal/net/nodeagent/wireguard_config.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/wireguard_config_test.go b/internal/net/nodeagent/wireguard_config_test.go similarity index 80% rename from cmd/unbounded-net-node/wireguard_config_test.go rename to internal/net/nodeagent/wireguard_config_test.go index 0fc827aec..1e1b08475 100644 --- a/cmd/unbounded-net-node/wireguard_config_test.go +++ b/internal/net/nodeagent/wireguard_config_test.go @@ -1,4 +1,4 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent From 0e61ea7bddbcb702f3b1e9045aceb9f215222ee4 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:35:26 +0000 Subject: [PATCH 38/43] kubectl: run opt-in bootstrap gateway --- .../app/site_bootstrap_netboot.go | 300 ++++++++++++++++-- .../app/site_bootstrap_netboot_test.go | 118 ++++++- 2 files changed, 384 insertions(+), 34 deletions(-) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index 023707ebb..5d9a748aa 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -34,6 +34,7 @@ import ( v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" "github.com/Azure/unbounded/internal/kube" + "github.com/Azure/unbounded/internal/net/nodeagent" "github.com/Azure/unbounded/internal/unbounded" ) @@ -43,32 +44,36 @@ const ( ) type siteBootstrapNetbootHandler struct { - site string - machine string - interfaceName string - address string - endpointName string - httpPort int - kubeconfigPath string - namespace string - metalmanBinary string - timeout time.Duration - routedCIDRs []string - resources client.Client - kubeClient kubernetes.Interface - restConfig *rest.Config - pollInterval time.Duration - dependencies bootstrapNetbootDependencies + site string + machine string + interfaceName string + address string + gatewayExternalAddress string + endpointName string + httpPort int + kubeconfigPath string + namespace string + metalmanBinary string + timeout time.Duration + routedCIDRs []string + resources client.Client + kubeClient kubernetes.Interface + restConfig *rest.Config + pollInterval time.Duration + dependencies bootstrapNetbootDependencies } type bootstrapNetbootDependencies struct { - resolveBinary func() (string, error) - preflightNetwork func() error - localPort func() (int, error) - portForward func(context.Context, int) (*bootstrapPortForward, error) - edgeToken func(context.Context) (*bootstrapEdgeToken, error) - startEdge func(string, []string) (bootstrapEdgeProcess, error) - dialEdge func(context.Context, string) error + resolveBinary func() (string, error) + preflightNetwork func() error + localPort func() (int, error) + portForward func(context.Context, int) (*bootstrapPortForward, error) + edgeToken func(context.Context) (*bootstrapEdgeToken, error) + startEdge func(string, []string) (bootstrapEdgeProcess, error) + dialEdge func(context.Context, string) error + preflightGateway func() error + gatewayRuntimeDir func() (string, error) + startGateway func(context.Context, nodeagent.ExternalGatewayOptions) (bootstrapEdgeProcess, error) } type bootstrapNetbootState struct { @@ -114,6 +119,14 @@ type commandBootstrapEdgeProcess struct { err error } +type embeddedBootstrapGatewayProcess struct { + cancel context.CancelFunc + done chan struct{} + once sync.Once + mu sync.Mutex + err error +} + func (p *commandBootstrapEdgeProcess) Done() <-chan struct{} { return p.done } @@ -152,6 +165,25 @@ func (p *commandBootstrapEdgeProcess) Stop(ctx context.Context) error { } } +func (p *embeddedBootstrapGatewayProcess) Done() <-chan struct{} { return p.done } + +func (p *embeddedBootstrapGatewayProcess) Err() error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.err +} + +func (p *embeddedBootstrapGatewayProcess) Stop(ctx context.Context) error { + p.once.Do(p.cancel) + select { + case <-p.done: + return p.Err() + case <-ctx.Done(): + return ctx.Err() + } +} + func (t *bootstrapEdgeToken) Path() string { return t.path } @@ -202,6 +234,7 @@ func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Co cmd.Flags().StringVar(&handler.kubeconfigPath, "kubeconfig", "", "Path to kubeconfig file") cmd.Flags().StringVar(&handler.namespace, "namespace", unbounded.SystemNamespace(), "Namespace containing Metalman workloads") cmd.Flags().StringVar(&handler.metalmanBinary, "metalman-binary", "", "Path to the metalman binary") + cmd.Flags().StringVar(&handler.gatewayExternalAddress, "gateway-external-address", "", "WireGuard address reachable by remote gateway peers (defaults to --address)") cmd.Flags().DurationVar(&handler.timeout, "timeout", defaultBootstrapNetbootTimeout, "Maximum time to wait for the designated Node to become Ready") cmd.Flags().StringSliceVar(&handler.routedCIDRs, "routed-cidr", nil, "CIDR routed through an ephemeral external gateway (repeatable)") @@ -219,6 +252,9 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if h.endpointName == "" { h.endpointName = "bootstrap-" + h.machine } + if h.gatewayExternalAddress == "" { + h.gatewayExternalAddress = h.address + } if err := h.initializeDependencies(); err != nil { return err @@ -230,6 +266,17 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if err := h.dependencies.preflightNetwork(); err != nil { return err } + var gatewayRuntimeDir string + if len(h.routedCIDRs) > 0 { + if err := h.dependencies.preflightGateway(); err != nil { + return err + } + gatewayRuntimeDir, err = h.dependencies.gatewayRuntimeDir() + if err != nil { + return fmt.Errorf("create external gateway runtime directory: %w", err) + } + defer func() { retErr = errors.Join(retErr, os.RemoveAll(gatewayRuntimeDir)) }() + } if err := h.initializeClients(); err != nil { return err } @@ -244,9 +291,11 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error } resourcesPrepared := true var ( - forward *bootstrapPortForward - token *bootstrapEdgeToken - process bootstrapEdgeProcess + forward *bootstrapPortForward + token *bootstrapEdgeToken + process bootstrapEdgeProcess + gatewayProcess bootstrapEdgeProcess + gatewayPrepared bool ) defer func() { cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) @@ -258,14 +307,36 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if process != nil { retErr = errors.Join(retErr, process.Stop(cleanupCtx)) } + if gatewayProcess != nil { + retErr = errors.Join(retErr, gatewayProcess.Stop(cleanupCtx)) + } if forward != nil { retErr = errors.Join(retErr, forward.Close()) } if token != nil { retErr = errors.Join(retErr, token.Close()) } + if gatewayPrepared { + retErr = errors.Join(retErr, h.cleanupGatewayResources(cleanupCtx)) + } }() + if len(h.routedCIDRs) > 0 { + if err := h.prepareGatewayResources(runCtx); err != nil { + return err + } + gatewayPrepared = true + gatewayProcess, err = h.dependencies.startGateway(runCtx, nodeagent.ExternalGatewayOptions{ + NodeName: h.endpointName, RuntimeDir: gatewayRuntimeDir, RESTConfig: h.restConfig, + }) + if err != nil { + return fmt.Errorf("start external gateway dataplane: %w", err) + } + if err := h.waitForGatewayReady(runCtx, gatewayProcess); err != nil { + return err + } + } + if err := h.waitForMetalman(runCtx); err != nil { return err } @@ -297,7 +368,7 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error return fmt.Errorf("get designated Machine %s: %w", h.machine, err) } - return h.waitForNodeReadyAndEdge(runCtx, designatedNodeName(&machine), process) + return h.waitForNodeReadyAndProcesses(runCtx, designatedNodeName(&machine), process, gatewayProcess) } func (h *siteBootstrapNetbootHandler) initializeDependencies() error { @@ -320,6 +391,17 @@ func (h *siteBootstrapNetbootHandler) initializeDependencies() error { if h.dependencies.dialEdge == nil { h.dependencies.dialEdge = dialBootstrapEdge } + if h.dependencies.preflightGateway == nil { + h.dependencies.preflightGateway = h.validateExternalGateway + } + if h.dependencies.gatewayRuntimeDir == nil { + h.dependencies.gatewayRuntimeDir = func() (string, error) { + return os.MkdirTemp("/run", "unbounded-netboot-") + } + } + if h.dependencies.startGateway == nil { + h.dependencies.startGateway = startEmbeddedBootstrapGateway + } return nil } @@ -390,6 +472,30 @@ func (h *siteBootstrapNetbootHandler) validateProvisioningInterface() error { return fmt.Errorf("provisioning interface %s does not own address %s", h.interfaceName, h.address) } +func (h *siteBootstrapNetbootHandler) validateExternalGateway() error { + if err := h.validateRoutedCIDRs(); err != nil { + return err + } + if os.Geteuid() != 0 { + return fmt.Errorf("external gateway dataplane requires root privileges") + } + if ip := net.ParseIP(h.gatewayExternalAddress); ip == nil || ip.To4() == nil { + return fmt.Errorf("gateway external address %q must be an IPv4 address", h.gatewayExternalAddress) + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) validateRoutedCIDRs() error { + for _, routedCIDR := range h.routedCIDRs { + if _, _, err := net.ParseCIDR(routedCIDR); err != nil { + return fmt.Errorf("invalid routed CIDR %q: %w", routedCIDR, err) + } + } + + return nil +} + func availableLoopbackPort() (int, error) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -494,10 +600,8 @@ func (h *siteBootstrapNetbootHandler) restoreClusterResources(ctx context.Contex } func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Context) (retErr error) { - for _, routedCIDR := range h.routedCIDRs { - if _, _, err := net.ParseCIDR(routedCIDR); err != nil { - return fmt.Errorf("invalid routed CIDR %q: %w", routedCIDR, err) - } + if err := h.validateRoutedCIDRs(); err != nil { + return err } protocol := netv1alpha1.TunnelProtocolWireGuard @@ -555,7 +659,7 @@ func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Contex } createdNode.Status.Addresses = []corev1.NodeAddress{ {Type: corev1.NodeInternalIP, Address: h.address}, - {Type: corev1.NodeExternalIP, Address: h.address}, + {Type: corev1.NodeExternalIP, Address: h.gatewayExternalAddress}, } if _, err := h.kubeClient.CoreV1().Nodes().UpdateStatus(ctx, createdNode, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("set bootstrap gateway Node %s addresses: %w", h.endpointName, err) @@ -726,6 +830,23 @@ func startBootstrapEdgeProcess(binary string, args []string, stdout, stderr io.W return process, nil } +func startEmbeddedBootstrapGateway( + ctx context.Context, + options nodeagent.ExternalGatewayOptions, +) (bootstrapEdgeProcess, error) { + runCtx, cancel := context.WithCancel(ctx) + process := &embeddedBootstrapGatewayProcess{cancel: cancel, done: make(chan struct{})} + go func() { + err := nodeagent.RunExternalGateway(runCtx, options) + process.mu.Lock() + process.err = err + process.mu.Unlock() + close(process.done) + }() + + return process, nil +} + func isSignalExit(err error) bool { var exitError *exec.ExitError if !errors.As(err, &exitError) || exitError.ProcessState == nil { @@ -815,6 +936,121 @@ func (h *siteBootstrapNetbootHandler) waitForNodeReadyAndEdge( } } +func (h *siteBootstrapNetbootHandler) waitForNodeReadyAndProcesses( + ctx context.Context, + nodeName string, + edge bootstrapEdgeProcess, + gateway bootstrapEdgeProcess, +) error { + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + + ready := make(chan error, 1) + go func() { ready <- h.waitForNodeReady(waitCtx, nodeName) }() + + var gatewayDone <-chan struct{} + if gateway != nil { + gatewayDone = gateway.Done() + } + select { + case err := <-ready: + return err + case <-edge.Done(): + return processExitError("Metalman edge", nodeName, edge.Err()) + case <-gatewayDone: + return processExitError("external gateway", nodeName, gateway.Err()) + case <-ctx.Done(): + return fmt.Errorf("wait for designated Node %s while bootstrap dataplanes are running: %w", nodeName, ctx.Err()) + } +} + +func processExitError(name, nodeName string, err error) error { + if err != nil { + return fmt.Errorf("%s exited before designated Node %s became Ready: %w", name, nodeName, err) + } + + return fmt.Errorf("%s exited before designated Node %s became Ready", name, nodeName) +} + +func (h *siteBootstrapNetbootHandler) waitForGatewayReady(ctx context.Context, process bootstrapEdgeProcess) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + ready, err := h.gatewayReady(ctx) + if err != nil { + return err + } + if ready { + return nil + } + select { + case <-process.Done(): + if err := process.Err(); err != nil { + return fmt.Errorf("external gateway exited before becoming ready: %w", err) + } + + return fmt.Errorf("external gateway exited before becoming ready") + case <-ctx.Done(): + return fmt.Errorf("wait for external gateway readiness: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func (h *siteBootstrapNetbootHandler) gatewayReady(ctx context.Context) (bool, error) { + node, err := h.kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("get bootstrap gateway Node %s: %w", h.endpointName, err) + } + if node.Annotations["net.unbounded-cloud.io/wg-pubkey"] == "" { + return false, nil + } + + var pool netv1alpha1.GatewayPool + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool); err != nil { + return false, fmt.Errorf("get bootstrap GatewayPool %s: %w", h.endpointName, err) + } + if pool.Status.NodeCount != 1 || !containsString(pool.Status.ConnectedSites, h.site) { + return false, nil + } + matchedNode := false + for _, poolNode := range pool.Status.Nodes { + if poolNode.Name == h.endpointName && poolNode.WireGuardPublicKey != "" { + matchedNode = true + break + } + } + if !matchedNode { + return false, nil + } + + var gatewayNode netv1alpha1.GatewayPoolNode + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &gatewayNode); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + + return false, fmt.Errorf("get bootstrap GatewayPoolNode %s: %w", h.endpointName, err) + } + + return !gatewayNode.Status.LastUpdated.IsZero() && len(gatewayNode.Status.Routes) > 0, nil +} + +func containsString(values []string, wanted string) bool { + for _, value := range values { + if value == wanted { + return true + } + } + + return false +} + func nodeReady(node *corev1.Node) bool { for _, condition := range node.Status.Conditions { if condition.Type == corev1.NodeReady { diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index ee18930f3..164402ff0 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -29,6 +29,7 @@ import ( v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/net/nodeagent" ) func TestSiteBootstrapNetbootCommandContract(t *testing.T) { @@ -47,6 +48,7 @@ func TestSiteBootstrapNetbootCommandContract(t *testing.T) { "kubeconfig", "namespace", "metalman-binary", + "gateway-external-address", "timeout", "routed-cidr", } { @@ -569,7 +571,8 @@ func TestBootstrapNetbootPreparesAndCleansExternalGatewayResources(t *testing.T) kubeClient := fake.NewSimpleClientset() h := &siteBootstrapNetbootHandler{ site: "rack-a", endpointName: "bootstrap-first-node", address: "192.0.2.10", - routedCIDRs: []string{"10.40.0.0/16", "10.50.0.0/24"}, resources: resources, kubeClient: kubeClient, + gatewayExternalAddress: "198.51.100.10", + routedCIDRs: []string{"10.40.0.0/16", "10.50.0.0/24"}, resources: resources, kubeClient: kubeClient, } require.NoError(t, h.prepareGatewayResources(ctx)) @@ -597,7 +600,7 @@ func TestBootstrapNetbootPreparesAndCleansExternalGatewayResources(t *testing.T) }) require.ElementsMatch(t, []corev1.NodeAddress{ {Type: corev1.NodeInternalIP, Address: h.address}, - {Type: corev1.NodeExternalIP, Address: h.address}, + {Type: corev1.NodeExternalIP, Address: h.gatewayExternalAddress}, }, node.Status.Addresses) require.NoError(t, h.cleanupGatewayResources(ctx)) @@ -607,6 +610,88 @@ func TestBootstrapNetbootPreparesAndCleansExternalGatewayResources(t *testing.T) require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) } +func TestBootstrapNetbootExecuteRunsOptInGatewayUntilNodeReady(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, true) + edgeProcess := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + gatewayProcess := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + forward := testBootstrapPortForward(ctx) + token := testBootstrapEdgeToken(t, ctx) + runtimeDir := t.TempDir() + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + gatewayExternalAddress: "198.51.100.10", routedCIDRs: []string{"10.40.0.0/16"}, + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, pollInterval: time.Millisecond, + } + h.dependencies = bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + preflightGateway: func() error { return nil }, + localPort: func() (int, error) { return 32123, nil }, + portForward: func(context.Context, int) (*bootstrapPortForward, error) { return forward, nil }, + edgeToken: func(context.Context) (*bootstrapEdgeToken, error) { return token, nil }, + startEdge: func(string, []string) (bootstrapEdgeProcess, error) { return edgeProcess, nil }, + dialEdge: func(context.Context, string) error { return nil }, + gatewayRuntimeDir: func() (string, error) { + return runtimeDir, nil + }, + startGateway: func(_ context.Context, options nodeagent.ExternalGatewayOptions) (bootstrapEdgeProcess, error) { + require.Equal(t, h.endpointName, options.NodeName) + require.Equal(t, runtimeDir, options.RuntimeDir) + require.Same(t, h.restConfig, options.RESTConfig) + markBootstrapGatewayReady(t, ctx, h, resources, kubeClient) + + return gatewayProcess, nil + }, + } + + require.NoError(t, h.execute(ctx)) + require.True(t, gatewayProcess.stopped) + require.True(t, edgeProcess.stopped) + var pool netv1alpha1.GatewayPool + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) + _, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.Error(t, err) +} + +func markBootstrapGatewayReady( + t *testing.T, + ctx context.Context, + h *siteBootstrapNetbootHandler, + resources client.Client, + kubeClient *fake.Clientset, +) { + t.Helper() + + node, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.NoError(t, err) + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + node.Annotations["net.unbounded-cloud.io/wg-pubkey"] = "public-key" + _, err = kubeClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + require.NoError(t, err) + + var pool netv1alpha1.GatewayPool + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) + pool.Status.NodeCount = 1 + pool.Status.Nodes = []netv1alpha1.GatewayNodeInfo{{Name: h.endpointName, WireGuardPublicKey: "public-key"}} + pool.Status.ConnectedSites = []string{h.site} + require.NoError(t, resources.Update(ctx, &pool)) + + require.NoError(t, resources.Create(ctx, &netv1alpha1.GatewayPoolNode{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: netv1alpha1.GatewayNodeSpec{NodeName: h.endpointName, GatewayPool: h.endpointName}, + Status: netv1alpha1.GatewayNodeStatus{ + LastUpdated: metav1.Now(), + Routes: map[string]netv1alpha1.GatewayNodeRoute{ + "10.40.0.0/16": {Type: "RoutedCidr"}, + }, + }, + })) +} + func TestBootstrapNetbootRejectsInvalidRoutedCIDRBeforeCreatingGateway(t *testing.T) { resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).Build() h := &siteBootstrapNetbootHandler{ @@ -621,6 +706,35 @@ func TestBootstrapNetbootRejectsInvalidRoutedCIDRBeforeCreatingGateway(t *testin require.Empty(t, pools.Items) } +func TestBootstrapNetbootExecuteRejectsInvalidRoutedCIDRBeforeClusterMutation(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, false) + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + routedCIDRs: []string{"not-a-cidr"}, httpPort: 8880, namespace: "unbounded-system", + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, + } + h.dependencies = bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + gatewayRuntimeDir: func() (string, error) { + t.Fatal("gateway runtime directory must not be created after invalid CIDR preflight") + + return "", nil + }, + } + + err := h.execute(ctx) + require.ErrorContains(t, err, "invalid routed CIDR") + + var actualMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &actualMachine)) + require.Equal(t, "permanent-edge", actualMachine.Spec.Netboot().EndpointRef) + var endpoints v1alpha3.NetbootEndpointList + require.NoError(t, resources.List(ctx, &endpoints)) + require.Empty(t, endpoints.Items) +} + func bootstrapLifecycleFixture( t *testing.T, nodeReady bool, From fd33cc9ec0d03c6ca826dd2f82d609812318b420 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:47:28 +0000 Subject: [PATCH 39/43] metalman: grant server cluster metadata access --- deploy/machina/06-metalman-rbac.yaml.tmpl | 3 +++ internal/operator/components/metalman/metalman_test.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/deploy/machina/06-metalman-rbac.yaml.tmpl b/deploy/machina/06-metalman-rbac.yaml.tmpl index 2e5c8c527..8b2df30da 100644 --- a/deploy/machina/06-metalman-rbac.yaml.tmpl +++ b/deploy/machina/06-metalman-rbac.yaml.tmpl @@ -144,6 +144,9 @@ subjects: - kind: ServiceAccount name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} roleRef: kind: Role name: metalman-controller diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index 0f124a7fb..a0bbf6ce0 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -114,6 +114,12 @@ func TestRBACSeparatesControllerAndServerIdentities(t *testing.T) { if strings.Contains(controllerRole, "resources: [\"tokenreviews\"]") { t.Fatalf("metalman-controller retains server TokenReview permission:\n%s", controllerRole) } + + kubeSystemBinding := manifest[strings.Index(manifest, "kind: RoleBinding\nmetadata:\n name: metalman-controller\n namespace: kube-system"):] + kubeSystemBinding = kubeSystemBinding[:strings.Index(kubeSystemBinding, "\n---")] + if !strings.Contains(kubeSystemBinding, "name: metalman-server") { + t.Fatalf("kube-system metadata binding excludes metalman-server:\n%s", kubeSystemBinding) + } } func TestOperatorRBACCanReconcileNetbootEndpoints(t *testing.T) { From aca1a70faea610b164cfaafa602cbf18a080406e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 19:59:12 +0000 Subject: [PATCH 40/43] test: migrate metalman smoke to split runtime --- hack/metalman-redfish-fixture.py | 16 +- hack/smoke-metalman-http.py | 86 ++++----- hack/smoke-metalman.py | 261 ++++++++++++++++++++++----- hack/smoke_metalman_contract_test.go | 66 +++++++ 4 files changed, 328 insertions(+), 101 deletions(-) create mode 100644 hack/smoke_metalman_contract_test.go diff --git a/hack/metalman-redfish-fixture.py b/hack/metalman-redfish-fixture.py index 35dc6f670..94c661e94 100644 --- a/hack/metalman-redfish-fixture.py +++ b/hack/metalman-redfish-fixture.py @@ -45,7 +45,6 @@ def __init__(self, args: argparse.Namespace) -> None: self.efi_source = Path(args.efi_source) if args.efi_source else None self.efi_active = Path(args.efi_active) if args.efi_active else None self.bridge = args.bridge - self.cache_dir = Path(args.cache_dir) if args.cache_dir else None self.manage_boot_order = args.manage_boot_order self.username = args.username self.password = args.password @@ -113,7 +112,7 @@ def power_state(self) -> str: return "On" if result.returncode == 0 and "running" in result.stdout else "Off" def set_efi_boundary(self, enabled: bool) -> None: - if not all((self.efi_source, self.efi_active, self.bridge, self.cache_dir)): + if not all((self.efi_source, self.efi_active, self.bridge)): raise ValueError("UefiHttp requested without HTTP boundary arguments") assert self.efi_source is not None @@ -127,14 +126,8 @@ def set_efi_boundary(self, enabled: bool) -> None: def stage(client_ip: str) -> None: with subprocess.Popen(["mktemp", "-d"], stdout=subprocess.PIPE, text=True) as proc: artifact_dir = Path(proc.communicate()[0].strip()) - entrypoint = boot_url.rsplit("/", 1)[-1] - candidates = list(self.cache_dir.glob(f"oci/*/amd64/disk/{entrypoint}")) - if len(candidates) != 1: - raise ValueError( - f"expected one cached HTTP entrypoint {entrypoint}, found {len(candidates)}" - ) - shutil.copyfile(candidates[0], artifact_dir / "http-entrypoint.efi") for path, url in ( + ("http-entrypoint.efi", boot_url), ("grubx64.efi", f"{base_url}/grubx64.efi"), ("vmlinuz", f"{base_url}/vmlinuz"), ("initrd", f"{base_url}/initrd"), @@ -357,15 +350,14 @@ def main() -> None: parser.add_argument("--efi-source") parser.add_argument("--efi-active") parser.add_argument("--bridge") - parser.add_argument("--cache-dir") parser.add_argument("--username", default="") parser.add_argument("--password", default="") parser.add_argument("--manage-boot-order", action="store_true") args = parser.parse_args() - boundary_values = (args.efi_source, args.efi_active, args.bridge, args.cache_dir) + boundary_values = (args.efi_source, args.efi_active, args.bridge) if any(boundary_values) and not all(boundary_values): - parser.error("--efi-source, --efi-active, --bridge, and --cache-dir must be used together") + parser.error("--efi-source, --efi-active, and --bridge must be used together") state = State(args) state.record.parent.mkdir(parents=True, exist_ok=True) diff --git a/hack/smoke-metalman-http.py b/hack/smoke-metalman-http.py index 01cd2ca3c..e17df3f9f 100644 --- a/hack/smoke-metalman-http.py +++ b/hack/smoke-metalman-http.py @@ -6,9 +6,9 @@ Stock Noble OVMF and sushy cannot emulate firmware-native DHCP-free UEFI HTTP. The VM therefore starts at the post-firmware EFI boundary: after Metalman has written standard Redfish static IPv4 and UefiHttp settings, the recording BMC -fetches the real Metalman boot artifacts with the VM's source IP and exposes -them on an EFI disk. From shim/GRUB onward the real kernel, installer initrd, -machine image, cloud-init, and branch-built agent path runs unchanged. +fetches the session capability URLs and exposes them on an EFI disk. From +shim/GRUB onward the real kernel, installer initrd, machine image, cloud-init, +and branch-built agent path runs unchanged. """ from __future__ import annotations @@ -49,13 +49,13 @@ AGENT_PORT = 8883 REDFISH_PORT = 8444 REGISTRY_PORT = 5556 -DHCP_PORT = 6768 REGISTRY = "unbounded-http-smoke-registry" HOST_IMAGE = "localhost:5556/unbounded/host-ubuntu2404:http-smoke" NETBOOT_IMAGE = "localhost:5556/unbounded/netboot:http-smoke" AGENT_IMAGE = "localhost:5556/unbounded/agent-ubuntu2404:http-smoke" +HOST_IMAGE_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/host-ubuntu2404:http-smoke" +NETBOOT_IMAGE_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/netboot:http-smoke" AGENT_IMAGE_VM = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/agent-ubuntu2404:http-smoke" -SERVE_URL = f"http://{SERVER_IP}:{HTTP_PORT}" RECORD = TMP / "redfish.jsonl" PCAP = TMP / "traffic.pcap" VIRSH = ["virsh", "--connect", "qemu:///system"] @@ -209,7 +209,11 @@ def create_vm() -> tuple[Path, Path]: def setup_kubernetes_and_images() -> None: log("Building binaries and applying Metalman RBAC/CRDs") run(["make", "machina-manifests"], cwd=ROOT) - for output, package in (("metalman", "./cmd/metalman"), ("unbounded-agent", "./cmd/agent")): + for output, package in ( + ("metalman", "./cmd/metalman"), + ("unbounded-agent", "./cmd/agent"), + ("kubectl-unbounded", "./cmd/kubectl-unbounded"), + ): run(["go", "build", "-o", str(ROOT / "bin" / output), package], cwd=ROOT) run(["kubectl", "apply", "--server-side", "--force-conflicts", "-f", str(ROOT / "deploy/machina/rendered/01-namespace.yaml")]) run(["kubectl", "apply", "--server-side", "--force-conflicts", "-f", str(ROOT / "deploy/machina/crd")]) @@ -263,51 +267,53 @@ def setup_kubernetes_and_images() -> None: def start_fixture(blank_efi: Path, active_efi: Path) -> None: run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", - "-subj", "/CN=metalman-http-fixture", "-addext", "subjectAltName=IP:127.0.0.1", + "-subj", "/CN=metalman-http-fixture", "-addext", f"subjectAltName=IP:{SERVER_IP}", "-keyout", str(TMP / "redfish.key"), "-out", str(TMP / "redfish.crt")]) spawn([sys.executable, str(ROOT / "hack/metalman-redfish-fixture.py"), - "--domain", VM, "--mac", MAC, "--port", str(REDFISH_PORT), - "--cert", str(TMP / "redfish.crt"), "--key", str(TMP / "redfish.key"), - "--record", str(RECORD), "--efi-source", str(blank_efi), - "--efi-active", str(active_efi), "--bridge", BRIDGE, - "--cache-dir", str(TMP / "cache"), "--username", "smoke", "--password", "smoke"], - "redfish.log") + "--domain", VM, "--mac", MAC, "--bind", SERVER_IP, "--port", str(REDFISH_PORT), + "--cert", str(TMP / "redfish.crt"), "--key", str(TMP / "redfish.key"), + "--record", str(RECORD), "--efi-source", str(blank_efi), + "--efi-active", str(active_efi), "--bridge", BRIDGE, + "--username", "smoke", "--password", "smoke"], + "redfish.log") time.sleep(1) def start_metalman_and_replace() -> None: - kubeconfig = TMP / "metalman.kubeconfig" - smoke.write_service_account_kubeconfig(smoke.METALMAN_NAMESPACE, "metalman-controller", kubeconfig) # The guest cannot resolve Docker's kind-control-plane hostname. Use the # control-plane address attached directly to this test's L2 network. api_url = f"https://{KIND_IP}:6443" - spawn(["sudo", "env", f"PATH={os.environ['PATH']}", f"KUBECONFIG={kubeconfig}", - f"METALMAN_APISERVER_URL={api_url}", str(ROOT / "bin/metalman"), "serve-pxe", - f"--site={SITE}", f"--bind-address={SERVER_IP}", f"--serve-url={SERVE_URL}", - f"--http-port={HTTP_PORT}", - f"--dhcp-port={DHCP_PORT}", - f"--cache-dir={TMP / 'cache'}", f"--default-netboot-image={NETBOOT_IMAGE}"], - "metalman.log") + smoke.deploy_split_metalman( + SITE, api_url, smoke.METALMAN_IMAGE, NETBOOT_IMAGE_CLUSTER, + ) machine = { "apiVersion": "unbounded-cloud.io/v1alpha3", "kind": "Machine", "metadata": {"name": NODE, "labels": {"unbounded-cloud.io/site": SITE}}, "spec": { - "pxe": { - "image": HOST_IMAGE, "netbootImage": NETBOOT_IMAGE, "bootProtocol": "HTTP", + "host": {"netboot": { + "image": HOST_IMAGE_CLUSTER, + "netbootImage": NETBOOT_IMAGE_CLUSTER, + "transport": "HTTP", + "configurationSource": "Redfish", + "networkMode": "Static", + "endpointRef": "bootstrap-pending", "targetDisk": "/dev/vda", "dhcpLeases": [{"mac": MAC, "ipv4": NODE_IP, "subnetMask": "255.255.255.0", "gateway": SERVER_IP, "dns": ["8.8.8.8"]}], - "redfish": {"url": f"https://127.0.0.1:{REDFISH_PORT}", "username": "smoke", + "redfish": {"url": f"https://{SERVER_IP}:{REDFISH_PORT}", "username": "smoke", "deviceID": VM, "passwordRef": {"name": "http-bmc-pass", "namespace": "default", "key": "password"}}, "cloudInit": {"userDataConfigMapRef": {"name": "http-smoke-user-data", "namespace": "default", "key": "user-data"}}, - }, + }}, "agent": {"image": AGENT_IMAGE_VM, "url": f"http://{SERVER_IP}:{AGENT_PORT}/unbounded-agent-linux-amd64.tar.gz"}, }, } run(["kubectl", "apply", "-f", "-"], input=json.dumps(machine), text=True) + bootstrap = smoke.start_bootstrap_netboot( + SITE, NODE, BRIDGE, SERVER_IP, HTTP_PORT, TMP / "bootstrap-netboot.log", + ) for _ in range(120): fingerprint = subprocess.run( ["kubectl", "get", "machine", NODE, "-o", "jsonpath={.status.redfish.certFingerprint}"], @@ -319,28 +325,11 @@ def start_metalman_and_replace() -> None: else: raise RuntimeError("Redfish certificate fingerprint was not recorded") - log("Waiting for host and netboot OCI images to be cached") - cache_dir = TMP / "cache" / "oci" - metalman_log = TMP / "metalman.log" - for _ in range(300): - disk_dirs = list(cache_dir.glob("*/amd64/disk")) - host_ready = any((path / "disk.img.gz").is_file() for path in disk_dirs) - netboot_ready = any((path / "bootx64.efi").is_file() for path in disk_dirs) - log_text = metalman_log.read_text(encoding="utf-8") if metalman_log.exists() else "" - host_published = any("OCI image cached" in line and f"image={HOST_IMAGE}" in line - for line in log_text.splitlines()) - netboot_published = any("OCI image cached" in line and f"image={NETBOOT_IMAGE}" in line - for line in log_text.splitlines()) - if host_ready and netboot_ready and host_published and netboot_published: - break - time.sleep(1) - else: - raise RuntimeError("host and netboot OCI images were not cached within 5 minutes") - operation = smoke.create_machine_operation( "http-smoke-host-replace", "HostReplace", machine_ref=NODE ) smoke.wait_machine_operation_complete(operation, timeout=1800) + smoke.wait_process_success(bootstrap, timeout=120) def fixture_writes() -> list[dict[str, Any]]: @@ -359,12 +348,15 @@ def assert_contract() -> None: boot_patches = [entry["body"].get("Boot", {}) for entry in writes if entry["method"] == "PATCH" and entry["path"] == f"/redfish/v1/Systems/{VM}"] if not any(boot.get("BootSourceOverrideTarget") == "UefiHttp" - and boot.get("BootSourceOverrideEnabled") == "Continuous" + and boot.get("BootSourceOverrideEnabled") == "Once" and boot.get("BootSourceOverrideMode") == "UEFI" - and boot.get("HttpBootUri") == f"{SERVE_URL}/bootx64.efi" for boot in boot_patches): + and "/v1/netboot/sessions/" in boot.get("HttpBootUri", "") + and boot.get("HttpBootUri", "").endswith("/artifacts/bootx64.efi") + for boot in boot_patches): raise AssertionError(f"standard UefiHttp PATCH not recorded: {boot_patches}") firmware_fetches = [entry for entry in writes if entry["method"] == "FIRMWARE_FETCH"] - if not any(entry["path"] == f"{SERVE_URL}/bootx64.efi" + if not any("/v1/netboot/sessions/" in entry["path"] + and entry["path"].endswith("/artifacts/bootx64.efi") and entry["body"] == {"source": NODE_IP} for entry in firmware_fetches): raise AssertionError(f"state-derived post-power-on firmware fetch not recorded: {firmware_fetches}") diff --git a/hack/smoke-metalman.py b/hack/smoke-metalman.py index f05eb3442..7166cbf00 100755 --- a/hack/smoke-metalman.py +++ b/hack/smoke-metalman.py @@ -8,6 +8,7 @@ import base64 import json import os +import secrets import signal import shutil import socket @@ -32,7 +33,7 @@ NODE_LABEL_KEY = "unbounded-cloud.io/smoke-test" NODE_LABEL_VALUE = "metalman" METALMAN_NAMESPACE = "unbounded-system" -METALMAN_CONTROLLER_SA = "metalman-controller" +METALMAN_IMAGE = "unbounded/metalman:smoke" VM_NAME = "unbounded-metal-smoke" NET_NAME = "unbounded-metal-smoke" SUBNET = "192.168.200" @@ -45,9 +46,7 @@ REDFISH_PORT = 8443 HTTP_PORT = 8880 AGENT_DOWNLOAD_PORT = 8881 -CACHE_DIR = TMPDIR / "cache" ARTIFACT_DIR = TMPDIR / "artifacts" -SERVE_URL = f"http://{SERVER_IP}:{HTTP_PORT}" AGENT_TARBALL = ARTIFACT_DIR / "unbounded-agent-linux-amd64.tar.gz" AGENT_DOWNLOAD_URL = f"http://{SERVER_IP}:{AGENT_DOWNLOAD_PORT}/{AGENT_TARBALL.name}" REGISTRY_PORT = 5555 @@ -55,6 +54,8 @@ IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/host-ubuntu2404:smoke" NETBOOT_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/netboot:smoke" AGENT_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/agent-ubuntu2404:smoke" +IMAGE_NAME_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/host-ubuntu2404:smoke" +NETBOOT_IMAGE_NAME_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/netboot:smoke" # The agent runs inside a VM on an isolated libvirt network. "localhost" inside # the VM resolves to the VM's own loopback, not the host. Use the host's # bridge IP so the VM can reach the registry over the virtual network. @@ -355,7 +356,7 @@ def clean_libvirt() -> None: run_quiet(["sudo", "ip", "link", "delete", "veth-kind-smoke"]) # Kill any leftover Redfish fixture from a previous run. run_quiet(["sudo", "pkill", "-f", "metalman-redfish-fixture.py"]) - # Kill any leftover metalman serve-pxe from a previous run. + # Kill any leftover local Metalman edge from a previous run. # Use the binary path to avoid matching this script (smoke-metalman.py). run_quiet(["sudo", "pkill", "-f", "bin/metalman"]) # Kill any leftover artifact download server from a previous run. @@ -478,6 +479,181 @@ def apiserver_url() -> str: return url +def deploy_split_metalman(site: str, api_url: str, metalman_image: str, + default_netboot_image: str) -> None: + """Deploy the controller/server roles used by the operator into kind.""" + inspect = subprocess.run( + ["docker", "image", "inspect", metalman_image], + stdout=DEVNULL, stderr=DEVNULL, + ) + if inspect.returncode != 0: + run([ + "docker", "build", "-t", metalman_image, + "-f", str(REPO_ROOT / "images/metalman/Containerfile"), str(REPO_ROOT), + ]) + run(["kind", "load", "docker-image", metalman_image, "--name", "kind"]) + + site_resource = { + "apiVersion": API_VERSION, + "kind": "Site", + "metadata": {"name": site}, + "spec": { + "nodeCidrs": ["192.168.200.0/24"], + "podCidrAssignments": [{"cidrBlocks": ["10.250.0.0/16"]}], + "components": {"metalman": {"enabled": True}}, + }, + } + kubectl(["apply", "-f", "-"], input=json.dumps(site_resource).encode(), stdout=DEVNULL) + + capability = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": f"metalman-capability-{site}", "namespace": METALMAN_NAMESPACE}, + "stringData": {"capability.key": secrets.token_hex(32)}, + } + kubectl(["apply", "-f", "-"], input=json.dumps(capability).encode(), stdout=DEVNULL) + + labels = { + "app": "unbounded-metalman", + f"{API_GROUP}/site": site, + } + for role, replicas in (("controller", 1), ("server", 2)): + role_labels = { + **labels, + "app.kubernetes.io/name": f"metalman-{role}", + "app.kubernetes.io/component": role, + } + ports = [{"name": "health", "containerPort": 8081}] + if role == "server": + ports.append({"name": "http", "containerPort": 8880}) + args = [ + role, + f"--site={site}", + "--cache-dir=/var/cache/metalman", + f"--default-netboot-image={default_netboot_image}", + ] + if role == "controller": + args.extend([ + "--leader-elect-lease-duration=60s", + "--leader-elect-renew-deadline=40s", + "--leader-elect-retry-period=5s", + ]) + deployment = { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": f"metalman-{role}-{site}", "namespace": METALMAN_NAMESPACE}, + "spec": { + "replicas": replicas, + "selector": {"matchLabels": role_labels}, + "template": { + "metadata": {"labels": role_labels}, + "spec": { + "serviceAccountName": f"metalman-{role}", + "containers": [{ + "name": "metalman", + "image": metalman_image, + "imagePullPolicy": "IfNotPresent", + "args": args, + "env": [ + {"name": "POD_NAMESPACE", "value": METALMAN_NAMESPACE}, + {"name": "METALMAN_APISERVER_URL", "value": api_url}, + ], + "ports": ports, + "volumeMounts": [ + {"name": "cache", "mountPath": "/var/cache/metalman"}, + {"name": "capability", "mountPath": "/var/run/secrets/metalman", "readOnly": True}, + ], + }], + "volumes": [ + {"name": "cache", "emptyDir": {}}, + {"name": "capability", "secret": {"secretName": f"metalman-capability-{site}"}}, + ], + }, + }, + }, + } + kubectl(["apply", "-f", "-"], input=json.dumps(deployment).encode(), stdout=DEVNULL) + + service = { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": f"metalman-server-{site}", "namespace": METALMAN_NAMESPACE}, + "spec": { + "selector": { + **labels, + "app.kubernetes.io/name": "metalman-server", + "app.kubernetes.io/component": "server", + }, + "ports": [{"name": "http", "port": 8880, "targetPort": "http"}], + }, + } + kubectl(["apply", "-f", "-"], input=json.dumps(service).encode(), stdout=DEVNULL) + for role in ("controller", "server"): + kubectl([ + "-n", METALMAN_NAMESPACE, "rollout", "status", + f"deployment/metalman-{role}-{site}", "--timeout=5m", + ]) + + +def start_bootstrap_netboot(site: str, machine: str, interface: str, address: str, + http_port: int, log_path: Path) -> subprocess.Popen[Any]: + process = spawn([ + str(KUBECTL_UNBOUNDED), "site", "bootstrap-netboot", site, + f"--machine={machine}", f"--interface={interface}", f"--address={address}", + f"--http-port={http_port}", f"--metalman-binary={BINARY}", "--timeout=30m", + ], log_path) + endpoint = f"bootstrap-{machine}" + for _ in range(180): + result = subprocess.run( + [KUBECTL, "get", "netbootendpoint", endpoint, "-o", "json"], + capture_output=True, text=True, + ) + if result.returncode == 0: + resource = json.loads(result.stdout) + ready = any( + condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in resource.get("status", {}).get("conditions", []) + ) + if ready: + return process + if process.poll() is not None: + die(f"bootstrap-netboot exited before endpoint {endpoint} became Ready") + time.sleep(1) + die(f"Timed out waiting for bootstrap endpoint {endpoint}") + + +def delete_metalman_server_during_provisioning(site: str) -> None: + """Delete one serving replica after a session exists to exercise edge reconnect.""" + for _ in range(300): + result = subprocess.run( + [KUBECTL, "get", "netbootsessions", "-o", "json"], + capture_output=True, text=True, + ) + if result.returncode == 0 and json.loads(result.stdout).get("items"): + break + time.sleep(1) + else: + die("Timed out waiting for a NetbootSession before server disruption") + pods = json.loads(run( + [KUBECTL, "-n", METALMAN_NAMESPACE, "get", "pod", + "-l", f"app.kubernetes.io/name=metalman-server,{API_GROUP}/site={site}", + "-o", "json"], + capture_output=True, text=True, + ).stdout).get("items", []) + ready_pods = sorted( + pod["metadata"]["name"] for pod in pods + if not pod["metadata"].get("deletionTimestamp") and any( + condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in pod.get("status", {}).get("conditions", []) + ) + ) + if not ready_pods: + die("No Metalman server pod available for disruption") + pod = ready_pods[0] + log(f"Deleting Metalman server pod {pod} during provisioning") + kubectl(["-n", METALMAN_NAMESPACE, "delete", "pod", pod, "--wait=false"]) + + def configure_kind_control_plane_node_ip(container: str, node_ip: str) -> None: """Make the kind control-plane Node advertise its VM-reachable IP.""" log(f"Configuring {container} kubelet node IP as {node_ip}") @@ -829,7 +1005,12 @@ def assert_cloud_init_done(timeout: int = 900) -> None: op_status = op.get("status", {}) phase = op_status.get("phase", "") message = op_status.get("message", "") - for c in op_status.get("conditions", []): + target = next( + (item for item in op_status.get("targets", []) + if item.get("machineRef") == NODE_NAME), + {}, + ) + for c in target.get("conditions", []): if c.get("type") == "CloudInitDone": status = c.get("status", "") reason = c.get("reason", "") @@ -1082,13 +1263,13 @@ def main() -> None: "-out", str(TMPDIR / "redfish.crt"), "-days", "1", "-nodes", "-subj", "/CN=metalman-redfish-fixture", - "-addext", "subjectAltName=IP:127.0.0.1", + "-addext", f"subjectAltName=IP:{SERVER_IP}", ], check=True) - redfish_url = f"https://127.0.0.1:{REDFISH_PORT}" + redfish_url = f"https://{SERVER_IP}:{REDFISH_PORT}" proc = spawn([ sys.executable, str(REPO_ROOT / "hack" / "metalman-redfish-fixture.py"), "--domain", VM_NAME, "--mac", MAC_ADDRESS, - "--bind", "127.0.0.1", "--port", str(REDFISH_PORT), + "--bind", SERVER_IP, "--port", str(REDFISH_PORT), "--cert", str(TMPDIR / "redfish.crt"), "--key", str(TMPDIR / "redfish.key"), "--record", str(TMPDIR / "redfish.jsonl"), @@ -1210,6 +1391,8 @@ def main() -> None: server_url = apiserver_url() log(f" API server URL: {server_url}") + deploy_split_metalman(SITE, server_url, METALMAN_IMAGE, NETBOOT_IMAGE_NAME_CLUSTER) + protonode = { "apiVersion": API_VERSION, "kind": "Machine", @@ -1218,21 +1401,28 @@ def main() -> None: "labels": {f"{API_GROUP}/site": SITE}, }, "spec": { - "pxe": { - "image": IMAGE_NAME, - "redfish": { - "url": redfish_url, - "username": "", - "deviceID": VM_NAME, - "passwordRef": {"name": "bmc-pass", "key": "password", "namespace": NODE_NS}, + "host": { + "netboot": { + "image": IMAGE_NAME_CLUSTER, + "netbootImage": NETBOOT_IMAGE_NAME_CLUSTER, + "transport": "TFTP", + "configurationSource": "DHCP", + "networkMode": "DHCP", + "endpointRef": "bootstrap-pending", + "redfish": { + "url": redfish_url, + "username": "", + "deviceID": VM_NAME, + "passwordRef": {"name": "bmc-pass", "key": "password", "namespace": NODE_NS}, + }, + "dhcpLeases": [{ + "mac": MAC_ADDRESS, + "ipv4": NODE_IP, + "subnetMask": "255.255.255.0", + "gateway": GATEWAY, + "dns": [DNS_SERVER], + }], }, - "dhcpLeases": [{ - "mac": MAC_ADDRESS, - "ipv4": NODE_IP, - "subnetMask": "255.255.255.0", - "gateway": GATEWAY, - "dns": [DNS_SERVER], - }], }, "agent": { "image": AGENT_IMAGE_NAME_VM, @@ -1247,26 +1437,11 @@ def main() -> None: stdout=DEVNULL) log(" Resources created") - log("Starting metalman serve-pxe") - metalman_kubeconfig = TMPDIR / "metalman-controller.kubeconfig" - write_service_account_kubeconfig(METALMAN_NAMESPACE, METALMAN_CONTROLLER_SA, metalman_kubeconfig) - metalman_env = [f"METALMAN_APISERVER_URL={server_url}"] - metalman_env.append(f"KUBECONFIG={metalman_kubeconfig}") - - proc = spawn([ - "sudo", "env", *metalman_env, - str(BINARY), "serve-pxe", f"--site={SITE}", f"--bind-address={SERVER_IP}", - f"--cache-dir={CACHE_DIR}", - f"--serve-url={SERVE_URL}", "--dhcp-interface=virbr-smoke", - f"--default-netboot-image={NETBOOT_IMAGE_NAME}", - "--leader-elect-lease-duration=60s", - "--leader-elect-renew-deadline=40s", - "--leader-elect-retry-period=5s", - ], TMPDIR / "serve.log") - log(f" serve PID={proc.pid}") - - time.sleep(2) - check_procs() + log("Starting local Metalman edge through kubectl-unbounded") + bootstrap_proc = start_bootstrap_netboot( + SITE, NODE_NAME, "virbr-smoke", SERVER_IP, HTTP_PORT, + TMPDIR / "bootstrap-netboot.log", + ) log("Triggering HostReplace through kubectl-unbounded") operation_log = TMPDIR / "kubectl-host-replace.log" @@ -1274,6 +1449,7 @@ def main() -> None: ["replace", NODE_NAME, "--force", "--ttl=3600"], operation_log.name, ) + delete_metalman_server_during_provisioning(SITE) # Log free space so we can correlate disk exhaustion with VM failures. df = subprocess.run(["df", "-h", str(TMPDIR)], capture_output=True, text=True) @@ -1289,6 +1465,7 @@ def main() -> None: wait_k8s_node(NODE_NAME, timeout=900) assert_node_ready(NODE_NAME, timeout=720) assert_node_label(NODE_NAME, NODE_LABEL_KEY, NODE_LABEL_VALUE) + wait_process_success(bootstrap_proc, timeout=120) run_operation_smoke_suite() diff --git a/hack/smoke_metalman_contract_test.go b/hack/smoke_metalman_contract_test.go new file mode 100644 index 000000000..7b7a9ba60 --- /dev/null +++ b/hack/smoke_metalman_contract_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package hack + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestMetalmanSmokeSuitesUseSplitRuntime(t *testing.T) { + t.Parallel() + + for _, name := range []string{"smoke-metalman.py", "smoke-metalman-http.py"} { + name := name + t.Run(name, func(t *testing.T) { + t.Parallel() + + contents, err := os.ReadFile(filepath.Join(".", name)) + if err != nil { + t.Fatal(err) + } + text := string(contents) + for _, stale := range []string{"serve-pxe", `"bootProtocol"`} { + if strings.Contains(text, stale) { + t.Errorf("%s still contains legacy %q wiring", name, stale) + } + } + for _, required := range []string{"deploy_split_metalman", "bootstrap-netboot"} { + if !strings.Contains(text, required) { + t.Errorf("%s does not exercise %q", name, required) + } + } + }) + } +} + +func TestTraditionalMetalmanSmokeDisruptsServerDuringProvisioning(t *testing.T) { + t.Parallel() + + contents, err := os.ReadFile(filepath.Join(".", "smoke-metalman.py")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(contents), "delete_metalman_server_during_provisioning") { + t.Fatal("traditional smoke does not disrupt a server pod during provisioning") + } +} + +func TestHTTPFixtureFetchesCapabilityEntrypoint(t *testing.T) { + t.Parallel() + + contents, err := os.ReadFile(filepath.Join(".", "metalman-redfish-fixture.py")) + if err != nil { + t.Fatal(err) + } + text := string(contents) + if strings.Contains(text, "cache_dir.glob") { + t.Fatal("HTTP fixture still reads a process-local Metalman cache") + } + if !strings.Contains(text, `("http-entrypoint.efi", boot_url)`) { + t.Fatal("HTTP fixture does not fetch the capability-scoped firmware URL") + } +} From be6f05dcd421478f2ca881b0a3705d9516bfc965 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 20:07:28 +0000 Subject: [PATCH 41/43] docs: describe resilient metalman netboot --- cmd/metalman/README.md | 322 ++++------------ designs/metalman-netboot-architecture.md | 328 +++++++++++++++++ docs/content/concepts/bare-metal.md | 86 +++-- docs/content/guides/pxe.md | 347 +++++++----------- docs/content/reference/agent/configuration.md | 2 +- docs/content/reference/architecture.md | 83 +++-- docs/content/reference/cli.md | 44 +++ docs/content/reference/machina-crd.md | 74 +++- 8 files changed, 743 insertions(+), 543 deletions(-) create mode 100644 designs/metalman-netboot-architecture.md diff --git a/cmd/metalman/README.md b/cmd/metalman/README.md index 696af5b43..c6b161d50 100644 --- a/cmd/metalman/README.md +++ b/cmd/metalman/README.md @@ -1,272 +1,110 @@ # metalman -Join bare metal nodes to a Kubernetes cluster using PXE and (optionally) Redfish. +Metalman provisions bare-metal Kubernetes nodes with DHCP/TFTP or UEFI HTTP +boot, optional Redfish control, and TPM 2.0 attestation. -A dedicated `metalman` binary is shipped as a container image for running the -PXE server inside a cluster. `unbounded-operator` deploys that image when a -Site enables the Metalman component. +## Runtime Roles -Run `metalman version` to print the binary version. - -## Usage - -```bash -# Create a Machine -kubectl apply -f - < [flags] -``` - -#### Deploying With Site Components - -Enable Metalman in the Site spec and let `unbounded-operator` create or update -the per-site Deployment running `metalman serve-pxe`: - -```yaml -apiVersion: unbounded-cloud.io/v1alpha3 -kind: Site -metadata: - name: rack-a -spec: - components: - metalman: - enabled: true - dhcpAutoInterface: true -``` - -The resulting Deployment (`metalman-controller-`) runs with host networking for DHCP. -It exposes ports 8880/tcp (HTTP), 8081/tcp (health), 67/udp (DHCP), and 69/udp (TFTP). - -The generated Deployment uses host networking, a `CriticalAddonsOnly` -toleration, DNS policy `ClusterFirstWithHostNet`, and a node selector -`unbounded-cloud.io/site=`. Resource requests are 100m CPU / 128Mi -memory with limits of 500m CPU / 256Mi memory. +Metalman is split into three roles: -#### DHCP Modes +| Command | Responsibility | Kubernetes access | Host networking | +|---------|----------------|-------------------|-----------------| +| `metalman controller` | MachineOperation, Redfish, immutable session creation, OCI preparation | Yes, leader-elected | No | +| `metalman server` | Session artifacts, callbacks, attestation, edge decision API | Yes | No | +| `metalman edge` | DHCP/TFTP on the provisioning LAN and HTTP proxying | No | Only when required for L2 | -The DHCP server operates in one of two modes depending on whether -`--dhcp-interface` is set: +Enabling `Site.spec.components.metalman.enabled` makes the operator deploy one +controller, two server replicas, a server Service, a PodDisruptionBudget, and a +shared capability-signing Secret. Endpoint resources determine edge placement. -- **Interface mode** (`--dhcp-interface=eth0`): Binds to a network interface - and listens for broadcast DHCP traffic. Use this when the controller is - directly attached to the provisioning network. +## Endpoint Types -- **Auto-interface mode** (`--dhcp-auto-interface`): Automatically detects - the network interface from the server bind address. Mutually exclusive - with `--dhcp-interface`. +- `ManagedL2` creates a host-network edge on selected nodes. It can own DHCP, + TFTP, and private HTTP on a directly attached provisioning network. +- `HTTP` creates replicated HTTP edge pods and a Service. Public endpoints must + use HTTPS, either from a TLS Secret or external TLS termination. +- `ExternalL2` describes an edge outside the cluster. No edge workload is + created. `kubectl unbounded site bootstrap-netboot` uses this mode. -- **Relay mode** (no `--dhcp-interface` or `--dhcp-auto-interface`): Listens - on a UDP port for unicast packets only. Use this when a DHCP relay agent - forwards requests from a remote subnet. +WireGuard is an L3 transport and does not carry DHCP broadcasts. Keep a DHCP +edge or relay on the client LAN. TFTP, HTTP, and Redfish can use routed paths. -Leader election is always enabled regardless of DHCP mode. Each site gets -its own leader-election lease (`metalman-`). - -#### Security Model - -A mostly-trusted network between the controller and the bare metal hosts is -assumed. Bootstrap tokens (Kubernetes ServiceAccount tokens) are issued to -nodes based on source IP - the controller looks up the Machine whose NIC -matches the requesting IP and issues a short-lived token for that node. - -Bootstrap tokens are delivered using the standard TPM 2.0 credential encryption workflow. -The client's endorsement key (EK) public key is stored in `status.tpm.ekPublicKey` when first seen. -So it's possible to prove that the bootstrap token was delivered only to trusted hosts. - -#### Sites - -The `--site` flag scopes a `metalman serve-pxe` instance to a subset of Machines. -The value is matched against the `unbounded-cloud.io/site` label on Machine -resources: - -```bash -# Manage only Machines labeled site=rack-a -metalman serve-pxe --site=rack-a --dhcp-interface=eth0 - -# Manage only unlabeled Machines (the default) -metalman serve-pxe --dhcp-interface=eth0 -``` - -Each site gets its own leader-election lease (`metalman-`), so -multiple sites can coexist on one cluster with independent HA. A `metalman serve-pxe` -instance with no `--site` manages Machines that do not have the site label -at all. - -### Images - -Metalman uses two OCI images when repaving a machine: - -Existing Machines may continue to use the deprecated top-level `spec.pxe` -shape; Metalman resolves both forms through the same compatibility accessor. - -- `spec.host.netboot.image` is the machine image. It contains `/disk/disk.img.gz`, a - gzip-compressed raw disk image written to the target disk. -- `spec.host.netboot.netbootImage` is the reusable PXE boot environment. It contains - bootloaders, kernel, initrd, templates, and metadata. Its cloud-init template - downloads and installs `unbounded-agent` from the configured release/source. - If omitted, Metalman uses the release-matched `--default-netboot-image`. - -Both images are built `FROM scratch` and use `/disk/` as the artifact root, -following the kubevirt containerDisk convention. Files with a `.tmpl` suffix in -the netboot image are Go templates rendered per-machine at serve time; other -files are served verbatim. A `metadata.yaml` file in the netboot image provides -image-level configuration such as `dhcpBootImageName` and `httpBootPath`. - -Images are built, tagged, and pushed using standard container tooling: - -```bash -docker build -t ghcr.io/azure/host-ubuntu2404:v1 -f images/host-ubuntu2404/Containerfile . -docker build -t ghcr.io/azure/netboot:v1 -f images/netboot/Containerfile . -docker push ghcr.io/azure/host-ubuntu2404:v1 -docker push ghcr.io/azure/netboot:v1 -``` - -### Machine - -A Machine is a cluster-scoped custom resource representing a single bare metal -host. At minimum it needs a NIC (MAC + static IP) and a machine image reference: +## Machine Example ```yaml apiVersion: unbounded-cloud.io/v1alpha3 kind: Machine metadata: name: node-01 + labels: + unbounded-cloud.io/site: rack-a spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - # Defaults to PXE. Set to HTTP to use Redfish UEFI HTTP boot. - bootProtocol: PXE - # Optional. Recommended when the host has multiple disks. - targetDisk: /dev/disk/by-id/example-os-disk - dhcpLeases: - - mac: "aa:bb:cc:dd:ee:01" - ipv4: "10.0.0.11" - subnetMask: "255.255.255.0" - gateway: "10.0.0.1" + host: + netboot: + image: ghcr.io/azure/host-ubuntu2404:v1 + netbootImage: ghcr.io/azure/netboot:v1 + endpointRef: rack-a-l2 + transport: TFTP + configurationSource: DHCP + networkMode: DHCP + dhcpLeases: + - mac: aa:bb:cc:dd:ee:01 + ipv4: 10.0.0.11 + subnetMask: 255.255.255.0 + gateway: 10.0.0.1 + dns: [10.0.0.1] + redfish: + url: https://10.0.10.11 + username: admin + passwordRef: + name: bmc-node-01-pass + namespace: default + key: password ``` -This is enough for the DHCP server to issue a lease and for TFTP/HTTP to serve -boot artifacts from the default netboot image. Set `spec.host.netboot.netbootImage` only -when a Machine needs a non-default PXE boot environment. The node must be -manually PXE-booted (or have PXE as its default boot option). - -When `spec.host.netboot.bootProtocol` is `HTTP`, `dhcpLeases` also supplies the static -UEFI HTTP boot client configuration. Metalman uses Redfish to disable DHCPv4 on -the host EthernetInterface matching the first lease MAC and writes that lease's -IPv4 address, subnet mask, gateway, and DNS servers before setting the UEFI HTTP -boot override. With Redfish access and an HTTP boot URL, repaving can run without -any DHCP server on the provisioning network. If a host has multiple NICs, put the -UEFI HTTP boot NIC first in `dhcpLeases`. +Supported boot combinations are: -The default netboot template passes the selected lease MAC to the installer -initrd, which uses it to select the provisioning NIC instead of assuming a fixed -interface name such as `eth0`. It also passes the lease DNS servers, configures -the installer network without DHCP, and writes matching MAC-based static netplan -configuration into the installed system before its first boot. It disables -cloud-init network rendering so fallback DHCP configuration cannot conflict with -that file. The default netboot image serves the same lease as NoCloud -`network-config`. If `spec.host.netboot.targetDisk` is -set, the installer writes the image to that disk; otherwise it falls back to -automatic disk selection. +| Transport | Configuration source | Network mode | +|-----------|----------------------|--------------| +| TFTP | DHCP | DHCP | +| HTTP | DHCP | DHCP | +| HTTP | Redfish | DHCP | +| HTTP | Redfish | Static | -Stock Ubuntu OVMF and sushy-emulator cannot emulate the complete DHCP-free -Redfish-to-firmware UEFI HTTP path. Repository CI therefore tests Metalman's -Redfish writes and then starts at a staged post-firmware EFI boundary, while -capturing the guest's traffic through installation and reboot to prove it emits -no DHCP packets. Applying Redfish settings and fetching the first EFI binary -remain firmware and BMC hardware-conformance responsibilities. +TFTP with Redfish configuration and static networking without Redfish are +rejected by API validation. -#### BMC +## Durable Provisioning -Adding a `redfish` block enables remote power management. Metalman uses it for -`MachineOperation` host actions without physical access: +Each HostReplace target gets an immutable `NetbootSession`. It snapshots the +Machine and operation identities, endpoint, boot settings, resolved OCI +digests, artifact allowlist, cluster settings, cloud-init data, and expiry. +Controller side effects wait until the session is ready. -```yaml -apiVersion: unbounded-cloud.io/v1alpha3 -kind: Machine -metadata: - name: node-01 -spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - dhcpLeases: - - mac: "aa:bb:cc:dd:ee:01" - ipv4: "10.0.0.11" - subnetMask: "255.255.255.0" - gateway: "10.0.0.1" - redfish: - url: https://bmc-node-01.example.com - username: admin - passwordRef: - name: bmc-node-01-pass - namespace: default - key: password -``` +Artifact and callback URLs contain an operation-scoped HMAC capability. HTTP, +TFTP, callbacks, and attestation resolve the exact session rather than trusting +source IP. Milestones are recorded on the exact session and operation target. +Artifacts are addressed by immutable digest and support HTTP ranges, allowing +an edge to resume a transfer through another server pod after disruption. -The BMC password is read from a Secret in the same namespace (key: `password`). -On first connection, the controller captures the BMC's TLS certificate -fingerprint and pins it in `status.redfish.certFingerprint` for subsequent -requests. +## Temporary First-Node Bootstrap -To repave a node with BMC access: +An administrator attached to the provisioning LAN can bootstrap the first Site +node without running Kubernetes controllers locally: ```bash -kubectl unbounded machine repave node-01 +kubectl unbounded site bootstrap-netboot rack-a \ + --machine node-01 \ + --interface eno1 \ + --address 10.0.0.2 ``` -This creates a `HostReplace` `MachineOperation`. Metalman handles the rest: it -configures the boot override for the selected `spec.host.netboot.bootProtocol`, executes -a Redfish force restart, waits for the installer `/pxe/disable` signal, tracks -first-boot cloud-init on the operation, and completes after the node is back up. +The command enables the in-cluster control and server planes, creates an +ephemeral ExternalL2 endpoint, port-forwards the local edge to a ready server +pod with reconnection, and stops when the designated Machine's Node is Ready. +The Machine endpoint reference is restored during cleanup. + +Use repeatable `--routed-cidr` flags when the administrator host must also act +as a temporary unbounded-net gateway for BMC or provisioning subnets. + +Run `metalman version` to print the binary version. diff --git a/designs/metalman-netboot-architecture.md b/designs/metalman-netboot-architecture.md new file mode 100644 index 000000000..47440730c --- /dev/null +++ b/designs/metalman-netboot-architecture.md @@ -0,0 +1,328 @@ +# Metalman Netboot Architecture + +## Status + +Implemented for the `v1alpha3` API. This is a breaking alpha redesign; the new +boot axes and endpoint reference are the supported contract. + +## Goals + +- Keep the normal Metalman control and serving planes off host networking. +- Keep active provisioning valid across controller failover, server pod loss, + and rolling updates. +- Support traditional DHCP/TFTP PXE on a local L2 and UEFI HTTP boot through + routed, NAT, NodePort, load-balancer, and internet paths. +- Configure HTTP boot through either DHCP or Redfish, with DHCP or static + firmware networking. +- Let an administrator bootstrap the first Site node without running Kubernetes + controllers on the administrator machine. +- Use the existing unbounded-net dataplane for optional routed access from an + external bootstrap host without pretending WireGuard provides L2. +- Remove source-address identity and correlate every artifact, callback, and + attestation request with one operation target. + +## Non-goals + +- Extending DHCP broadcasts through WireGuard. +- Hiding loss of a ManagedL2 address with ordinary pod replication. The L2 + address must be stable through node placement, a VIP, or an external edge. +- Preserving the old alpha `bootProtocol`, Site replica, or DHCP auto-interface + API fields. +- Providing a multi-machine temporary bootstrap handoff. The command owns one + designated first Machine and exits when that Node is Ready. +- Requiring shared writable storage for OCI caches. + +## Runtime Roles + +```mermaid +flowchart LR + K["Kubernetes API"] + C["metalman controller
leader elected"] + SVC["metalman-server Service"] + S1["server replica A"] + S2["server replica B"] + E["edge
ManagedL2, HTTP, or ExternalL2"] + F["Firmware / installer / agent"] + B["Redfish BMC"] + R["OCI registry"] + + C <--> K + S1 <--> K + S2 <--> K + C --> R + S1 --> R + S2 --> R + C --> B + SVC --> S1 + SVC --> S2 + E --> SVC + F <--> E +``` + +### Controller + +The singleton per-Site controller owns leader-elected reconciliation, Redfish +certificate pinning and power/boot operations, OCI resolution, and immutable +session creation. It has no DHCP, TFTP, artifact, callback, or attestation +listener and does not use host networking. + +### Server + +Two server replicas sit behind a ClusterIP Service. They materialize disposable +digest-addressed OCI caches and serve authenticated session artifacts, +callbacks, DHCP decisions, and TPM attestation. A PodDisruptionBudget, +zero-unavailable rollout, health probes, and topology spread preserve service +availability. A lost local cache is reconstructed from the immutable digest. + +### Edge + +The edge has no controller RBAC. It proxies HTTP and, when attached to a +provisioning LAN, owns DHCP and TFTP. Its authenticated internal DHCP decision +calls use a projected `metalman-edge` ServiceAccount token. Firmware-facing +paths carry the session capability and do not rely on client IP. + +## Endpoint Model + +`NetbootEndpoint` is cluster-scoped and declares a stable external URL, Site, +trust boundary, TLS policy, placement, and readiness. + +| Type | Ownership | Networking | Typical use | +|------|-----------|------------|-------------| +| `ManagedL2` | Operator Deployment | Host network | Rack-local DHCP, TFTP, and private HTTP | +| `HTTP` | Operator Deployment and Service | Pod network | UEFI HTTP through ClusterIP, NodePort, load balancer, or ingress | +| `ExternalL2` | External process | Host network outside cluster | First-node bootstrap or dedicated appliance | + +Only `ManagedL2` uses host networking in the normal operator deployment. Public +endpoints require an `https://` URL and TLS mode `Secret` or `External`. Secret +mode mirrors the selected certificate into the operator namespace and rolls +the edge when its checksum changes. + +An endpoint is usable only when status observes the current generation and has +`Ready=True`. External edges claim and renew status themselves. + +## Machine Boot Axes + +`Machine.spec.host.netboot` separates concepts previously conflated as one boot +protocol: + +- `transport`: `TFTP` or `HTTP`. +- `configurationSource`: `DHCP` or `Redfish` supplies the firmware boot target. +- `networkMode`: `DHCP` or `Static` configures firmware networking. +- `endpointRef`: required stable NetbootEndpoint identity. + +Supported combinations are: + +| Transport | Configuration source | Network mode | Behavior | +|-----------|----------------------|--------------|----------| +| TFTP | DHCP | DHCP | DHCP supplies lease and tokenized TFTP path. | +| HTTP | DHCP | DHCP | DHCP supplies lease and signed HTTP boot URL. | +| HTTP | Redfish | DHCP | Redfish writes the signed URL; firmware obtains networking through DHCP. | +| HTTP | Redfish | Static | Redfish writes both the signed URL and static NIC configuration. | + +TFTP+Redfish, static networking without Redfish, and Redfish configuration +without BMC details are rejected by validation. + +## Immutable Sessions + +One `NetbootSession` is created per HostReplace operation target. Its spec is +immutable and records: + +- Exact Machine, MachineOperation, and endpoint names, UIDs, and generations. +- Endpoint external URL, boot axes, architecture, leases, and target disk. +- Machine and netboot OCI references resolved to SHA-256 digests. +- The complete artifact allowlist and firmware artifact. +- Cluster API, CA, DNS, Kubernetes version, agent settings, provider labels, + and resolved cloud-init user-data. +- Expiration. + +The operation target stores the session name and UID before any Redfish side +effect. Reconciliation returns after persisting this reference, then proceeds +only when the same session is Ready or Active. A replaced endpoint, Machine, or +session UID fails closed. + +```mermaid +stateDiagram-v2 + [*] --> Pending + Pending --> Preparing: session persisted + Preparing --> Ready: digests and endpoint ready + Ready --> Active: first target milestone + Ready --> RequestsRejected: expiresAt elapsed + Active --> RequestsRejected: expiresAt elapsed +``` + +Milestones are conditions on both the exact session and exact +`MachineOperation.status.targets[]` entry. Operation-wide conditions are not +used for target progress, preventing multi-target cross-talk and stale callback +completion. + +## Capabilities and Trust + +The controller and servers share a per-Site HMAC-SHA256 key. A capability binds +the session name, UID, expiration, and key ID. The bearer token is never stored +in Kubernetes status. Canonical routes are rooted at: + +```text +/v1/netboot/sessions/// +``` + +Allowlisted artifacts, installer callbacks, cloud-init reporting, logs, and TPM +attestation all use this identity. Server validation checks the current session +UID and expiration before resolving any file or Machine. Internal edge decision +routes additionally use TokenReview with audience `metalman-edge`. + +Trusted-LAN endpoints may use HTTP. Public endpoints require HTTPS because URL +capabilities are bearer credentials. Redfish and TPM identities retain their +existing trust-on-first-use pinning. + +## Artifact Availability + +Session artifacts resolve by OCI digest and architecture, not by mutable tag. +Static files use HTTP range semantics. The edge preserves client Range requests; +if a backend response truncates, it reconnects and requests exactly the missing +range from any healthy server replica. TFTP uses the same immutable backend and +resuming reader before reporting `BootLoaderDownloaded`. + +No RWX volume is required. Pod-local cache loss may increase latency but cannot +change the selected bits. Server readiness and retry behavior must distinguish a +cold cache from an invalid session. + +```mermaid +sequenceDiagram + participant F as Firmware/installer + participant E as Edge + participant A as Server A + participant B as Server B + + F->>E: GET capability/artifacts/disk.img.gz + E->>A: GET artifact + A-->>E: bytes 0..N (connection lost) + E->>B: GET artifact, Range: bytes=N+1- + B-->>E: 206 remaining bytes + E-->>F: continuous response +``` + +## Provisioning Flows + +### Traditional PXE + +DHCP identity is the NIC MAC. The server selects one unexpired Ready/Active +session for the endpoint and MAC, returning conflict on ambiguity. DHCP returns +a tokenized TFTP path. TFTP and all subsequent HTTP requests use the capability, +not source IP. A remote DHCP server requires a relay on the client LAN. + +### UEFI HTTP + +DHCP may supply the signed URL directly, or the controller may configure it +through Redfish. Redfish static NIC writes occur only for `networkMode: Static`. +HTTP boot can traverse routed and NAT paths because identity is in the URL. + +### First-node Bootstrap + +```mermaid +sequenceDiagram + participant CLI as kubectl unbounded + participant K as Kubernetes API + participant PF as Reconnecting port-forward + participant E as Local edge + participant H as First host + + CLI->>K: enable Metalman; wait controller/server + CLI->>K: create ExternalL2 endpoint; update designated Machine + CLI->>PF: forward loopback port to a Ready server pod + CLI->>E: start edge with projected edge token + CLI->>K: claim endpoint Ready + H->>E: DHCP/TFTP/HTTP provisioning + H->>K: kubelet creates designated Node + CLI->>K: observe designated Node Ready + CLI->>E: stop + CLI->>K: restore Machine endpoint; delete ephemeral resources +``` + +`kubectl unbounded site bootstrap-netboot` never runs Metalman controllers +locally. It reconnects its local-to-pod SPDY forward across server replacement +and stops automatically only for the Node selected by the requested Machine. + +With `--routed-cidr`, the command also starts the reusable unbounded-net node +agent as a temporary external gateway. It creates a synthetic unschedulable, +tainted Node, an External WireGuard GatewayPool, and a Site assignment. The +agent runs without kubelet, CNI, or status-transport assumptions and cleans host +networking on shutdown. WireGuard provides only L3 reachability for BMC and +downstream CIDRs; DHCP remains on the local L2. + +## Failure Semantics + +| Failure | Expected behavior | +|---------|-------------------| +| Controller leader loss | Durable operation/session references allow the new leader to resume before issuing another side effect. | +| Server pod deletion | Edge reconnects; range-capable transfers resume against another replica. | +| Mutable OCI tag changes | Existing sessions continue using their pinned digest. | +| Machine edit during provision | Existing session rendering and boot configuration remain unchanged. | +| Stale callback | Capability/session UID mismatch rejects it; sibling targets are untouched. | +| Endpoint replacement | UID mismatch blocks the session rather than using a new endpoint under the same name. | +| ManagedL2 pod/node loss | New sessions wait for endpoint readiness; active firmware connections may fail unless the L2 address itself is redundant. | +| Bootstrap CLI or edge exits | Ephemeral endpoint, Machine endpoint override, gateway resources, token files, port-forward, and child processes are cleaned. | + +## Deployment and RBAC + +- Controller and server have separate ServiceAccounts and least-role-specific + Kubernetes permissions. +- The edge ServiceAccount has no controller permissions; its internal requests + are authenticated by TokenReview. +- Controller and server mount the capability key. Only TLS-terminating edges + mount serving certificates. +- Controller/server use ordinary pod networking. ManagedL2 alone may use the + host network and required node placement. +- Synthetic external gateway Nodes are marked unschedulable, tainted, and + labeled so the normal unbounded-net DaemonSet excludes them. + +## Alternatives Rejected + +### Replicate the old monolith + +Nonleaders served from independent caches while only the leader reconciled +images, and every replica advertised a different node-local URL. Replication did +not provide a stable endpoint or coherent state. + +### Keep Metalman on every Site host network + +This preserves direct DHCP but creates a first-node scheduling dependency, +couples controllers to privileged ports, and makes ordinary rollouts disrupt +the advertised endpoint. + +### Shared RWX artifact cache + +Shared storage can reduce cold-cache latency but does not solve mutable request +identity, stale callbacks, or active connection loss. Digest snapshots and range +resume provide correctness without requiring a storage class. + +### Identify clients by source IP + +NAT, proxies, address reuse, and multiple interfaces make source addresses +ambiguous and forgeable. Session capabilities provide explicit operation-target +identity across every routed topology. + +### Run controllers in the kubectl process + +Local controller state would compete with the in-cluster deployment and require +broad persistent credentials. The bootstrap CLI instead runs only an edge and +uses a reconnecting tunnel to the in-cluster server. + +### Carry DHCP over WireGuard + +The existing dataplane is routed L3, not an Ethernet extension. A local edge or +DHCP relay is simpler and preserves the routing model. + +## Verification + +- API schema tests cover endpoint/session scope, immutability, and valid axes. +- MachineOperation tests prove session persistence before Redfish side effects, + immutable inputs, and target-scoped milestones. +- Server and edge tests cover capability validation, digest selection, Range + requests, truncated-backend resume, DHCP decisions, and tokenized TFTP. +- Operator tests cover split roles, no host networking on controller/server, + Services, PDBs, TLS rotation, endpoint workload types, and RBAC separation. +- CLI tests cover preflight-before-mutation, rollout waits, token rotation, + port-forward reconnection, exact-node handoff, early child exit, rollback, and + optional external gateway cleanup. +- Smoke suites exercise traditional PXE and Redfish HTTP boot through split + roles; the traditional suite deletes a server pod during provisioning. diff --git a/docs/content/concepts/bare-metal.md b/docs/content/concepts/bare-metal.md index 22d4b1e5d..5d54a0b2e 100644 --- a/docs/content/concepts/bare-metal.md +++ b/docs/content/concepts/bare-metal.md @@ -18,30 +18,30 @@ If your machines already have Linux installed and are reachable via SSH, use the ## How PXE Boot Works -PXE (Preboot Execution Environment) is a firmware feature that lets a machine -boot from the network instead of a local disk. metalman acts as the PXE -infrastructure: +PXE and UEFI HTTP boot let a machine start from the network instead of a local +disk. Metalman separates the Kubernetes control plane, replicated artifact +servers, and network-facing edges: ![PXE boot flow: Bare-Metal Machine boots via DHCP, TFTP, and HTTP from metalman, then joins the Kubernetes API with a bootstrap token](../../img/bare-metal-pxe-boot.svg) The boot flow in detail: -1. **DHCP Discovery** -- The machine's PXE firmware broadcasts a DHCP request. - metalman responds with an IP address and the location of the bootloader. +1. **Session creation** -- A HostReplace operation creates an immutable + NetbootSession containing the endpoint, resolved OCI digests, boot settings, + rendered inputs, and expiry. -2. **TFTP Boot** -- The firmware downloads the bootloader via TFTP. +2. **Firmware configuration** -- DHCP or Redfish supplies a capability-scoped + TFTP path or HTTP URL and, when selected, network configuration. -3. **HTTP Artifacts** -- The bootloader fetches the kernel, initramfs, and - configuration files from metalman's HTTP server. These are sourced from - the Machine's `spec.pxe.netbootImage`, or from Metalman's default netboot - image when that field is omitted. +3. **Boot artifacts** -- The edge obtains the bootloader, kernel, initramfs, and + configuration from replicated servers. Files come from the Machine's + `spec.host.netboot.netbootImage`, or Metalman's default netboot image. 4. **Kernel Boot** -- The machine boots into the downloaded kernel and initramfs. -5. **Token Retrieval** -- The init process contacts metalman's health endpoint - to retrieve a bootstrap token. If TPM 2.0 is available, the token is - encrypted to the machine's TPM. +5. **Token retrieval** -- The agent uses the authenticated session attestation + route. If TPM 2.0 is available, the token is encrypted to that machine's TPM. 6. **Cluster Join** -- kubelet uses the bootstrap token to join the Kubernetes cluster, just like an SSH-provisioned node. @@ -51,29 +51,31 @@ The boot flow in detail: ## Key Concepts -### DHCP Modes +### Endpoints -metalman supports two DHCP modes depending on your network topology: +`NetbootEndpoint` declares the stable client-facing address and edge placement: -- **Interface mode** (`--dhcp-interface eth0`) -- metalman listens for - broadcast DHCP on a specific NIC. Use this when metalman runs on the same - L2 segment as the bare-metal machines. +- **ManagedL2** -- The operator places a host-network edge on a selected node + attached to the provisioning LAN. +- **HTTP** -- The operator creates replicated HTTP edge pods and a Service. + Public endpoints require HTTPS. +- **ExternalL2** -- An edge outside the cluster owns DHCP/TFTP/private HTTP. + The first-node bootstrap command uses this type temporarily. -- **Relay mode** (default, no `--dhcp-interface`) -- metalman listens for - unicast DHCP forwarded by a DHCP relay agent. Use this when metalman and - the machines are on different subnets. +DHCP broadcasts require an edge on the target L2 or a local relay. WireGuard is +L3 only and does not extend the broadcast domain. ### OCI Images Metalman uses two OCI images during PXE provisioning: -- The machine image, referenced by `spec.pxe.image`, contains `/disk/disk.img.gz`. -- The netboot image, referenced by `spec.pxe.netbootImage` or by Metalman's +- The machine image, referenced by `spec.host.netboot.image`, contains `/disk/disk.img.gz`. +- The netboot image, referenced by `spec.host.netboot.netbootImage` or by Metalman's default, contains the reusable PXE boot environment. Netboot images are built `FROM scratch` and contain all files needed for PXE booting a machine under `/disk/`. Files with a `.tmpl` suffix are Go templates -rendered per-machine at serve time; other files are served verbatim. A +rendered from immutable session data; other files are served verbatim. A `metadata.yaml` file provides image-level configuration such as `dhcpBootImageName`. @@ -83,39 +85,42 @@ rendered per-machine at serve time; other files are served verbatim. A | **Templates** | Files with `.tmpl` suffix - rendered from Go templates with per-machine context (e.g., kernel command line) | | **Configuration** | `metadata.yaml` - image-level settings such as the DHCP boot filename | -Machine and netboot images are pulled and cached locally by the OCI reconciler. +Machine and netboot image tags are resolved to digests before the host is +powered on. Server caches are disposable; the durable session records the exact +digest and allowlisted files. ### Machine CRD (PXE Fields) For PXE-provisioned machines, the `Machine` resource includes: -- **`spec.pxe.image`** -- OCI machine image reference containing `/disk/disk.img.gz` +- **`spec.host.netboot.image`** -- OCI machine image reference containing `/disk/disk.img.gz` (e.g. `"ghcr.io/azure/host-ubuntu2404:v1"`). -- **`spec.pxe.architecture`** -- Optional target CPU architecture for PXE boot +- **`spec.host.netboot.architecture`** -- Optional target CPU architecture for boot artifacts and machine images. Defaults to `amd64`; allowed values are `amd64` and `arm64`. -- **`spec.pxe.netbootImage`** -- Optional OCI netboot image reference containing +- **`spec.host.netboot.netbootImage`** -- Optional OCI netboot image reference containing PXE boot artifacts. When omitted, Metalman uses its configured default `netboot` image. -- **`spec.pxe.dhcpLeases`** -- NIC specifications: MAC address and IP +- **`spec.host.netboot.endpointRef`** -- Required NetbootEndpoint name. +- **`spec.host.netboot.transport`**, **`configurationSource`**, and + **`networkMode`** -- Independent firmware transport, boot configuration + source, and network configuration mode. +- **`spec.host.netboot.dhcpLeases`** -- NIC specifications: MAC address and IP assignment for each interface. During install, the default netboot template passes the matching lease MAC to the initrd so it can select the provisioning NIC without relying on names such as `eth0`. -- **`spec.pxe.targetDisk`** -- Optional block device path for the disk that +- **`spec.host.netboot.targetDisk`** -- Optional block device path for the disk that receives the machine image. Set this on hosts with multiple disks; when omitted, the installer selects a disk automatically. -- **`spec.pxe.redfish`** -- Optional BMC connection details (endpoint, username, +- **`spec.host.netboot.redfish`** -- Optional BMC connection details (endpoint, username, password secret) for remote power management. -- **`spec.pxe.cloudInit`** -- Optional cloud-init customization. References a +- **`spec.host.netboot.cloudInit`** -- Optional cloud-init customization. References a ConfigMap containing user-data that is merged with the vendor-data managed by Unbounded. -### Site Isolation - -In environments with multiple metalman instances (e.g., different racks or -sites), the `--site` flag scopes each instance to machines labeled with -`unbounded-cloud.io/site=`. This prevents one metalman from interfering -with another's machines. +Supported combinations are TFTP+DHCP+DHCP, HTTP+DHCP+DHCP, +HTTP+Redfish+DHCP, and HTTP+Redfish+Static. Invalid combinations are rejected by +the API. ### TPM 2.0 Attestation @@ -131,8 +136,9 @@ metalman uses TPM 2.0 for secure bootstrap token delivery: 3. **AES-256-GCM** -- The actual token payload is encrypted with AES-256-GCM, with the key wrapped by the TPM credential. -This ensures that bootstrap tokens cannot be intercepted by other machines on -the network. +The attestation request is also bound to an expiring session capability. Public +HTTP boot endpoints require HTTPS so artifact and callback capabilities are not +exposed in transit. ### MachineOperation-Based Operations diff --git a/docs/content/guides/pxe.md b/docs/content/guides/pxe.md index fff8fac5f..fd7ab3309 100644 --- a/docs/content/guides/pxe.md +++ b/docs/content/guides/pxe.md @@ -1,271 +1,194 @@ --- -title: "Bare Metal (PXE)" +title: "Bare Metal Netboot" weight: 3 -description: "Netboot bare metal machines into your cluster." +description: "Provision bare-metal machines with resilient PXE or UEFI HTTP boot." --- ## Overview -Metalman is a controller that PXE-boots bare-metal servers and joins them to your Kubernetes cluster. It bundles DHCP, TFTP, and HTTP servers into a single binary, integrates with Redfish BMCs for remote power management, and uses TPM 2.0 attestation for secure bootstrap token delivery. +Metalman provisions bare-metal nodes from OCI images and joins them to a +Kubernetes cluster. Its controller, serving, and network-edge responsibilities +are separate so the normal control plane does not need host networking and a +server pod restart does not invalidate an in-flight provision. -API group: `unbounded-cloud.io/v1alpha3`. CRD: **Machine** (`mach`), cluster-scoped. +## Architecture -## Prerequisites - -- A Kubernetes cluster with access to `kubectl`. -- Bare-metal servers with UEFI PXE firmware and a BMC exposing a Redfish API. -- Layer-2 network connectivity (or a DHCP relay agent) between metalman and the PXE NICs. -- Network access from metalman to each BMC (HTTPS/443) and to the Kubernetes API (TCP/6443). -- Network access from target machines to metalman (UDP/67, UDP/69, TCP/8880) and to the Kubernetes API (TCP/6443). -- TPM 2.0 modules on target machines (required for secure attestation). - -## Deploy Metalman - -Metalman is a per-site component managed by the unbounded operator. First install -the operator (this also installs the CRDs): - -```bash -kubectl unbounded install -``` - -Then enable metalman on a Site by passing `--enable-metalman` to `site init`, or by -setting the metalman entry in `Site.spec.components`: - -```bash -kubectl unbounded site init --name my-edge-site --enable-metalman -``` +The operator deploys these per-Site resources after Metalman is enabled: -`kubectl unbounded site init` also runs `install` by default, so a fresh site only -needs the single command above. The operator then reconciles the `unbounded-system` -namespace, ServiceAccounts (`metalman-controller`, `metalman-bootstrap`), RBAC roles, -and a metalman Deployment for the Site. - -Key `serve-pxe` flags (baked into the operator-managed per-site Deployment; `--site` -scoping is inherent to the per-site component): - -| Flag | Default | Description | -|------|---------|-------------| -| `--dhcp-interface` | *(none - relay mode)* | NIC for broadcast DHCP | -| `--site` | *(none)* | Scope to machines with a specific site label | -| `--http-port` | 8880 | HTTP server port | -| `--cache-dir` | `~/.unbounded/metalman/cache` | Local cache for downloaded images | -| `--health-port` | 8081 | Health/readiness probe port | -| `--serve-url` | | External URL of this metalman instance | -| `--default-netboot-image` | release-matched `netboot` image | PXE boot environment used when `spec.pxe.netbootImage` is omitted | - -When `--dhcp-interface` is set, metalman binds to the interface for broadcast DHCP, and the DHCP server requires leader election. Without it, metalman accepts relayed (unicast) DHCP packets and the DHCP server responds regardless of leader status. Leader election always runs at the manager level for the reconcilers. - -## Images - -Metalman uses a machine image and a netboot image for each PXE repave. - -- `spec.pxe.image` is the machine image. It contains `/disk/disk.img.gz`, a - gzip-compressed raw disk image written to the target disk. -- `spec.pxe.architecture` selects the target architecture (`amd64` or `arm64`) - used when pulling machine and netboot image platform manifests. It defaults - to `amd64`. -- `spec.pxe.netbootImage` is the reusable PXE boot environment. It contains - bootloaders, kernel, initrd, templates, metadata, and `unbounded-agent`. If - omitted, Metalman uses the release-matched `--default-netboot-image`. -- `spec.pxe.bootProtocol` selects the network boot trigger. `PXE` is the - default and uses DHCP/TFTP bootfile options. `HTTP` uses Redfish UEFI HTTP - boot and requires a Redfish block. - -Both images are standard OCI container images built `FROM scratch` with -artifacts under `/disk/`. Files with a `.tmpl` suffix in the netboot image are -Go templates rendered per-machine at serve time; other files are served -verbatim. A `metadata.yaml` file in the netboot image provides image-level -configuration such as `dhcpBootImageName` and `httpBootPath`. - -Images are built, tagged, and pushed using standard container tooling: +- One leader-elected `metalman-controller-` Deployment for operations, + Redfish, OCI resolution, and immutable session creation. +- A two-replica `metalman-server-` Deployment and ClusterIP Service for + artifact serving, callbacks, and TPM attestation. +- A capability-signing Secret shared by the controller and servers. +- Edge workloads selected by `NetbootEndpoint` resources. -```bash -docker build -t ghcr.io/azure/host-ubuntu2404:v1 -f images/host-ubuntu2404/Containerfile . -docker build -t ghcr.io/azure/netboot:v1 -f images/netboot/Containerfile . -docker push ghcr.io/azure/host-ubuntu2404:v1 -docker push ghcr.io/azure/netboot:v1 -``` +Only a `ManagedL2` edge uses host networking. `HTTP` edges are replicated +ordinary pods. `ExternalL2` endpoints run outside the cluster, including the +temporary administrator-machine bootstrap flow. -Template context includes `.Machine`, `.ApiserverURL`, `.ServeURL`, `.KubernetesVersion`, and `.ClusterDNS`. +## Prerequisites -See the [CRD Reference]({{< relref "/reference/machina-crd" >}}) for the full Machine spec. +- An installed Unbounded operator and a Site with Metalman enabled. +- A target Machine with a UEFI-capable NIC and TPM 2.0. +- BMC connectivity from the controller when Redfish is configured. +- Target access to the endpoint and Kubernetes API. +- For TFTP: UDP 69 plus negotiated TFTP transfer ports. +- For DHCP: an edge on the target L2 or a DHCP relay. WireGuard does not carry + Ethernet broadcasts. +- For internet-facing HTTP boot: HTTPS with a trusted certificate. -## Machine CRD +## Define an Endpoint -A Machine represents a single bare-metal host. The `spec.pxe` section ties together the machine image, optional netboot image override, network config, and BMC credentials: +This endpoint places DHCP, TFTP, and HTTP on a selected provisioning node: ```yaml apiVersion: unbounded-cloud.io/v1alpha3 -kind: Machine +kind: NetbootEndpoint metadata: - name: server-01 - labels: - unbounded-cloud.io/site: rack-a + name: rack-a-l2 spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - architecture: amd64 - # Optional. Defaults to PXE. Set to HTTP for Redfish UEFI HTTP boot. - bootProtocol: PXE - # Optional. Omit to use Metalman's default netboot image. - netbootImage: ghcr.io/azure/netboot:v1 - # Optional. Recommended on hosts with multiple disks. - targetDisk: /dev/disk/by-id/example-os-disk - dhcpLeases: - - ipv4: "10.10.0.50" - mac: "aa:bb:cc:dd:ee:ff" - subnetMask: "255.255.255.0" - gateway: "10.10.0.1" - dns: ["8.8.8.8"] - redfish: - url: "https://bmc-01.example.com" - username: admin - passwordRef: - name: bmc-passwords - namespace: unbounded-system - key: bmc-01 + siteRef: rack-a + type: ManagedL2 + externalURL: http://10.20.0.2:8880 + trust: TrustedLAN + tls: + mode: Disabled + managedL2: + interface: eno1 + address: 10.20.0.2 + nodeSelector: + matchLabels: + provisioning.unbounded-cloud.io/rack: a ``` -Store BMC passwords in a Secret referenced by `passwordRef`. See the [CRD Reference]({{< relref "/reference/machina-crd" >}}) for all fields. +Use an `HTTP` endpoint for a Service-backed HTTP edge. Set `trust: Public`, an +`https://` external URL, and TLS mode `Secret` or `External` for public access. -## Cloud-Init Customization - -Cloud-init on PXE-booted machines uses two data sources that are merged at boot: - -- **Vendor-data** (managed by Unbounded) -- Contains the agent configuration, bootstrap scripts, and system defaults required for the node to join the cluster. This is not user-editable. -- **User-data** (managed by the cluster operator) -- Optional customization such as SSH keys, additional packages, or host-level configuration. - -When no user-data is configured, metalman serves a minimal `#cloud-config` document. To supply custom user-data, create a ConfigMap and reference it from the Machine spec: - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: my-cloud-init - namespace: unbounded-system -data: - user-data: | - #cloud-config - ssh_authorized_keys: - - ssh-rsa AAAA... - packages: - - vim - - htop -``` - -Then reference the ConfigMap in the Machine: +## Define a Machine ```yaml apiVersion: unbounded-cloud.io/v1alpha3 kind: Machine metadata: name: server-01 + labels: + unbounded-cloud.io/site: rack-a spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - dhcpLeases: - - ipv4: "10.10.0.50" - mac: "aa:bb:cc:dd:ee:ff" - subnetMask: "255.255.255.0" - gateway: "10.10.0.1" - dns: ["8.8.8.8"] - cloudInit: - userDataConfigMapRef: - name: my-cloud-init - namespace: unbounded-system + host: + netboot: + image: ghcr.io/azure/host-ubuntu2404:v1 + netbootImage: ghcr.io/azure/netboot:v1 + architecture: amd64 + endpointRef: rack-a-l2 + transport: TFTP + configurationSource: DHCP + networkMode: DHCP + targetDisk: /dev/disk/by-id/example-os-disk + dhcpLeases: + - mac: aa:bb:cc:dd:ee:ff + ipv4: 10.20.0.50 + subnetMask: 255.255.255.0 + gateway: 10.20.0.1 + dns: [10.20.0.1] + redfish: + url: https://bmc-01.example.com + username: admin + passwordRef: + name: bmc-passwords + namespace: unbounded-system + key: bmc-01 ``` -The `key` field defaults to `user-data` but can be overridden to select a different key from the ConfigMap. Both `data` and `binaryData` entries are supported. - -If the referenced ConfigMap does not exist, metalman falls back to the default minimal cloud-config. If the ConfigMap exists but the referenced key is not found, metalman returns an error and the machine will not receive user-data. - -## Boot Flow +The supported axes are independent: -1. **Repave requested.** A `HostReplace` `MachineOperation` targets the Machine. Metalman sets the boot override and force-restarts the server. For `bootProtocol: PXE`, it selects PXE boot. For `bootProtocol: HTTP`, it sets a Redfish UEFI HTTP boot URL from the netboot image metadata. -2. **Network boot.** In PXE mode, DHCP assigns the static IP by MAC, advertises the TFTP bootfile, and TFTP serves `shimx64.efi`. In HTTP mode, Metalman configures the first lease as a static Redfish EthernetInterface and the firmware downloads the Redfish-supplied URL from Metalman's HTTP server; DHCP is not required for this path. -3. **GRUB decision.** A rendered `grub.cfg` checks for an active `HostReplace` `MachineOperation` targeting the Machine. If a repave is requested, GRUB boots the PXE installer; otherwise it chainloads the local OS. When a Machine has multiple DHCP leases, metalman renders the lease matching the request source IP and passes that lease's MAC as `unbounded.boot_mac`. -4. **Installer (initrd overlay).** An init script in the initrd: - - Loads storage and network drivers, selects the provisioning NIC by MAC, and configures the static IP and DNS from kernel cmdline. - - Writes matching MAC-based static netplan configuration into the installed system before reboot and disables cloud-init network rendering so fallback DHCP configuration cannot conflict with it. The default netboot image also serves the selected lease as NoCloud `network-config`. - - Downloads the gzip-compressed raw disk image from the machine image over HTTP (retries up to 120 times). - - Writes the image to `spec.pxe.targetDisk` when set, otherwise to an automatically selected block device. - - Mounts the root filesystem and injects cloud-init config and the agent configuration. - - Calls `/pxe/disable` on metalman to signal completion, then reboots. -5. **First boot.** cloud-init downloads the `unbounded-agent` binary from metalman and runs `unbounded-agent start`. -6. **Node join.** The agent installs containerd and kubelet, performs TPM attestation to obtain a bootstrap token (see below), configures kubelet, and starts it. The node TLS-bootstraps and reaches `Ready`. +| Transport | Boot target source | Firmware network | +|-----------|--------------------|------------------| +| TFTP | DHCP | DHCP | +| HTTP | DHCP | DHCP | +| HTTP | Redfish | DHCP | +| HTTP | Redfish | Static | -## TPM Attestation +For HTTP with Redfish and static networking, the first DHCP lease supplies the +Redfish EthernetInterface address, gateway, DNS, and NIC identity. HTTP with +DHCP receives its capability boot URL through DHCP instead. -Metalman uses TPM 2.0 to securely deliver a bootstrap token without embedding secrets in the image. +## Provision -1. On first boot, the `unbounded-agent` creates a TPM Endorsement Key (EK) and Storage Root Key (SRK), then POSTs them to metalman's `/attest` endpoint. -{{< callout type="important" >}} -Metalman uses trust-on-first-use (TOFU) for TPM attestation: once a machine's EK public key is stored in `status.tpm.ekPublicKey`, any attestation from a different EK is rejected (HTTP 403). If a TPM is legitimately replaced, you must clear `status.tpm.ekPublicKey` from the Machine CR before the machine can re-enroll. -{{< /callout >}} +Create a HostReplace operation with the CLI: -3. Metalman wraps an AES-256 key via `tpm2.CreateCredential` (bound to the EK and SRK), then encrypts a 1-hour ServiceAccount token with AES-256-GCM and returns both to the client. -4. The agent uses the TPM `ActivateCredential` operation to recover the AES key, decrypts the token, and writes a bootstrap kubeconfig for kubelet to use during TLS bootstrapping. +```bash +kubectl unbounded machine replace server-01 --force +``` -The `metalman-bootstrap` ServiceAccount has RBAC for `system:node-bootstrapper` and `certificatesigningrequests:nodeclient` auto-approval. +The controller creates an immutable `NetbootSession` for that operation target. +The session pins OCI digests and all rendered inputs before the BMC is powered +on. Firmware, installer, cloud-init, and attestation use capability-scoped URLs. +Callbacks update only the matching operation target, so stale boots and +multi-target operations cannot satisfy each other's milestones. -## Site Isolation +Static artifacts support HTTP ranges. If an edge loses its backend server pod, +it reconnects and resumes the missing byte range against another replica. -Use the `--site` flag to scope a metalman instance to machines labeled `unbounded-cloud.io/site=`. Each site gets its own leader-election lease. +## Bootstrap the First Site Node -Run separate metalman instances for different racks or network segments: +When no Site node exists for a managed L2 edge, run an edge temporarily on an +administrator machine attached to that LAN: ```bash -# Instance for rack-a -metalman serve-pxe --site=rack-a --dhcp-interface=eth1 - -# Instance for rack-b -metalman serve-pxe --site=rack-b --dhcp-interface=eth2 +kubectl unbounded site bootstrap-netboot rack-a \ + --machine server-01 \ + --interface eno1 \ + --address 10.20.0.2 ``` -## Operations +The command runs only the edge data plane locally. Controllers remain in the +cluster. It creates an ephemeral endpoint, uses a reconnecting port-forward to +the server Service, and automatically stops after the designated Node becomes +Ready. This first-node policy intentionally supports one designated Machine. -Use `kubectl unbounded` to create and watch `MachineOperation` objects for day-2 actions: +For routed BMC or downstream LAN access, add repeatable CIDRs: ```bash -kubectl unbounded machine reboot server-01 -kubectl unbounded machine repave server-01 +kubectl unbounded site bootstrap-netboot rack-a \ + --machine server-01 \ + --interface eno1 \ + --address 10.20.0.2 \ + --routed-cidr 10.30.0.0/24 \ + --gateway-external-address 203.0.113.20 ``` -`machine reboot` creates a `HostReboot` operation. `machine repave` creates a -`HostReplace` operation. Metalman records progress on the operation target and -conditions, including `BootImageWritten` and `CloudInitDone`. - -## Troubleshooting +This starts the existing unbounded-net dataplane as an ephemeral external +gateway. It still provides L3 routing only; DHCP remains local to the LAN. -{{< callout type="warning" >}} -Running metalman's DHCP server on a network segment that already has an active DHCP server will cause conflicts. Ensure metalman is the only DHCP server on the PXE segment, or use relay mode to isolate DHCP traffic. -{{< /callout >}} +## Cloud-Init and Attestation -**Machine stuck in repaving.** Check the active `HostReplace` `MachineOperation` target stage and conditions. Verify the target machine can reach metalman on TCP/8880 and that the BMC is reachable from metalman. Metalman retries the Redfish boot request while the operation remains active. +Optional cloud-init user-data comes from +`spec.host.netboot.cloudInit.userDataConfigMapRef`. Metalman snapshots it into +the session, while vendor-data installs and configures `unbounded-agent`. -**DHCP not responding.** Confirm `--dhcp-interface` points to the correct NIC (broadcast mode) or that your relay agent forwards to metalman's DHCP port. Check that no other DHCP server is competing on the same segment. +The agent sends TPM EK/SRK material to the session's authenticated attestation +URL. Metalman uses TPM CreateCredential and AES-GCM to deliver a short-lived +bootstrap token. The TPM EK and Redfish certificate use trust on first use and +are pinned in Machine status. -**BMC connection failures.** Metalman uses TLS TOFU for Redfish - the first connection captures the BMC certificate fingerprint in `status.redfish.certFingerprint`. If a BMC certificate rotates, clear the fingerprint from the Machine status. Verify HTTPS/443 connectivity from metalman to the BMC. +## Troubleshooting -**TPM attestation rejected (403).** The EK public key has changed since the initial TOFU. If the TPM was legitimately replaced, clear `status.tpm.ekPublicKey` from the Machine CR to allow re-enrollment. +**Session remains Preparing.** Check endpoint `Ready`, controller/server +rollouts, registry reachability, and whether both OCI images resolve. -**Node not joining.** Verify the target machine can reach the Kubernetes API on TCP/6443. Check that the `metalman-bootstrap` ServiceAccount and RBAC are in place. Inspect kubelet logs on the target for certificate signing request errors. +**DHCP does not respond.** Confirm the ManagedL2 or external edge owns the +correct interface. For remote subnets, use a DHCP relay; do not expect a +WireGuard tunnel to forward broadcasts. -## Limitations +**Firmware gets 401 or 404.** Verify it is using the current session capability +URL. Capabilities are session-bound and expire with the session. -{{< callout type="note" >}} -Only Ubuntu 24.04 images are currently supported. The repave boot request timeout is fixed at 30 minutes. -{{< /callout >}} +**Transfer stops after pod deletion.** Check that another server replica is +Ready and the edge can reconnect. Immutable artifact requests are range +resumable, but firmware itself must retry if it loses its direct edge connection. -## See Also +**Attestation is rejected.** If the TPM was intentionally replaced, clear the +pinned EK only after validating the hardware change. -- **[Bare Metal Concepts]({{< relref "concepts/bare-metal" >}})** -- Deeper - explanation of PXE boot, DHCP modes, TPM attestation, and pool isolation. -- **[Project Overview]({{< relref "concepts/overview" >}})** -- How metalman - fits into the broader system. -- **[CRD Reference]({{< relref "reference/machina-crd" >}})** -- Complete - Machine API specification. -- **[Architecture]({{< relref "reference/architecture" >}})** -- Internal - design of the PXE provisioning pipeline. -- **[SSH Guide]({{< relref "guides/ssh" >}})** -- Alternative provisioning - path for machines with an existing OS. +See the [CRD reference]({{< relref "/reference/machina-crd" >}}) and +[CLI reference]({{< relref "/reference/cli" >}}) for all fields and flags. diff --git a/docs/content/reference/agent/configuration.md b/docs/content/reference/agent/configuration.md index da6cb0b49..66f435eb4 100644 --- a/docs/content/reference/agent/configuration.md +++ b/docs/content/reference/agent/configuration.md @@ -51,4 +51,4 @@ for command usage, exit behavior, and the current check list. | `Kubelet.Labels` | Key-value labels applied to the Node on registration. | | `Kubelet.RegisterWithTaints` | Taints applied to the Node on registration (`key=value:effect`). | | `OCIImage` | *(optional)* OCI image reference for the rootfs. Uses the built-in default image when empty. | -| `Attest.URL` | *(optional)* Base URL of a metalman serve-pxe instance for TPM attestation. | +| `Attest.URL` | *(optional)* Capability-scoped Metalman session URL for TPM attestation. | diff --git a/docs/content/reference/architecture.md b/docs/content/reference/architecture.md index ecfd244de..7c84fccf2 100644 --- a/docs/content/reference/architecture.md +++ b/docs/content/reference/architecture.md @@ -44,24 +44,27 @@ service host to a public FQDN. but the shipped ConfigMap (rendered from `deploy/machina/03-config.yaml.tmpl`) sets it to 50. `provisioningTimeout` defaults to 5 minutes. -### metalman -- Bare Metal PXE Controller +### metalman -- Bare Metal Netboot -Binary `cmd/metalman`, deployed as `metalman-controller` in `unbounded-system`. +Binary `cmd/metalman`, deployed in separate per-Site roles in +`unbounded-system`: -Runs three reconcilers and four network servers: +| Role | Responsibility | +|------|----------------| +| `metalman controller` | Leader-elected MachineOperation and Redfish reconciliation, OCI resolution, and immutable NetbootSession creation. | +| `metalman server` | Replicated, Service-backed artifact serving, callbacks, TPM attestation, and authenticated edge decisions. | +| `metalman edge` | DHCP/TFTP on a provisioning LAN and HTTP proxying. It has no controller credentials. | -| Reconciler / Server | Role | -|-------------------------|-------------------------------------------------------------| -| OCIReconciler | Pulls and caches OCI netboot images from container registries. | -| Redfish Reconciler | TOFU TLS cert pinning for BMC Redfish endpoints. | -| MachineOperation Reconciler | BMC power control, boot override, reboot, and repave operations via Redfish REST. | -| DHCP server (UDP/67) | Static IP assignment by MAC address. | -| TFTP server (UDP/69) | Bootloader delivery. | -| HTTP server (TCP/8880) | Kernel, initrd, Go-templated configs, `/attest` (TPM), `/pxe/disable`. | -| Health server (TCP/8081)| Liveness and readiness probes. | +The operator deploys one controller, two server replicas, a server Service and +PodDisruptionBudget, and a shared capability-signing Secret. Controller and +server pods use ordinary pod networking. `NetbootEndpoint` resources determine +edge placement; only a `ManagedL2` edge uses host networking. -Site-based scoping: the `--site` flag and the `unbounded-cloud.io/site` label -restrict each instance to a subset of Machines. Leader election is per-site. +Each HostReplace target receives an immutable `NetbootSession` that pins the +endpoint, OCI digests, artifact allowlist, rendered inputs, and expiry. HMAC +capability URLs identify the exact session for HTTP, TFTP, callbacks, and +attestation. Static HTTP transfers support ranges so an edge can reconnect to a +different server replica and resume an interrupted download. ### kubectl-unbounded -- CLI Plugin @@ -71,6 +74,7 @@ Binary `cmd/kubectl-unbounded`. Provides subcommands: |--------------------|---------| | `install` | Bootstraps CRDs and `unbounded-operator`; component workloads are reconciled from `Site.spec.components`. | | `site init` | Initializes a new site by bootstrapping Unbounded when needed, creating site resources, and creating the bootstrap token. | +| `site bootstrap-netboot` | Runs a temporary local netboot edge until one designated first Site Node is Ready. | | `machine register` | Registers a machine to a site, creating a `Machine` CR with auto-discovery of SSH secrets and bootstrap tokens. | ### inventory -- Hardware Collector @@ -92,7 +96,7 @@ Represents a host and drives its lifecycle. | Spec field | Description | |-----------------------|-------------| | `spec.ssh` | SSH connectivity (host, port, user, privateKeyRef) and optional bastion config. | -| `spec.pxe` | PXE config: machine image reference, optional netboot image override, dhcpLeases, redfish settings. | +| `spec.host.netboot` | Netboot image, endpoint, independent transport/configuration/network axes, DHCP leases, and Redfish settings. | | `spec.kubernetes` | Kubernetes version, bootstrapTokenRef, nodeRef, nodeLabels. | Status includes phase, message, conditions, SSH fingerprint, Redfish cert @@ -103,15 +107,17 @@ fingerprint, and TPM info. The API defines condition type constants including ### Netboot OCI Images -Metalman uses two OCI images for PXE repaves. `Machine.spec.pxe.image` references -the machine image containing `/disk/disk.img.gz`. `Machine.spec.pxe.netbootImage` -optionally references the reusable PXE boot environment; when omitted, Metalman -uses its configured default `netboot` image. `Machine.spec.pxe.architecture` +Metalman uses two OCI images for netboot repaves. +`Machine.spec.host.netboot.image` references the machine image containing +`/disk/disk.img.gz`. `Machine.spec.host.netboot.netbootImage` optionally +references the reusable boot environment; when omitted, Metalman uses its +configured default netboot image. `Machine.spec.host.netboot.architecture` selects the OCI platform manifest for both images and defaults to `amd64`. -Netboot images contain all files needed for PXE booting under `/disk/`. Files -with a `.tmpl` suffix are Go templates rendered per-machine at serve time. A -`metadata.yaml` provides image-level configuration such as `dhcpBootImageName`. +Netboot images contain all files needed for network booting under `/disk/`. +Files with a `.tmpl` suffix are Go templates rendered from the immutable session +snapshot. A `metadata.yaml` provides image-level configuration such as the TFTP +or HTTP firmware artifact path. ### Resource relationships @@ -153,18 +159,19 @@ For a walkthrough, see the [SSH Provisioning Guide]({{< ref "guides/ssh" >}}). ### PXE Path (metalman) -1. `Machine` CR created with `spec.pxe`. -2. A `HostReplace` `MachineOperation` requests a repave; metalman sets the PXE or HTTP boot override and force-restarts the host through Redfish. -3. Host PXE-boots: DHCP (IP + boot filename) -> TFTP (bootloader) -> HTTP - (kernel, initrd, configs). -4. Init script: writes disk image, injects configs, calls `/pxe/disable`, - reboots. -5. Cloud-init: installs containerd, kubelet, tpm2-tools. -6. TPM attestation: TOFU Endorsement Key pinning, - `MakeCredential`/`ActivateCredential` exchange, AES-256-GCM encrypted - bootstrap token delivered via `/attest`. -7. kubelet TLS-bootstraps into the cluster. -8. Subsequent reboots: GRUB chainloads the local OS (no PXE). +1. A `Machine` selects a `NetbootEndpoint` through `spec.host.netboot`. +2. A `HostReplace` operation creates an immutable session and waits for its + endpoint and digest-addressed artifacts to become ready. +3. Metalman configures TFTP or UEFI HTTP boot through DHCP or Redfish, according + to the Machine's independent boot axes, and restarts the host. +4. Firmware and the installer fetch capability-scoped artifacts through an edge. +5. The installer writes the disk image and posts the exact target's + `BootImageWritten` callback before rebooting. +6. Cloud-init installs the agent and reports target-scoped progress. +7. TPM attestation uses TOFU Endorsement Key pinning, + `MakeCredential`/`ActivateCredential` exchange, and an AES-256-GCM encrypted + bootstrap token delivered through the authenticated session route. +8. kubelet TLS-bootstraps into the cluster; subsequent boots chainload the local OS. For a walkthrough, see the [PXE Provisioning Guide]({{< ref "guides/pxe" >}}). @@ -177,8 +184,9 @@ For a walkthrough, see the [PXE Provisioning Guide]({{< ref "guides/pxe" >}}). | Bootstrap tokens | Standard kubeadm tokens (`token-id` + `token-secret`). SSH path passes as env var; PXE path encrypts via TPM. | | TPM attestation | TOFU EK pinning. AES-256-GCM encrypted service-account tokens with 1-hour expiry. | | Redfish TLS | TOFU cert fingerprint pinning stored in `status.redfish.certFingerprint`. | +| Netboot requests | Expiring HMAC capabilities bound to one immutable session; public endpoints require HTTPS. | | RBAC | Separate ServiceAccounts, Roles, and ClusterRoles per controller. | -| Secret access | Via Kubernetes API only; never mounted as volumes. | +| Secret access | Scoped API reads; capability and TLS keys mount only into roles that need them. | ## Deployment @@ -195,8 +203,9 @@ applied by hand. | `deploy/machina/crd/` | `Machine` CRD definition. | | `deploy/machina/` | Namespace, RBAC (machina + metalman), ConfigMap, Deployment, Service. | -Resource defaults for both controllers: 100m CPU / 128Mi memory requests, -500m CPU / 256Mi memory limits. Both tolerate `CriticalAddonsOnly`. +Metalman controller and server workloads request 100m CPU and 128Mi memory and +limit themselves to 2 CPU and 2Gi memory. Server replicas use topology spread, +readiness/liveness probes, zero-unavailable rollouts, and a PodDisruptionBudget. Container images are multi-stage builds on Azure Linux 3.0, built with `podman`. CRDs are generated with `controller-gen` v0.20.1. diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index 3ba70474e..98fee7ccd 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -149,6 +149,50 @@ kubectl unbounded site init \ --- +### `kubectl unbounded site bootstrap-netboot` + +Run a temporary netboot edge on the administrator machine until one designated +Site Node becomes Ready. The command leaves controllers in the cluster, enables +the Site Metalman component, waits for the controller and server rollouts, +creates an ephemeral ExternalL2 endpoint, and connects the local edge to a +server pod through a reconnecting port-forward. + +```bash +kubectl unbounded site bootstrap-netboot rack-a \ + --machine server-01 \ + --interface eno1 \ + --address 10.20.0.2 +``` + +Required flags: + +| Flag | Description | +|------|-------------| +| `--machine` | Machine whose corresponding Node readiness completes bootstrap | +| `--interface` | Administrator-machine interface attached to the provisioning L2 | +| `--address` | IPv4 address on that interface, advertised to netboot clients | + +Optional flags: + +| Flag | Default | Description | +|------|---------|-------------| +| `--endpoint-name` | `bootstrap-` | Ephemeral NetbootEndpoint name | +| `--http-port` | `8880` | Local HTTP artifact port | +| `--namespace` | `unbounded-system` | Namespace containing Metalman workloads | +| `--metalman-binary` | `metalman` on PATH | Edge binary to execute | +| `--timeout` | `30m` | Maximum wait for the designated Node | +| `--routed-cidr` | none | CIDR routed through an ephemeral external gateway; repeatable | +| `--gateway-external-address` | `--address` | WireGuard endpoint address reachable by remote peers | +| `--kubeconfig` | standard lookup | Path to kubeconfig | + +The command restores the Machine's previous endpoint and deletes its ephemeral +resources during cleanup. It intentionally stops after the one designated Node +is Ready. Using `--routed-cidr` starts the unbounded-net dataplane and requires +root networking privileges. The gateway provides L3 routing, not DHCP broadcast +extension. + +--- + ### `kubectl unbounded machine` Manage Unbounded machines. diff --git a/docs/content/reference/machina-crd.md b/docs/content/reference/machina-crd.md index 9f3f90527..6ef95631b 100644 --- a/docs/content/reference/machina-crd.md +++ b/docs/content/reference/machina-crd.md @@ -56,7 +56,10 @@ top-level `spec.pxe` remains a deprecated fallback for existing Machines. | `host.netboot.image` | string | Yes | - | OCI machine image reference containing `/disk/disk.img.gz` (e.g. `"ghcr.io/azure/host-ubuntu2404:v1"`). | | `host.netboot.architecture` | string | No | `amd64` | Target CPU architecture for PXE boot artifacts and machine images. Allowed values: `amd64`, `arm64`. | | `host.netboot.netbootImage` | string | No | Metalman default | OCI netboot image reference containing PXE boot artifacts. | -| `host.netboot.bootProtocol` | string | No | `PXE` | Network boot trigger protocol for repaves. `PXE` uses DHCP/TFTP bootfile options. `HTTP` uses Redfish UEFI HTTP boot with a URL derived from the netboot image metadata. Allowed values: `PXE`, `HTTP`. | +| `host.netboot.transport` | string | No | `TFTP` | Firmware artifact transport. Allowed values: `TFTP`, `HTTP`. | +| `host.netboot.configurationSource` | string | No | `DHCP` | Source of the firmware boot target. Allowed values: `DHCP`, `Redfish`. | +| `host.netboot.networkMode` | string | No | `DHCP` | Firmware provisioning network mode. Allowed values: `DHCP`, `Static`. Static requires Redfish. | +| `host.netboot.endpointRef` | string | Yes | - | Name of the `NetbootEndpoint` serving this Machine. | | `host.netboot.dhcpLeases` | []DHCPLease | No | - | Provisioning network settings. They are served as static DHCP leases during PXE boot and used for Redfish firmware, installer, NoCloud, and installed-system static configuration during HTTP boot. | | `host.netboot.dhcpLeases[].ipv4` | string | Yes | - | Static IPv4 address to assign. | | `host.netboot.dhcpLeases[].mac` | string | Yes | - | NIC MAC address (matched case-insensitively). | @@ -170,6 +173,55 @@ spec: name: remote-oci-auth ``` +## NetbootEndpoint + +Cluster-scoped `NetbootEndpoint` resources declare stable client-facing +netboot addresses. A Machine references one by name through +`spec.host.netboot.endpointRef`. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `spec.siteRef` | string | Yes | Site whose Machines may use the endpoint. | +| `spec.type` | string | Yes | `ManagedL2`, `ExternalL2`, or `HTTP`. | +| `spec.externalURL` | string | Yes | Stable HTTP(S) base URL snapshotted into each session. | +| `spec.tls.trust` | string | Yes | `TrustedLAN` or `Public`. Public endpoints require HTTPS. | +| `spec.tls.mode` | string | Yes | `Disabled`, `Secret`, or `External`. | +| `spec.tls.secretRef` | Secret reference | For Secret mode | Namespaced TLS Secret copied to the managed edge. | +| `spec.managedL2.nodeSelector` | LabelSelector | For ManagedL2 | Selects nodes attached to the provisioning network. | +| `spec.managedL2.interface` | string | For ManagedL2 | Host interface used for DHCP and TFTP. | +| `spec.managedL2.address` | string | For ManagedL2 | Stable edge address on the provisioning network. | +| `spec.http.serviceType` | string | For HTTP | `ClusterIP`, `NodePort`, or `LoadBalancer`; defaults to `ClusterIP`. | + +`ManagedL2` creates a host-network edge. `HTTP` creates replicated HTTP edge +pods and a Service. `ExternalL2` creates no in-cluster workload; an external +process claims it through status. + +Status records the processed generation, current external claimant and renewal +time, and conditions. A session becomes Ready only after endpoint +`status.observedGeneration` matches its generation and `Ready=True`. + +## NetbootSession + +Cluster-scoped `NetbootSession` is the immutable provisioning contract for one +MachineOperation target. Metalman creates it automatically; users should not +edit or reuse sessions. + +| Field | Description | +|-------|-------------| +| `spec.machine` | Exact Machine name, UID, and generation. | +| `spec.operation` | Exact MachineOperation name, UID, and generation. | +| `spec.endpoint` | Endpoint name, UID, and external URL snapshot. | +| `spec.boot` | Transport, configuration source, network mode, firmware artifact, architecture, leases, and target disk. | +| `spec.provisioning` | Cluster, Kubernetes, agent, provider-label, and resolved cloud-init inputs used for rendering. | +| `spec.artifacts` | Machine/netboot OCI references pinned to SHA-256 digests and the allowed public file names. | +| `spec.expiresAt` | Last time authenticated session requests are accepted. | + +Session spec is immutable through CRD validation. Status contains the +preparation phase, signing-key identifier, endpoint readiness, and exact-target +milestones such as `BootLoaderDownloaded`, `BootImageWritten`, +`CloudInitDone`, and `Attested`. The bearer capability itself is never stored +in Kubernetes. + ## MachineOperation | Property | Value | @@ -193,8 +245,8 @@ spec: | `status.message` | string | No | Human-readable status message. | | `status.startedAt` | time | No | Operation start timestamp. | | `status.completedAt` | time | No | Terminal phase timestamp. | -| `status.targets` | []TargetStatus | No | Per-Machine target status snapshot used by host operation controllers. | -| `status.conditions` | []Condition | No | Operation conditions. `Completed` tracks terminal state. `BootLoaderDownloaded=True` is latched by metalman when a target first downloads the initial PXE boot loader, usually over TFTP. `BootImageWritten` starts as `Unknown` for metalman `HostReplace`, transitions to `False` when the PXE installer requests `disk.img.gz`, and transitions to `True` when the existing `/pxe/disable` completion signal is received. `CloudInitDone` starts as `Unknown`, transitions to `False` when first-boot cloud-init starts, and transitions to `True` on final cloud-init success or `False` with reason `Failed` and a summarized error when cloud-init reports a failure. | +| `status.targets` | []TargetStatus | No | Per-Machine target status, immutable input, session reference, attempts, and target-scoped conditions. Metalman records `BootLoaderDownloaded`, `BootImageWritten`, and `CloudInitDone` on the exact target. | +| `status.conditions` | []Condition | No | Operation-wide conditions such as terminal `Completed`. Provisioning milestones are target-scoped. | `AgentUpgrade` is handled by the in-host agent and requires `spec.parameters.downloadURL`. The URL must point to an `unbounded-agent` release tarball; the agent stages it as the inactive blue/green daemon binary, records the previous binary as last known good, and restarts `unbounded-agent-daemon.service`. If systemd cannot keep the upgraded daemon running, `unbounded-agent-daemon-recovery.service` switches the daemon back to the last known good binary. @@ -456,7 +508,7 @@ Both images are standard OCI container images built `FROM scratch` with artifact under `/disk/`. This follows the kubevirt containerDisk convention. Files with a `.tmpl` suffix in the netboot image are Go templates rendered -per-machine at serve time; other files are served verbatim. A `metadata.yaml` +from the immutable NetbootSession snapshot; other files are served verbatim. A `metadata.yaml` file in the netboot image provides image-level configuration such as `dhcpBootImageName` and `httpBootPath`. @@ -470,10 +522,11 @@ Templates receive the following data object: | Field | Type | Description | |-------|------|-------------| -| `.Machine` | *Machine | The Machine CR that initiated the request. | -| `.BootLease` | *DHCPLease | The DHCP lease matching the request source IP, or the first lease when no match is available. Netboot templates use this to pass the provisioning NIC MAC, static IP, gateway, and DNS to the installer and NoCloud network configuration. | +| `.Machine` | *Machine | A synthetic Machine built from the immutable session snapshot. | +| `.BootLease` | *DHCPLease | The snapshotted provisioning lease selected for rendering. | | `.ApiserverURL` | string | External Kubernetes API server URL. | -| `.ServeURL` | string | External metalman HTTP URL. | +| `.ArtifactBaseURL` | string | Capability-scoped session artifact URL. | +| `.ServeURL` | string | Capability-scoped session base URL used by callbacks. | | `.KubernetesVersion` | string | Resolved Kubernetes version for the machine. | | `.ClusterDNS` | string | Cluster DNS service IP. | @@ -505,11 +558,10 @@ httpBootPath: shimx64.efi ``` The `dhcpBootImageName` field specifies the boot filename included in DHCP -responses (option 67) for `spec.host.netboot.bootProtocol: PXE`. +responses for `transport: TFTP`. -The `httpBootPath` field specifies the file path, relative to metalman's HTTP -artifact server, used for `spec.host.netboot.bootProtocol: HTTP`. If `httpBootPath` is -omitted, metalman falls back to `dhcpBootImageName` for the UEFI HTTP boot URL. +The `httpBootPath` field specifies the firmware artifact used for +`transport: HTTP`. Metalman signs a session capability URL for that artifact. --- From 175cd7fe279220e913977b604c05d2c594e0dd7a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 20:17:40 +0000 Subject: [PATCH 42/43] operator: wait for netboot CRDs --- internal/operator/bootstrap.go | 2 ++ internal/operator/bootstrap_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/internal/operator/bootstrap.go b/internal/operator/bootstrap.go index 0dcb71109..60cf8d439 100644 --- a/internal/operator/bootstrap.go +++ b/internal/operator/bootstrap.go @@ -53,6 +53,8 @@ var RequiredCRDNames = [...]string{ "machineconfigurations.unbounded-cloud.io", "machineoperationcredentials.unbounded-cloud.io", "machineconfigurationversions.unbounded-cloud.io", + "netbootendpoints.unbounded-cloud.io", + "netbootsessions.unbounded-cloud.io", "sitenodeslices.net.unbounded-cloud.io", "gatewaypools.net.unbounded-cloud.io", "gatewaypoolnodes.net.unbounded-cloud.io", diff --git a/internal/operator/bootstrap_test.go b/internal/operator/bootstrap_test.go index 30f3d1eee..15e160e6f 100644 --- a/internal/operator/bootstrap_test.go +++ b/internal/operator/bootstrap_test.go @@ -364,6 +364,8 @@ func TestRequiredCRDNames(t *testing.T) { "machineconfigurations.unbounded-cloud.io", "machineoperationcredentials.unbounded-cloud.io", "machineconfigurationversions.unbounded-cloud.io", + "netbootendpoints.unbounded-cloud.io", + "netbootsessions.unbounded-cloud.io", "sitenodeslices.net.unbounded-cloud.io", "gatewaypools.net.unbounded-cloud.io", "gatewaypoolnodes.net.unbounded-cloud.io", From 8c5b392fb98f05358a3e7adb46403d97ab162fdb Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Wed, 22 Jul 2026 20:17:49 +0000 Subject: [PATCH 43/43] metalman: satisfy verification checks --- .../v1alpha3/netboot_validation_test.go | 7 + api/machina/v1alpha3/site_types_test.go | 3 + .../app/site_bootstrap_netboot.go | 186 ++++++++++++++---- .../app/site_bootstrap_netboot_test.go | 33 ++++ cmd/unbounded-net-node/main.go | 1 + hack/smoke_metalman_contract_test.go | 5 + .../metalman/attestation/attestation_test.go | 2 + internal/metalman/commands/edge.go | 96 ++++++++- internal/metalman/commands/roles_test.go | 18 ++ internal/metalman/commands/serve_pxe.go | 15 ++ internal/metalman/dhcp/backend.go | 11 ++ internal/metalman/dhcp/backend_test.go | 13 ++ internal/metalman/dhcp/dhcp.go | 5 + internal/metalman/dhcp/dhcp_test.go | 7 + internal/metalman/machineops/controller.go | 10 + .../metalman/machineops/controller_test.go | 5 + .../metalman/machineops/session_manager.go | 16 +- .../machineops/session_manager_test.go | 3 + .../metalman/machineops/session_status.go | 10 + internal/metalman/netboot/capability.go | 7 + internal/metalman/netboot/edge_auth.go | 2 + internal/metalman/netboot/edge_auth_test.go | 1 + internal/metalman/netboot/netboot.go | 3 + internal/metalman/netboot/oci_cache.go | 1 + .../metalman/netboot/session_dhcp_test.go | 11 ++ internal/metalman/netboot/session_http.go | 58 ++++++ .../metalman/netboot/session_http_test.go | 18 ++ internal/metalman/netboot/tftp.go | 5 + internal/metalman/netboot/tftp_backend.go | 16 ++ .../metalman/netboot/tftp_backend_test.go | 23 +++ internal/net/nodeagent/external.go | 4 + .../nodeagent/site_watch_reconcile_test.go | 3 + .../operator/components/metalman/metalman.go | 54 +++++ .../components/metalman/metalman_test.go | 109 ++++++---- internal/operator/migrate_test.go | 1 + 35 files changed, 672 insertions(+), 90 deletions(-) diff --git a/api/machina/v1alpha3/netboot_validation_test.go b/api/machina/v1alpha3/netboot_validation_test.go index dea7545a8..48f2e330f 100644 --- a/api/machina/v1alpha3/netboot_validation_test.go +++ b/api/machina/v1alpha3/netboot_validation_test.go @@ -25,6 +25,7 @@ func TestNetbootEndpointSchema(t *testing.T) { t.Errorf("endpoint spec is missing %q", field) } } + assertSchemaValidations(t, spec, map[string]string{ "self.tls.trust != 'Public' || (self.externalURL.startsWith('https://') && self.tls.mode != 'Disabled')": "public endpoints require HTTPS", "self.type == 'ManagedL2' ? has(self.managedL2) : !has(self.managedL2)": "managedL2 configuration must be set only for ManagedL2 endpoints", @@ -56,23 +57,28 @@ func TestNetbootSessionSchema(t *testing.T) { t.Errorf("session spec is missing %q", field) } } + assertSchemaValidations(t, spec, map[string]string{ "self == oldSelf": "netboot session spec is immutable", }) boot := spec.Properties["boot"] + firmware, ok := boot.Properties["firmwareArtifact"] if !ok { t.Fatal("session boot snapshot is missing firmwareArtifact") } + if firmware.MinLength == nil || *firmware.MinLength != 1 { t.Error("session boot firmwareArtifact must be non-empty") } + provisioning := spec.Properties["provisioning"] for _, field := range []string{"cluster", "kubernetes", "agent", "providerLabels", "userData"} { if _, ok := provisioning.Properties[field]; !ok { t.Errorf("session provisioning snapshot is missing %q", field) } } + artifactSource := spec.Properties["artifacts"].Properties["files"].Items.Schema.Properties["source"] requireEnumValue(t, artifactSource.Enum, "Session") @@ -101,6 +107,7 @@ func TestMachineOperationTargetInputHasNetbootSessionRef(t *testing.T) { t.Parallel() crd := readCRD(t, "../../../deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml") + input := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"]. Properties["targets"].Items.Schema.Properties["input"] if _, ok := input.Properties["netbootSessionRef"]; !ok { diff --git a/api/machina/v1alpha3/site_types_test.go b/api/machina/v1alpha3/site_types_test.go index 07c6318d8..bd4aa26c9 100644 --- a/api/machina/v1alpha3/site_types_test.go +++ b/api/machina/v1alpha3/site_types_test.go @@ -23,14 +23,17 @@ func TestMetalmanComponentSchemaExposesOnlyEnablement(t *testing.T) { if err != nil { t.Fatalf("read Site CRD: %v", err) } + var crd apiextensionsv1.CustomResourceDefinition if err := yaml.Unmarshal(data, &crd); err != nil { t.Fatalf("parse Site CRD: %v", err) } + metalman := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"].Properties["components"].Properties["metalman"] if _, ok := metalman.Properties["enabled"]; !ok { t.Error("Metalman component schema lacks enabled") } + for _, field := range []string{"dhcpAutoInterface", "replicas"} { if _, ok := metalman.Properties[field]; ok { t.Errorf("Metalman component schema still exposes %q", field) diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go index 5d9a748aa..5df145dd0 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net" "net/http" "net/url" @@ -140,9 +141,11 @@ func (p *commandBootstrapEdgeProcess) Err() error { func (p *commandBootstrapEdgeProcess) Stop(ctx context.Context) error { var signalErr error + p.stopOnce.Do(func() { signalErr = p.cmd.Process.Signal(syscall.SIGTERM) }) + if signalErr != nil && !errors.Is(signalErr, os.ErrProcessDone) { return fmt.Errorf("stop Metalman edge: %w", signalErr) } @@ -159,6 +162,7 @@ func (p *commandBootstrapEdgeProcess) Stop(ctx context.Context) error { if err := p.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { return fmt.Errorf("kill Metalman edge: %w", err) } + <-p.done return ctx.Err() @@ -176,6 +180,7 @@ func (p *embeddedBootstrapGatewayProcess) Err() error { func (p *embeddedBootstrapGatewayProcess) Stop(ctx context.Context) error { p.once.Do(p.cancel) + select { case <-p.done: return p.Err() @@ -190,6 +195,7 @@ func (t *bootstrapEdgeToken) Path() string { func (t *bootstrapEdgeToken) Close() error { var err error + t.once.Do(func() { t.cancel() <-t.done @@ -238,9 +244,17 @@ func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Co cmd.Flags().DurationVar(&handler.timeout, "timeout", defaultBootstrapNetbootTimeout, "Maximum time to wait for the designated Node to become Ready") cmd.Flags().StringSliceVar(&handler.routedCIDRs, "routed-cidr", nil, "CIDR routed through an ephemeral external gateway (repeatable)") - _ = cmd.MarkFlagRequired("machine") - _ = cmd.MarkFlagRequired("interface") - _ = cmd.MarkFlagRequired("address") + if err := cmd.MarkFlagRequired("machine"); err != nil { + panic(fmt.Sprintf("mark machine flag required: %v", err)) + } + + if err := cmd.MarkFlagRequired("interface"); err != nil { + panic(fmt.Sprintf("mark interface flag required: %v", err)) + } + + if err := cmd.MarkFlagRequired("address"); err != nil { + panic(fmt.Sprintf("mark address flag required: %v", err)) + } return cmd } @@ -249,9 +263,11 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if h.timeout <= 0 { h.timeout = defaultBootstrapNetbootTimeout } + if h.endpointName == "" { h.endpointName = "bootstrap-" + h.machine } + if h.gatewayExternalAddress == "" { h.gatewayExternalAddress = h.address } @@ -259,27 +275,35 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if err := h.initializeDependencies(); err != nil { return err } + binary, err := h.dependencies.resolveBinary() if err != nil { return err } + if err := h.dependencies.preflightNetwork(); err != nil { return err } + var gatewayRuntimeDir string + if len(h.routedCIDRs) > 0 { if err := h.dependencies.preflightGateway(); err != nil { return err } + gatewayRuntimeDir, err = h.dependencies.gatewayRuntimeDir() if err != nil { return fmt.Errorf("create external gateway runtime directory: %w", err) } + defer func() { retErr = errors.Join(retErr, os.RemoveAll(gatewayRuntimeDir)) }() } + if err := h.initializeClients(); err != nil { return err } + h.initializeRuntimeDependencies() runCtx, cancel := context.WithTimeout(ctx, h.timeout) @@ -289,7 +313,9 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if err != nil { return err } + resourcesPrepared := true + var ( forward *bootstrapPortForward token *bootstrapEdgeToken @@ -297,6 +323,7 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error gatewayProcess bootstrapEdgeProcess gatewayPrepared bool ) + defer func() { cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cleanupCancel() @@ -304,18 +331,23 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if resourcesPrepared { retErr = errors.Join(retErr, h.restoreClusterResources(cleanupCtx, state)) } + if process != nil { retErr = errors.Join(retErr, process.Stop(cleanupCtx)) } + if gatewayProcess != nil { retErr = errors.Join(retErr, gatewayProcess.Stop(cleanupCtx)) } + if forward != nil { retErr = errors.Join(retErr, forward.Close()) } + if token != nil { retErr = errors.Join(retErr, token.Close()) } + if gatewayPrepared { retErr = errors.Join(retErr, h.cleanupGatewayResources(cleanupCtx)) } @@ -325,13 +357,16 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if err := h.prepareGatewayResources(runCtx); err != nil { return err } + gatewayPrepared = true + gatewayProcess, err = h.dependencies.startGateway(runCtx, nodeagent.ExternalGatewayOptions{ NodeName: h.endpointName, RuntimeDir: gatewayRuntimeDir, RESTConfig: h.restConfig, }) if err != nil { return fmt.Errorf("start external gateway dataplane: %w", err) } + if err := h.waitForGatewayReady(runCtx, gatewayProcess); err != nil { return err } @@ -340,25 +375,31 @@ func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error if err := h.waitForMetalman(runCtx); err != nil { return err } + localPort, err := h.dependencies.localPort() if err != nil { return fmt.Errorf("select local port for Metalman server: %w", err) } + forward, err = h.dependencies.portForward(runCtx, localPort) if err != nil { return err } + token, err = h.dependencies.edgeToken(runCtx) if err != nil { return err } + process, err = h.dependencies.startEdge(binary, h.edgeArguments(forward.URL(), token.Path())) if err != nil { return err } + if err := h.waitForEdgeReady(runCtx, process, h.dependencies.dialEdge); err != nil { return err } + if err := h.claimEndpoint(runCtx, "kubectl-unbounded/"+h.endpointName); err != nil { return err } @@ -377,28 +418,35 @@ func (h *siteBootstrapNetbootHandler) initializeDependencies() error { return h.resolveMetalmanBinary(exec.LookPath) } } + if h.dependencies.preflightNetwork == nil { h.dependencies.preflightNetwork = h.validateProvisioningInterface } + if h.dependencies.localPort == nil { h.dependencies.localPort = availableLoopbackPort } + if h.dependencies.startEdge == nil { h.dependencies.startEdge = func(binary string, args []string) (bootstrapEdgeProcess, error) { return startBootstrapEdgeProcess(binary, args, os.Stdout, os.Stderr) } } + if h.dependencies.dialEdge == nil { h.dependencies.dialEdge = dialBootstrapEdge } + if h.dependencies.preflightGateway == nil { h.dependencies.preflightGateway = h.validateExternalGateway } + if h.dependencies.gatewayRuntimeDir == nil { h.dependencies.gatewayRuntimeDir = func() (string, error) { return os.MkdirTemp("/run", "unbounded-netboot-") } } + if h.dependencies.startGateway == nil { h.dependencies.startGateway = startEmbeddedBootstrapGateway } @@ -415,10 +463,12 @@ func (h *siteBootstrapNetbootHandler) initializeClients() error { if err != nil { return fmt.Errorf("create Kubernetes client for netboot bootstrap: %w", err) } + resources, err := client.New(config, client.Options{Scheme: buildScheme()}) if err != nil { return fmt.Errorf("create resource client for netboot bootstrap: %w", err) } + h.kubeClient = kubeClient h.resources = resources h.restConfig = config @@ -442,6 +492,7 @@ func (h *siteBootstrapNetbootHandler) initializeRuntimeDependencies() { ) } } + if h.dependencies.edgeToken == nil { h.dependencies.edgeToken = func(ctx context.Context) (*bootstrapEdgeToken, error) { return newBootstrapEdgeToken(ctx, h.kubeClient, h.namespace, "", 0) @@ -454,14 +505,17 @@ func (h *siteBootstrapNetbootHandler) validateProvisioningInterface() error { if err != nil { return fmt.Errorf("find provisioning interface %s: %w", h.interfaceName, err) } + wanted := net.ParseIP(h.address) if wanted == nil || wanted.To4() == nil { return fmt.Errorf("provisioning address %q must be an IPv4 address", h.address) } + addresses, err := interfaceInfo.Addrs() if err != nil { return fmt.Errorf("list addresses on provisioning interface %s: %w", h.interfaceName, err) } + for _, address := range addresses { ip, _, parseErr := net.ParseCIDR(address.String()) if parseErr == nil && ip.Equal(wanted) { @@ -476,9 +530,11 @@ func (h *siteBootstrapNetbootHandler) validateExternalGateway() error { if err := h.validateRoutedCIDRs(); err != nil { return err } + if os.Geteuid() != 0 { return fmt.Errorf("external gateway dataplane requires root privileges") } + if ip := net.ParseIP(h.gatewayExternalAddress); ip == nil || ip.To4() == nil { return fmt.Errorf("gateway external address %q must be an IPv4 address", h.gatewayExternalAddress) } @@ -501,13 +557,19 @@ func availableLoopbackPort() (int, error) { if err != nil { return 0, err } - defer listener.Close() + defer listener.Close() //nolint:errcheck // Best-effort cleanup after reserving the port. + + tcpAddress, ok := listener.Addr().(*net.TCPAddr) + if !ok { + return 0, fmt.Errorf("loopback listener returned address type %T", listener.Addr()) + } - return listener.Addr().(*net.TCPAddr).Port, nil + return tcpAddress.Port, nil } func dialBootstrapEdge(ctx context.Context, address string) error { dialer := net.Dialer{} + connection, err := dialer.DialContext(ctx, "tcp", address) if err != nil { return err @@ -523,9 +585,11 @@ func (h *siteBootstrapNetbootHandler) prepareClusterResources(ctx context.Contex } enabled := true + if site.Spec.Components.Metalman == nil { site.Spec.Components.Metalman = &v1alpha3.MetalmanComponentSpec{} } + site.Spec.Components.Metalman.Enabled = &enabled if err := h.resources.Update(ctx, &site); err != nil { @@ -538,12 +602,12 @@ func (h *siteBootstrapNetbootHandler) prepareClusterResources(ctx context.Contex } if machine.Labels[v1alpha3.MachineSiteLabelKey] != h.site { - return bootstrapNetbootState{}, fmt.Errorf("Machine %s does not belong to Site %s", h.machine, h.site) + return bootstrapNetbootState{}, fmt.Errorf("machine %s does not belong to Site %s", h.machine, h.site) } netboot := machine.Spec.Netboot() if netboot == nil { - return bootstrapNetbootState{}, fmt.Errorf("Machine %s has no netboot configuration", h.machine) + return bootstrapNetbootState{}, fmt.Errorf("machine %s has no netboot configuration", h.machine) } state := bootstrapNetbootState{originalEndpointRef: netboot.EndpointRef} @@ -570,7 +634,12 @@ func (h *siteBootstrapNetbootHandler) prepareClusterResources(ctx context.Contex netboot.EndpointRef = h.endpointName if err := h.resources.Update(ctx, &machine); err != nil { - _ = h.resources.Delete(ctx, endpoint) + if deleteErr := h.resources.Delete(ctx, endpoint); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return bootstrapNetbootState{}, errors.Join( + fmt.Errorf("select bootstrap endpoint for Machine %s: %w", h.machine, err), + fmt.Errorf("delete NetbootEndpoint %s after Machine update failure: %w", h.endpointName, deleteErr), + ) + } return bootstrapNetbootState{}, fmt.Errorf("select bootstrap endpoint for Machine %s: %w", h.machine, err) } @@ -586,6 +655,7 @@ func (h *siteBootstrapNetbootHandler) restoreClusterResources(ctx context.Contex if netboot := machine.Spec.Netboot(); netboot != nil && netboot.EndpointRef == h.endpointName { netboot.EndpointRef = state.originalEndpointRef + if err := h.resources.Update(ctx, &machine); err != nil { return fmt.Errorf("restore Machine %s endpoint: %w", h.machine, err) } @@ -607,6 +677,7 @@ func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Contex protocol := netv1alpha1.TunnelProtocolWireGuard enabled := true selector := map[string]string{"net.unbounded-cloud.io/bootstrap-gateway": h.endpointName} + pool := &netv1alpha1.GatewayPool{ ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, Spec: netv1alpha1.GatewayPoolSpec{ @@ -619,6 +690,7 @@ func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Contex if err := h.resources.Create(ctx, pool); err != nil { return fmt.Errorf("create bootstrap GatewayPool %s: %w", h.endpointName, err) } + defer func() { if retErr != nil { retErr = errors.Join(retErr, h.cleanupGatewayResources(context.WithoutCancel(ctx))) @@ -653,10 +725,12 @@ func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Contex }}, }, } + createdNode, err := h.kubeClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("create bootstrap gateway Node %s: %w", h.endpointName, err) } + createdNode.Status.Addresses = []corev1.NodeAddress{ {Type: corev1.NodeInternalIP, Address: h.address}, {Type: corev1.NodeExternalIP, Address: h.gatewayExternalAddress}, @@ -670,10 +744,12 @@ func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Contex func (h *siteBootstrapNetbootHandler) cleanupGatewayResources(ctx context.Context) error { assignment := &netv1alpha1.SiteGatewayPoolAssignment{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + assignmentErr := h.resources.Delete(ctx, assignment) if apierrors.IsNotFound(assignmentErr) { assignmentErr = nil } + if assignmentErr != nil { assignmentErr = fmt.Errorf("delete bootstrap SiteGatewayPoolAssignment %s: %w", h.endpointName, assignmentErr) } @@ -682,15 +758,18 @@ func (h *siteBootstrapNetbootHandler) cleanupGatewayResources(ctx context.Contex if apierrors.IsNotFound(nodeErr) { nodeErr = nil } + if nodeErr != nil { nodeErr = fmt.Errorf("delete bootstrap gateway Node %s: %w", h.endpointName, nodeErr) } pool := &netv1alpha1.GatewayPool{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + poolErr := h.resources.Delete(ctx, pool) if apierrors.IsNotFound(poolErr) { poolErr = nil } + if poolErr != nil { poolErr = fmt.Errorf("delete bootstrap GatewayPool %s: %w", h.endpointName, poolErr) } @@ -704,9 +783,11 @@ func (h *siteBootstrapNetbootHandler) metalmanDeploymentsReady(ctx context.Conte if apierrors.IsNotFound(err) { return false, nil } + if err != nil { return false, fmt.Errorf("get Deployment %s/%s: %w", h.namespace, name, err) } + if !deploymentRolloutComplete(deployment) { return false, nil } @@ -729,6 +810,7 @@ func (h *siteBootstrapNetbootHandler) waitForMetalman(ctx context.Context) error if err != nil { return err } + if ready { return nil } @@ -761,10 +843,11 @@ func (h *siteBootstrapNetbootHandler) resolveMetalmanBinary(lookPath func(string if h.metalmanBinary != "" { info, err := os.Stat(h.metalmanBinary) if err != nil { - return "", fmt.Errorf("Metalman binary %q is not accessible: %w", h.metalmanBinary, err) + return "", fmt.Errorf("metalman binary %q is not accessible: %w", h.metalmanBinary, err) } + if info.IsDir() || info.Mode()&0o111 == 0 { - return "", fmt.Errorf("Metalman binary %q is not executable", h.metalmanBinary) + return "", fmt.Errorf("metalman binary %q is not executable", h.metalmanBinary) } return h.metalmanBinary, nil @@ -787,7 +870,9 @@ func (h *siteBootstrapNetbootHandler) waitForEdgeReady( if interval <= 0 { interval = defaultBootstrapPollInterval } + address := net.JoinHostPort(h.address, strconv.Itoa(h.httpPort)) + ticker := time.NewTicker(interval) defer ticker.Stop() @@ -799,10 +884,10 @@ func (h *siteBootstrapNetbootHandler) waitForEdgeReady( select { case <-process.Done(): if err := process.Err(); err != nil { - return fmt.Errorf("Metalman edge exited before becoming ready: %w", err) + return fmt.Errorf("metalman edge exited before becoming ready: %w", err) } - return errors.New("Metalman edge exited before becoming ready") + return errors.New("metalman edge exited before becoming ready") case <-ctx.Done(): return fmt.Errorf("wait for Metalman edge listener: %w", ctx.Err()) case <-ticker.C: @@ -813,14 +898,17 @@ func (h *siteBootstrapNetbootHandler) waitForEdgeReady( func startBootstrapEdgeProcess(binary string, args []string, stdout, stderr io.Writer) (bootstrapEdgeProcess, error) { cmd := exec.Command(binary, args...) cmd.Stdout = stdout + cmd.Stderr = stderr if err := cmd.Start(); err != nil { return nil, fmt.Errorf("start Metalman edge: %w", err) } process := &commandBootstrapEdgeProcess{cmd: cmd, done: make(chan struct{})} + go func() { err := cmd.Wait() + process.mu.Lock() process.err = err process.mu.Unlock() @@ -836,8 +924,10 @@ func startEmbeddedBootstrapGateway( ) (bootstrapEdgeProcess, error) { runCtx, cancel := context.WithCancel(ctx) process := &embeddedBootstrapGatewayProcess{cancel: cancel, done: make(chan struct{})} + go func() { err := nodeagent.RunExternalGateway(runCtx, options) + process.mu.Lock() process.err = err process.mu.Unlock() @@ -852,7 +942,8 @@ func isSignalExit(err error) bool { if !errors.As(err, &exitError) || exitError.ProcessState == nil { return false } - status, ok := exitError.ProcessState.Sys().(syscall.WaitStatus) + + status, ok := exitError.Sys().(syscall.WaitStatus) return ok && status.Signaled() } @@ -899,6 +990,7 @@ func (h *siteBootstrapNetbootHandler) waitForNodeReady(ctx context.Context, node if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("get designated Node %s: %w", nodeName, err) } + if err == nil && nodeReady(node) { return nil } @@ -911,31 +1003,6 @@ func (h *siteBootstrapNetbootHandler) waitForNodeReady(ctx context.Context, node } } -func (h *siteBootstrapNetbootHandler) waitForNodeReadyAndEdge( - ctx context.Context, - nodeName string, - process bootstrapEdgeProcess, -) error { - waitCtx, cancel := context.WithCancel(ctx) - defer cancel() - - ready := make(chan error, 1) - go func() { ready <- h.waitForNodeReady(waitCtx, nodeName) }() - - select { - case err := <-ready: - return err - case <-process.Done(): - if err := process.Err(); err != nil { - return fmt.Errorf("Metalman edge exited before designated Node %s became Ready: %w", nodeName, err) - } - - return fmt.Errorf("Metalman edge exited before designated Node %s became Ready", nodeName) - case <-ctx.Done(): - return fmt.Errorf("wait for designated Node %s while Metalman edge is running: %w", nodeName, ctx.Err()) - } -} - func (h *siteBootstrapNetbootHandler) waitForNodeReadyAndProcesses( ctx context.Context, nodeName string, @@ -946,17 +1013,19 @@ func (h *siteBootstrapNetbootHandler) waitForNodeReadyAndProcesses( defer cancel() ready := make(chan error, 1) + go func() { ready <- h.waitForNodeReady(waitCtx, nodeName) }() var gatewayDone <-chan struct{} if gateway != nil { gatewayDone = gateway.Done() } + select { case err := <-ready: return err case <-edge.Done(): - return processExitError("Metalman edge", nodeName, edge.Err()) + return processExitError("metalman edge", nodeName, edge.Err()) case <-gatewayDone: return processExitError("external gateway", nodeName, gateway.Err()) case <-ctx.Done(): @@ -977,6 +1046,7 @@ func (h *siteBootstrapNetbootHandler) waitForGatewayReady(ctx context.Context, p if interval <= 0 { interval = defaultBootstrapPollInterval } + ticker := time.NewTicker(interval) defer ticker.Stop() @@ -985,9 +1055,11 @@ func (h *siteBootstrapNetbootHandler) waitForGatewayReady(ctx context.Context, p if err != nil { return err } + if ready { return nil } + select { case <-process.Done(): if err := process.Err(); err != nil { @@ -1007,6 +1079,7 @@ func (h *siteBootstrapNetbootHandler) gatewayReady(ctx context.Context) (bool, e if err != nil { return false, fmt.Errorf("get bootstrap gateway Node %s: %w", h.endpointName, err) } + if node.Annotations["net.unbounded-cloud.io/wg-pubkey"] == "" { return false, nil } @@ -1015,16 +1088,20 @@ func (h *siteBootstrapNetbootHandler) gatewayReady(ctx context.Context) (bool, e if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool); err != nil { return false, fmt.Errorf("get bootstrap GatewayPool %s: %w", h.endpointName, err) } + if pool.Status.NodeCount != 1 || !containsString(pool.Status.ConnectedSites, h.site) { return false, nil } + matchedNode := false + for _, poolNode := range pool.Status.Nodes { if poolNode.Name == h.endpointName && poolNode.WireGuardPublicKey != "" { matchedNode = true break } } + if !matchedNode { return false, nil } @@ -1096,6 +1173,7 @@ func newBootstrapPortForward( } forwardCtx, cancel := context.WithCancel(ctx) + podName, err := readyDeploymentPod(forwardCtx, kubeClient, namespace, deploymentName, "") if err != nil { cancel() @@ -1120,6 +1198,7 @@ func newBootstrapPortForward( defer close(forward.done) current := attempt + for { select { case <-forwardCtx.Done(): @@ -1168,6 +1247,7 @@ func readyDeploymentPod( } selector := metav1.FormatLabelSelector(deployment.Spec.Selector) + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) if err != nil { return "", fmt.Errorf("list Pods for Deployment %s/%s: %w", namespace, deploymentName, err) @@ -1175,6 +1255,7 @@ func readyDeploymentPod( ready := make([]string, 0, len(pods.Items)) excludedReady := false + for i := range pods.Items { pod := &pods.Items[i] if pod.DeletionTimestamp == nil && podReady(pod) { @@ -1183,14 +1264,17 @@ func readyDeploymentPod( continue } + ready = append(ready, pod.Name) } } + if len(ready) == 0 && excludedReady { ready = append(ready, excludedPod) } + if len(ready) == 0 { - return "", fmt.Errorf("Deployment %s/%s has no Ready Pods", namespace, deploymentName) + return "", fmt.Errorf("deployment %s/%s has no Ready Pods", namespace, deploymentName) } sort.Strings(ready) @@ -1239,6 +1323,7 @@ func newSPDYBootstrapPortForwardStarter( Name(podName). SubResource("portforward"). URL() + transport, upgrader, err := spdy.RoundTripperFor(config) if err != nil { return nil, fmt.Errorf("create SPDY transport: %w", err) @@ -1247,6 +1332,7 @@ func newSPDYBootstrapPortForwardStarter( stop := make(chan struct{}) ready := make(chan struct{}) dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, targetURL) + forwarder, err := portforward.NewOnAddresses( dialer, []string{"127.0.0.1"}, @@ -1261,6 +1347,7 @@ func newSPDYBootstrapPortForwardStarter( } attempt := &spdyBootstrapPortForwardAttempt{stop: stop, done: make(chan error, 1)} + go func() { attempt.done <- forwarder.ForwardPorts() }() select { @@ -1296,7 +1383,9 @@ func newBootstrapEdgeToken( path := filepath.Join(directory, "edge-token") if err := refreshBootstrapEdgeToken(ctx, kubeClient, namespace, path); err != nil { - _ = os.RemoveAll(directory) + if removeErr := os.RemoveAll(directory); removeErr != nil { + return nil, errors.Join(err, fmt.Errorf("remove edge token directory: %w", removeErr)) + } return nil, err } @@ -1304,6 +1393,7 @@ func newBootstrapEdgeToken( if refreshInterval <= 0 { refreshInterval = 20 * time.Minute } + tokenCtx, cancel := context.WithCancel(ctx) credential := &bootstrapEdgeToken{ path: path, @@ -1322,7 +1412,9 @@ func newBootstrapEdgeToken( case <-tokenCtx.Done(): return case <-ticker.C: - _ = refreshBootstrapEdgeToken(tokenCtx, kubeClient, namespace, path) + if err := refreshBootstrapEdgeToken(tokenCtx, kubeClient, namespace, path); err != nil && tokenCtx.Err() == nil { + slog.WarnContext(tokenCtx, "refreshing Metalman edge token failed", "err", err) + } } } }() @@ -1337,6 +1429,7 @@ func refreshBootstrapEdgeToken( path string, ) error { expirationSeconds := int64(time.Hour / time.Second) + response, err := kubeClient.CoreV1().ServiceAccounts(namespace).CreateToken( ctx, "metalman-edge", @@ -1349,6 +1442,7 @@ func refreshBootstrapEdgeToken( if err != nil { return fmt.Errorf("request metalman-edge token: %w", err) } + if response.Status.Token == "" { return fmt.Errorf("request metalman-edge token: API returned an empty token") } @@ -1357,8 +1451,14 @@ func refreshBootstrapEdgeToken( if err := os.WriteFile(temporaryPath, []byte(response.Status.Token), 0o600); err != nil { return fmt.Errorf("write metalman-edge token: %w", err) } + if err := os.Rename(temporaryPath, path); err != nil { - _ = os.Remove(temporaryPath) + if removeErr := os.Remove(temporaryPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return errors.Join( + fmt.Errorf("replace metalman-edge token: %w", err), + fmt.Errorf("remove temporary metalman-edge token: %w", removeErr), + ) + } return fmt.Errorf("replace metalman-edge token: %w", err) } diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go index 164402ff0..6892ca67d 100644 --- a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -177,12 +177,14 @@ func TestBootstrapNetbootWaitsUntilMetalmanRolloutCompletes(t *testing.T) { go func() { time.Sleep(5 * time.Millisecond) + server.Status.AvailableReplicas = 2 _, _ = kubeClient.AppsV1().Deployments(h.namespace).UpdateStatus(context.Background(), server, metav1.UpdateOptions{}) }() ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() + require.NoError(t, h.waitForMetalman(ctx)) } @@ -252,6 +254,7 @@ func TestBootstrapNetbootClaimsEndpointAndWaitsForDesignatedNode(t *testing.T) { } require.NoError(t, h.claimEndpoint(ctx, "bootstrap/123")) + var claimed v1alpha3.NetbootEndpoint require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: endpoint.Name}, &claimed)) require.Equal(t, endpoint.Generation, claimed.Status.ObservedGeneration) @@ -260,10 +263,13 @@ func TestBootstrapNetbootClaimsEndpointAndWaitsForDesignatedNode(t *testing.T) { waitCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() + ready := make(chan error, 1) + go func() { ready <- h.waitForNodeReady(waitCtx, node.Name) }() time.Sleep(5 * time.Millisecond) + other := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{Name: "other", Labels: map[string]string{v1alpha3.MachineSiteLabelKey: "rack-a"}}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}, @@ -331,6 +337,7 @@ func TestBootstrapPortForwardReconnectsToReadyServerPod(t *testing.T) { require.Equal(t, "server-b", <-starter.started) starter.fail("server-b", errors.New("connection lost")) + select { case podName := <-starter.started: require.Equal(t, "server-c", podName) @@ -368,7 +375,9 @@ func TestBootstrapPortForwardClosesWhileWaitingToReconnect(t *testing.T) { time.Sleep(time.Millisecond) closed := make(chan error, 1) + go func() { closed <- forward.Close() }() + select { case err := <-closed: require.NoError(t, err) @@ -380,8 +389,11 @@ func TestBootstrapPortForwardClosesWhileWaitingToReconnect(t *testing.T) { func TestBootstrapEdgeTokenUsesAudienceAndRotatesSecureFile(t *testing.T) { ctx := context.Background() kubeClient := fake.NewSimpleClientset() + var mu sync.Mutex + requests := 0 + kubeClient.PrependReactor("create", "serviceaccounts", func(action clienttesting.Action) (bool, runtime.Object, error) { create := action.(clienttesting.CreateAction) request := create.GetObject().(*authenticationv1.TokenRequest) @@ -400,6 +412,7 @@ func TestBootstrapEdgeTokenUsesAudienceAndRotatesSecureFile(t *testing.T) { credential, err := newBootstrapEdgeToken(ctx, kubeClient, "unbounded-system", t.TempDir(), time.Millisecond) require.NoError(t, err) + path := credential.Path() require.Equal(t, "edge-token", filepath.Base(path)) @@ -412,6 +425,7 @@ func TestBootstrapEdgeTokenUsesAudienceAndRotatesSecureFile(t *testing.T) { }, time.Second, time.Millisecond) require.NoError(t, credential.Close()) + _, err = os.Stat(filepath.Dir(path)) require.ErrorIs(t, err, os.ErrNotExist) } @@ -422,6 +436,7 @@ func TestBootstrapWaitsForEdgeListenerAndReportsEarlyExit(t *testing.T) { attempts := 0 dial := func(_ context.Context, address string) error { require.Equal(t, "192.0.2.10:8880", address) + attempts++ if attempts < 2 { return errors.New("not listening") @@ -505,8 +520,10 @@ func TestBootstrapNetbootExecuteCleansUpAfterDesignatedNodeReady(t *testing.T) { var gotMachine v1alpha3.Machine require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + var endpoint v1alpha3.NetbootEndpoint require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) + _, err := os.Stat(filepath.Dir(token.Path())) require.ErrorIs(t, err, os.ErrNotExist) } @@ -516,6 +533,7 @@ func TestBootstrapNetbootExecuteReportsEarlyEdgeExitAndRollsBack(t *testing.T) { site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, false) process := &fakeBootstrapEdgeProcess{done: make(chan struct{}), err: errors.New("bind: address already in use")} close(process.done) + h := &siteBootstrapNetbootHandler{ site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, @@ -541,6 +559,7 @@ func TestBootstrapNetbootExecuteReportsEarlyEdgeExitAndRollsBack(t *testing.T) { var gotMachine v1alpha3.Machine require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + var endpoint v1alpha3.NetbootEndpoint require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) } @@ -576,6 +595,7 @@ func TestBootstrapNetbootPreparesAndCleansExternalGatewayResources(t *testing.T) } require.NoError(t, h.prepareGatewayResources(ctx)) + var pool netv1alpha1.GatewayPool require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) require.Equal(t, "External", pool.Spec.Type) @@ -649,6 +669,7 @@ func TestBootstrapNetbootExecuteRunsOptInGatewayUntilNodeReady(t *testing.T) { require.NoError(t, h.execute(ctx)) require.True(t, gatewayProcess.stopped) require.True(t, edgeProcess.stopped) + var pool netv1alpha1.GatewayPool require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) _, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) @@ -666,9 +687,11 @@ func markBootstrapGatewayReady( node, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) require.NoError(t, err) + if node.Annotations == nil { node.Annotations = map[string]string{} } + node.Annotations["net.unbounded-cloud.io/wg-pubkey"] = "public-key" _, err = kubeClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) require.NoError(t, err) @@ -701,6 +724,7 @@ func TestBootstrapNetbootRejectsInvalidRoutedCIDRBeforeCreatingGateway(t *testin err := h.prepareGatewayResources(context.Background()) require.ErrorContains(t, err, "invalid routed CIDR") + var pools netv1alpha1.GatewayPoolList require.NoError(t, resources.List(context.Background(), &pools)) require.Empty(t, pools.Items) @@ -730,6 +754,7 @@ func TestBootstrapNetbootExecuteRejectsInvalidRoutedCIDRBeforeClusterMutation(t var actualMachine v1alpha3.Machine require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &actualMachine)) require.Equal(t, "permanent-edge", actualMachine.Spec.Netboot().EndpointRef) + var endpoints v1alpha3.NetbootEndpointList require.NoError(t, resources.List(ctx, &endpoints)) require.Empty(t, endpoints.Items) @@ -757,6 +782,7 @@ func bootstrapLifecycleFixture( Build() controller := readyDeployment("metalman-controller-rack-a", 1) server := readyDeployment("metalman-server-rack-a", 2) + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}} if nodeReady { node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} @@ -768,6 +794,7 @@ func bootstrapLifecycleFixture( func testBootstrapPortForward(ctx context.Context) *bootstrapPortForward { forwardCtx, cancel := context.WithCancel(ctx) done := make(chan struct{}) + go func() { <-forwardCtx.Done() close(done) @@ -781,8 +808,10 @@ func testBootstrapEdgeToken(t *testing.T, ctx context.Context) *bootstrapEdgeTok directory := t.TempDir() path := filepath.Join(directory, "edge-token") require.NoError(t, os.WriteFile(path, []byte("token"), 0o600)) + tokenCtx, cancel := context.WithCancel(ctx) done := make(chan struct{}) + go func() { <-tokenCtx.Done() close(done) @@ -822,13 +851,16 @@ func (f *fakeBootstrapPortForwardStarter) start(_ context.Context, podName strin } attempt := &fakeBootstrapPortForwardAttempt{done: make(chan error, 1)} + f.mu.Lock() if f.attempts == nil { f.attempts = map[string]*fakeBootstrapPortForwardAttempt{} } + f.attempts[podName] = attempt f.local = append(f.local, localPort) f.mu.Unlock() + f.started <- podName return attempt, nil @@ -838,6 +870,7 @@ func (f *fakeBootstrapPortForwardStarter) fail(podName string, err error) { f.mu.Lock() attempt := f.attempts[podName] f.mu.Unlock() + attempt.done <- err } diff --git a/cmd/unbounded-net-node/main.go b/cmd/unbounded-net-node/main.go index 11de9a9b6..43858363a 100644 --- a/cmd/unbounded-net-node/main.go +++ b/cmd/unbounded-net-node/main.go @@ -18,6 +18,7 @@ func main() { cmd := nodeagent.NewCommand() cmd.SetContext(ctx) + if err := cmd.Execute(); err != nil { os.Exit(1) } diff --git a/hack/smoke_metalman_contract_test.go b/hack/smoke_metalman_contract_test.go index 7b7a9ba60..38c17c134 100644 --- a/hack/smoke_metalman_contract_test.go +++ b/hack/smoke_metalman_contract_test.go @@ -22,12 +22,14 @@ func TestMetalmanSmokeSuitesUseSplitRuntime(t *testing.T) { if err != nil { t.Fatal(err) } + text := string(contents) for _, stale := range []string{"serve-pxe", `"bootProtocol"`} { if strings.Contains(text, stale) { t.Errorf("%s still contains legacy %q wiring", name, stale) } } + for _, required := range []string{"deploy_split_metalman", "bootstrap-netboot"} { if !strings.Contains(text, required) { t.Errorf("%s does not exercise %q", name, required) @@ -44,6 +46,7 @@ func TestTraditionalMetalmanSmokeDisruptsServerDuringProvisioning(t *testing.T) if err != nil { t.Fatal(err) } + if !strings.Contains(string(contents), "delete_metalman_server_during_provisioning") { t.Fatal("traditional smoke does not disrupt a server pod during provisioning") } @@ -56,10 +59,12 @@ func TestHTTPFixtureFetchesCapabilityEntrypoint(t *testing.T) { if err != nil { t.Fatal(err) } + text := string(contents) if strings.Contains(text, "cache_dir.glob") { t.Fatal("HTTP fixture still reads a process-local Metalman cache") } + if !strings.Contains(text, `("http-entrypoint.efi", boot_url)`) { t.Fatal("HTTP fixture does not fetch the capability-scoped firmware URL") } diff --git a/internal/metalman/attestation/attestation_test.go b/internal/metalman/attestation/attestation_test.go index b0849bdd1..85d19f18a 100644 --- a/internal/metalman/attestation/attestation_test.go +++ b/internal/metalman/attestation/attestation_test.go @@ -351,10 +351,12 @@ func TestAttestMachineDoesNotUseRequestSourceIP(t *testing.T) { t.Fatal("session attestation must not resolve a Machine by source IP") return nil, nil } + body, err := json.Marshal(AttestRequest{EKPub: ekPub, SRKPub: srkPub}) if err != nil { t.Fatal(err) } + request := httptest.NewRequest(http.MethodPost, "/session/attest", bytes.NewReader(body)) request.RemoteAddr = "198.51.100.20:12345" response := httptest.NewRecorder() diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go index d0e380f6f..8695b9b98 100644 --- a/internal/metalman/commands/edge.go +++ b/internal/metalman/commands/edge.go @@ -56,6 +56,7 @@ func EdgeCmd() *cobra.Command { if err != nil { return err } + if err := validateEdgeTLSFiles(tlsCertFile, tlsKeyFile); err != nil { return err } @@ -64,6 +65,7 @@ func EdgeCmd() *cobra.Command { defer stop() addr := fmt.Sprintf("%s:%d", bindAddress, httpPort) + server := &http.Server{ Addr: addr, Handler: newEdgeProxy(backend), @@ -74,11 +76,14 @@ func EdgeCmd() *cobra.Command { if err != nil { return fmt.Errorf("creating DHCP backend: %w", err) } + serverIP, err := edgeDHCPServerIP(dhcpServerIP, dhcpInterface) if err != nil { return err } + dhcpServer := &dhcp.Server{Interface: dhcpInterface, Port: dhcpPort, DecisionProvider: decisionProvider, ServerIP: serverIP} + go func() { if err := dhcpServer.Start(ctx); err != nil && ctx.Err() == nil { slog.ErrorContext(ctx, "Metalman edge DHCP server failed", "err", err) @@ -86,12 +91,15 @@ func EdgeCmd() *cobra.Command { } }() } + if tftpEnabled { artifactBackend, err := netboot.NewHTTPArtifactBackend(backend.String(), nil) if err != nil { return fmt.Errorf("creating TFTP backend: %w", err) } + tftpServer := &netboot.TFTPServer{BindAddr: tftpBindAddr, Port: tftpPort, Backend: artifactBackend} + go func() { if err := tftpServer.Start(ctx); err != nil && ctx.Err() == nil { slog.ErrorContext(ctx, "Metalman edge TFTP server failed", "err", err) @@ -102,7 +110,10 @@ func EdgeCmd() *cobra.Command { go func() { <-ctx.Done() - _ = server.Shutdown(context.Background()) + + if err := server.Shutdown(context.Background()); err != nil { + slog.Warn("shutting down Metalman edge HTTP server failed", "err", err) + } }() PrintConfig("role", string(metalmanRoleEdge)) @@ -133,8 +144,14 @@ func EdgeCmd() *cobra.Command { cmd.Flags().BoolVar(&tftpEnabled, "tftp-enabled", false, "Enable the TFTP protocol edge") cmd.Flags().StringVar(&tftpBindAddr, "tftp-bind-address", "0.0.0.0", "IP address to bind the TFTP listener") cmd.Flags().IntVar(&tftpPort, "tftp-port", 69, "TFTP listener port") - _ = cmd.MarkFlagRequired("backend-url") - _ = cmd.MarkFlagRequired("endpoint") + + if err := cmd.MarkFlagRequired("backend-url"); err != nil { + panic(fmt.Sprintf("mark backend-url flag required: %v", err)) + } + + if err := cmd.MarkFlagRequired("endpoint"); err != nil { + panic(fmt.Sprintf("mark endpoint flag required: %v", err)) + } return cmd } @@ -164,6 +181,7 @@ func edgeDHCPServerIP(configured, iface string) (net.IP, error) { return ip, nil } + if iface != "" { return InterfaceIPv4(iface) } @@ -181,9 +199,11 @@ func parseEdgeBackendURL(value string) (*url.URL, error) { if err != nil { return nil, fmt.Errorf("parsing --backend-url: %w", err) } + if backend.Scheme != "http" && backend.Scheme != "https" { return nil, errors.New("--backend-url must use http or https") } + if backend.Host == "" { return nil, errors.New("--backend-url must include a host") } @@ -199,12 +219,14 @@ func newEdgeProxy(backend *url.URL) http.Handler { } return &edgeProxy{ + backend: backend, proxy: proxy, transport: http.DefaultTransport, } } type edgeProxy struct { + backend *url.URL proxy *httputil.ReverseProxy transport http.RoundTripper } @@ -220,12 +242,13 @@ func (e *edgeProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (e *edgeProxy) serveArtifact(w http.ResponseWriter, r *http.Request) { request := r.Clone(r.Context()) - e.proxy.Director(request) + e.rewriteRequest(request) response, err := e.transport.RoundTrip(request) if err != nil { slog.Warn("Metalman edge artifact request failed", "err", err) http.Error(w, "Metalman backend unavailable", http.StatusBadGateway) + return } @@ -234,21 +257,31 @@ func (e *edgeProxy) serveArtifact(w http.ResponseWriter, r *http.Request) { remaining := response.ContentLength if remaining <= 0 || (response.StatusCode != http.StatusOK && response.StatusCode != http.StatusPartialContent) { - _, _ = io.Copy(w, response.Body) + if _, err := io.Copy(w, response.Body); err != nil { + slog.Warn("Metalman edge response copy failed", "path", r.URL.Path, "err", err) + } + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + return } start, end, ok := responseByteRange(response) if !ok { - _, _ = io.Copy(w, response.Body) + if _, err := io.Copy(w, response.Body); err != nil { + slog.Warn("Metalman edge response copy failed", "path", r.URL.Path, "err", err) + } + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + return } written, copyErr := io.Copy(w, response.Body) response.Body.Close() //nolint:errcheck // The response body is no longer needed. + start += written + remaining -= written if copyErr == nil && remaining == 0 { return @@ -256,28 +289,35 @@ func (e *edgeProxy) serveArtifact(w http.ResponseWriter, r *http.Request) { for attempt := 2; attempt <= maxArtifactBackendAttempts && remaining > 0 && start <= end; attempt++ { request = r.Clone(r.Context()) - e.proxy.Director(request) + e.rewriteRequest(request) request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + response, err = e.transport.RoundTrip(request) if err != nil { slog.Warn("Metalman edge artifact resume failed", "path", r.URL.Path, "err", err) continue } + if response.StatusCode != http.StatusPartialContent || response.ContentLength != remaining { response.Body.Close() //nolint:errcheck // Invalid resume response. slog.Warn("Metalman edge artifact resume returned an invalid range", "path", r.URL.Path, "status", response.StatusCode) + return } + resumeStart, resumeEnd, valid := responseByteRange(response) if !valid || resumeStart != start || resumeEnd != end { response.Body.Close() //nolint:errcheck // Invalid resume response. slog.Warn("Metalman edge artifact resume returned mismatched bytes", "path", r.URL.Path) + return } written, copyErr = io.Copy(w, response.Body) response.Body.Close() //nolint:errcheck // The response body is no longer needed. + start += written + remaining -= written if copyErr == nil && remaining == 0 { return @@ -287,6 +327,44 @@ func (e *edgeProxy) serveArtifact(w http.ResponseWriter, r *http.Request) { slog.Warn("Metalman edge artifact transfer failed", "path", r.URL.Path, "err", copyErr) } +func (e *edgeProxy) rewriteRequest(request *http.Request) { + request.URL.Scheme = e.backend.Scheme + request.URL.Host = e.backend.Host + request.URL.Path, request.URL.RawPath = joinURLPath(e.backend, request.URL) + request.Host = e.backend.Host + + if e.backend.RawQuery == "" || request.URL.RawQuery == "" { + request.URL.RawQuery = e.backend.RawQuery + request.URL.RawQuery + } else { + request.URL.RawQuery = e.backend.RawQuery + "&" + request.URL.RawQuery + } +} + +func joinURLPath(base, request *url.URL) (string, string) { + if base.RawPath == "" && request.RawPath == "" { + return singleJoiningSlash(base.Path, request.Path), "" + } + + basePath := base.EscapedPath() + requestPath := request.EscapedPath() + + return singleJoiningSlash(base.Path, request.Path), singleJoiningSlash(basePath, requestPath) +} + +func singleJoiningSlash(left, right string) string { + leftSlash := strings.HasSuffix(left, "/") + rightSlash := strings.HasPrefix(right, "/") + + switch { + case leftSlash && rightSlash: + return left + right[1:] + case !leftSlash && !rightSlash: + return left + "/" + right + default: + return left + right + } +} + func responseByteRange(response *http.Response) (int64, int64, bool) { if response.StatusCode == http.StatusOK { return 0, response.ContentLength - 1, true @@ -296,18 +374,22 @@ func responseByteRange(response *http.Response) (int64, int64, bool) { if !strings.HasPrefix(value, "bytes ") { return 0, 0, false } + rangeAndSize := strings.SplitN(strings.TrimPrefix(value, "bytes "), "/", 2) if len(rangeAndSize) != 2 { return 0, 0, false } + bounds := strings.SplitN(rangeAndSize[0], "-", 2) if len(bounds) != 2 { return 0, 0, false } + start, err := strconv.ParseInt(bounds[0], 10, 64) if err != nil { return 0, 0, false } + end, err := strconv.ParseInt(bounds[1], 10, 64) if err != nil || start < 0 || end < start || end-start+1 != response.ContentLength { return 0, 0, false diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go index 4fdb139d2..ca7712aa3 100644 --- a/internal/metalman/commands/roles_test.go +++ b/internal/metalman/commands/roles_test.go @@ -134,9 +134,11 @@ func TestEdgeProxyPreservesSessionPathAndRange(t *testing.T) { if r.URL.Path != "/v1/netboot/capability/artifact/disk.img.gz" { t.Errorf("backend path = %q", r.URL.Path) } + if got := r.Header.Get("Range"); got != "bytes=4096-8191" { t.Errorf("backend Range = %q", got) } + w.Header().Set("Content-Range", "bytes 4096-8191/16384") w.WriteHeader(http.StatusPartialContent) _, _ = w.Write([]byte("range")) @@ -150,6 +152,7 @@ func TestEdgeProxyPreservesSessionPathAndRange(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "/v1/netboot/capability/artifact/disk.img.gz", nil) request.Header.Set("Range", "bytes=4096-8191") + response := httptest.NewRecorder() newEdgeProxy(backendURL).ServeHTTP(response, request) @@ -157,13 +160,16 @@ func TestEdgeProxyPreservesSessionPathAndRange(t *testing.T) { if response.Code != http.StatusPartialContent { t.Fatalf("status = %d, want %d", response.Code, http.StatusPartialContent) } + if got := response.Header().Get("Content-Range"); got != "bytes 4096-8191/16384" { t.Errorf("Content-Range = %q", got) } + body, err := io.ReadAll(response.Result().Body) if err != nil { t.Fatal(err) } + if got := string(body); got != "range" { t.Errorf("body = %q", got) } @@ -175,18 +181,21 @@ func TestEdgeProxyResumesTruncatedArtifactFromBackendRange(t *testing.T) { const artifact = "immutable-artifact" var requests atomic.Int32 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch requests.Add(1) { case 1: if got := r.Header.Get("Range"); got != "" { t.Errorf("initial Range = %q, want empty", got) } + w.Header().Set("Content-Length", "18") _, _ = io.WriteString(w, artifact[:9]) case 2: if got := r.Header.Get("Range"); got != "bytes=9-17" { t.Errorf("resume Range = %q, want %q", got, "bytes=9-17") } + w.Header().Set("Content-Length", "9") w.Header().Set("Content-Range", "bytes 9-17/18") w.WriteHeader(http.StatusPartialContent) @@ -202,6 +211,7 @@ func TestEdgeProxyResumesTruncatedArtifactFromBackendRange(t *testing.T) { if err != nil { t.Fatal(err) } + edge := httptest.NewServer(newEdgeProxy(backendURL)) defer edge.Close() @@ -215,9 +225,11 @@ func TestEdgeProxyResumesTruncatedArtifactFromBackendRange(t *testing.T) { if err != nil { t.Fatal(err) } + if got := string(body); got != artifact { t.Errorf("body = %q, want %q", got, artifact) } + if got := requests.Load(); got != 2 { t.Errorf("backend requests = %d, want 2", got) } @@ -229,6 +241,7 @@ func TestEdgeProxyRetriesFailedArtifactResumeRequest(t *testing.T) { const artifact = "immutable-artifact" var requests atomic.Int32 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch requests.Add(1) { case 1: @@ -240,11 +253,13 @@ func TestEdgeProxyRetriesFailedArtifactResumeRequest(t *testing.T) { t.Errorf("hijacking failed resume request: %v", err) return } + _ = conn.Close() case 3: if got := r.Header.Get("Range"); got != "bytes=9-17" { t.Errorf("resume Range = %q, want %q", got, "bytes=9-17") } + w.Header().Set("Content-Length", "9") w.Header().Set("Content-Range", "bytes 9-17/18") w.WriteHeader(http.StatusPartialContent) @@ -260,6 +275,7 @@ func TestEdgeProxyRetriesFailedArtifactResumeRequest(t *testing.T) { if err != nil { t.Fatal(err) } + edge := httptest.NewServer(newEdgeProxy(backendURL)) defer edge.Close() @@ -273,9 +289,11 @@ func TestEdgeProxyRetriesFailedArtifactResumeRequest(t *testing.T) { if err != nil { t.Fatal(err) } + if got := string(body); got != artifact { t.Errorf("body = %q, want %q", got, artifact) } + if got := requests.Load(); got != 3 { t.Errorf("backend requests = %d, want 3", got) } diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index 78a2cfb84..f88bfd987 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -67,6 +67,7 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { capabilityKeyID string edgeServiceAccount string ) + components := componentsForRole(role) cmd := &cobra.Command{ @@ -261,19 +262,23 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { if components.machineOps { var sessionHTTPBootURL func(*v1alpha3.NetbootSession) (string, error) + if components.sessionManager { capabilityKey, err := os.ReadFile(capabilityKeyFile) if err != nil { return fmt.Errorf("reading capability key: %w", err) } + capabilities, err := netboot.NewCapabilitySigner(capabilityKey, capabilityKeyID, nil) if err != nil { return fmt.Errorf("creating capability signer: %w", err) } + sessionHTTPBootURL = func(session *v1alpha3.NetbootSession) (string, error) { return netboot.SessionArtifactURL(capabilities, session, session.Spec.Boot.FirmwareArtifact) } } + if err := (&metalmachineops.Reconciler{ Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), @@ -348,6 +353,7 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { if components.http { httpMux := http.NewServeMux() + var attestHandler *attestation.Handler if components.attestation { attestHandler = &attestation.Handler{ @@ -357,15 +363,18 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { StatusUpdater: &StatusUpdater{Client: mgr.GetClient()}, } } + if components.sessionHTTP { capabilityKey, err := os.ReadFile(capabilityKeyFile) if err != nil { return fmt.Errorf("reading capability key: %w", err) } + capabilities, err := netboot.NewCapabilitySigner(capabilityKey, capabilityKeyID, nil) if err != nil { return fmt.Errorf("creating capability signer: %w", err) } + (&netboot.SessionHTTPServer{ Client: mgr.GetClient(), Cache: ociCache, @@ -378,6 +387,7 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { }, }).RegisterHandlers(httpMux) } + if attestHandler != nil && !components.sessionHTTP { httpMux.HandleFunc("POST /attest", attestHandler.Attest) } @@ -421,12 +431,15 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { if components.tftp { PrintService("TFTP", fmt.Sprintf("%s:69", bindAddress)) } + if components.http { PrintService("HTTP", fmt.Sprintf("%s:%d", bindAddress, httpPort)) } + if components.redfish { PrintService("Redfish", "reconciler") } + PrintReady() return mgr.Start(ctx) @@ -450,10 +463,12 @@ func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { cmd.Flags().DurationVar(&operationPollInterval, "operation-poll-interval", 5*time.Second, "Poll interval for in-progress MachineOperations") cmd.Flags().StringVar(&defaultNetbootImage, "default-netboot-image", DefaultNetbootImage, "Default OCI image containing PXE netboot artifacts") cmd.Flags().StringVar(&defaultNetbootPullSecret, "default-netboot-pull-secret", "", "Namespaced Secret reference (namespace/name) for pulling the default netboot OCI image") + if components.sessionHTTP || components.sessionManager { cmd.Flags().StringVar(&capabilityKeyFile, "capability-key-file", "/var/run/secrets/metalman/capability.key", "File containing the shared capability HMAC key") cmd.Flags().StringVar(&capabilityKeyID, "capability-key-id", "v1", "Identifier for the active capability HMAC key") } + if components.sessionHTTP { cmd.Flags().StringVar(&edgeServiceAccount, "edge-service-account", "metalman-edge", "ServiceAccount name accepted by internal edge APIs") } diff --git a/internal/metalman/dhcp/backend.go b/internal/metalman/dhcp/backend.go index 1d069229d..aa9114fd8 100644 --- a/internal/metalman/dhcp/backend.go +++ b/internal/metalman/dhcp/backend.go @@ -29,10 +29,12 @@ func NewHTTPDecisionProviderFromTokenFile(backendURL, endpoint, tokenFile string if strings.TrimSpace(tokenFile) == "" { return nil, errors.New("edge authentication token file is required") } + provider, err := newHTTPDecisionProvider(backendURL, endpoint, client) if err != nil { return nil, err } + provider.tokenFile = tokenFile return provider, nil @@ -43,9 +45,11 @@ func NewHTTPDecisionProvider(backendURL, endpoint, token string, client *http.Cl if err != nil { return nil, err } + if strings.TrimSpace(token) == "" { return nil, errors.New("edge authentication token is required") } + provider.token = token return provider, nil @@ -56,12 +60,15 @@ func newHTTPDecisionProvider(backendURL, endpoint string, client *http.Client) ( if err != nil { return nil, fmt.Errorf("parsing backend URL: %w", err) } + if (backend.Scheme != "http" && backend.Scheme != "https") || backend.Host == "" { return nil, errors.New("backend URL must use HTTP or HTTPS and include a host") } + if strings.TrimSpace(endpoint) == "" { return nil, errors.New("netboot endpoint name is required") } + if client == nil { client = http.DefaultClient } @@ -80,14 +87,17 @@ func (p *HTTPDecisionProvider) Decide(ctx context.Context, mac string, httpClien if err != nil { return nil, fmt.Errorf("creating DHCP decision request: %w", err) } + token := p.token if p.tokenFile != "" { tokenBytes, err := os.ReadFile(p.tokenFile) if err != nil { return nil, fmt.Errorf("reading edge authentication token: %w", err) } + token = strings.TrimSpace(string(tokenBytes)) } + request.Header.Set("Authorization", "Bearer "+token) response, err := p.client.Do(request) @@ -99,6 +109,7 @@ func (p *HTTPDecisionProvider) Decide(ctx context.Context, mac string, httpClien if response.StatusCode == http.StatusNotFound { return nil, nil } + if response.StatusCode != http.StatusOK { return nil, fmt.Errorf("DHCP decision server returned %s", response.Status) } diff --git a/internal/metalman/dhcp/backend_test.go b/internal/metalman/dhcp/backend_test.go index ec2c258f8..42ebf132a 100644 --- a/internal/metalman/dhcp/backend_test.go +++ b/internal/metalman/dhcp/backend_test.go @@ -20,12 +20,15 @@ func TestHTTPDecisionProviderAuthenticatesEndpointRequest(t *testing.T) { if got := r.URL.Path; got != "/v1/netboot/endpoints/edge-1/dhcp/aa:bb:cc:dd:ee:ff" { t.Errorf("path = %q", got) } + if got := r.URL.Query().Get("httpClient"); got != "true" { t.Errorf("httpClient = %q", got) } + if got := r.Header.Get("Authorization"); got != "Bearer edge-token" { t.Errorf("Authorization = %q", got) } + w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"lease":{"mac":"aa:bb:cc:dd:ee:ff","ipv4":"10.0.1.20","subnetMask":"255.255.255.0"},"transport":"HTTP","bootFile":"https://boot.example/shim.efi"}`)) })) @@ -35,10 +38,12 @@ func TestHTTPDecisionProviderAuthenticatesEndpointRequest(t *testing.T) { if err != nil { t.Fatal(err) } + decision, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", true) if err != nil { t.Fatal(err) } + if decision.Transport != v1alpha3.NetbootTransportHTTP || decision.Lease.IPv4 != "10.0.1.20" { t.Fatalf("decision = %#v", decision) } @@ -54,10 +59,12 @@ func TestHTTPDecisionProviderTreatsMissingSessionAsNoDecision(t *testing.T) { if err != nil { t.Fatal(err) } + decision, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false) if err != nil { t.Fatal(err) } + if decision != nil { t.Fatalf("decision = %#v, want nil", decision) } @@ -70,11 +77,14 @@ func TestHTTPDecisionProviderReloadsProjectedToken(t *testing.T) { if err := os.WriteFile(tokenFile, []byte("first"), 0o600); err != nil { t.Fatal(err) } + wantToken := "first" + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if got := r.Header.Get("Authorization"); got != "Bearer "+wantToken { t.Errorf("Authorization = %q, want token %q", got, wantToken) } + http.NotFound(w, r) })) defer backend.Close() @@ -83,13 +93,16 @@ func TestHTTPDecisionProviderReloadsProjectedToken(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false); err != nil { t.Fatal(err) } + wantToken = "second" if err := os.WriteFile(tokenFile, []byte(wantToken), 0o600); err != nil { t.Fatal(err) } + if _, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false); err != nil { t.Fatal(err) } diff --git a/internal/metalman/dhcp/dhcp.go b/internal/metalman/dhcp/dhcp.go index 8f9d75e26..45cfa8247 100644 --- a/internal/metalman/dhcp/dhcp.go +++ b/internal/metalman/dhcp/dhcp.go @@ -144,9 +144,11 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { log.Error("resolving DHCP decision", "err", err) return } + if decision == nil { return } + lease := &decision.Lease clientIP := net.ParseIP(lease.IPv4).To4() @@ -207,6 +209,7 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { if netbootImage == "" || s.OCICache == nil { goto send } + architecture := node.Spec.Netboot().TargetArchitecture() meta, err := s.OCICache.MetadataForRefArchitecture(netbootImage, architecture) @@ -255,6 +258,7 @@ func (s *Server) decision(ctx context.Context, mac string, httpClient bool) (*De decision, err := s.DecisionProvider.Decide(ctx, mac, httpClient) return decision, nil, err } + if s.Reader == nil { return nil, nil, errors.New("DHCP decision provider is not configured") } @@ -263,6 +267,7 @@ func (s *Server) decision(ctx context.Context, mac string, httpClient bool) (*De if err := s.Reader.List(ctx, &list, client.MatchingFields{indexing.IndexNodeByMAC: mac}); err != nil { return nil, nil, fmt.Errorf("listing Machines by MAC: %w", err) } + if len(list.Items) == 0 || list.Items[0].Spec.Netboot() == nil { return nil, nil, nil } diff --git a/internal/metalman/dhcp/dhcp_test.go b/internal/metalman/dhcp/dhcp_test.go index b20004295..a7d7d52fb 100644 --- a/internal/metalman/dhcp/dhcp_test.go +++ b/internal/metalman/dhcp/dhcp_test.go @@ -421,6 +421,7 @@ func TestDHCPHandlerUsesBackendSessionDecision(t *testing.T) { if err != nil { t.Fatal(err) } + serverIP := net.ParseIP("10.0.1.254").To4() provider := &fakeDecisionProvider{decision: &Decision{ Lease: v1alpha3.DHCPLease{ @@ -442,6 +443,7 @@ func TestDHCPHandlerUsesBackendSessionDecision(t *testing.T) { if err != nil { t.Fatal(err) } + discover.UpdateOption(dhcpv4.OptClassIdentifier("HTTPClient:Arch:00016:UNDI:003016")) conn := &fakePacketConn{} @@ -450,19 +452,24 @@ func TestDHCPHandlerUsesBackendSessionDecision(t *testing.T) { if provider.mac != mac.String() { t.Errorf("provider MAC = %q, want %q", provider.mac, mac.String()) } + if !provider.httpClient { t.Error("provider did not receive HTTP client identity") } + if conn.written == nil { t.Fatal("expected DHCP response, got none") } + response, err := dhcpv4.FromBytes(conn.written) if err != nil { t.Fatal(err) } + if got := response.BootFileNameOption(); got != provider.decision.BootFile { t.Errorf("bootfile = %q, want %q", got, provider.decision.BootFile) } + if got := response.YourIPAddr.String(); got != provider.decision.Lease.IPv4 { t.Errorf("lease IP = %q, want %q", got, provider.decision.Lease.IPv4) } diff --git a/internal/metalman/machineops/controller.go b/internal/metalman/machineops/controller.go index 960863511..b3d73d075 100644 --- a/internal/metalman/machineops/controller.go +++ b/internal/metalman/machineops/controller.go @@ -399,8 +399,10 @@ func (r *Reconciler) advanceTarget(ctx context.Context, op *v1alpha3.MachineOper return r.advanceReboot(ctx, op, &machine, target, now) case v1alpha3.OperationHostReplace: var session *v1alpha3.NetbootSession + if r.Sessions != nil { var err error + session, err = r.Sessions.Ensure(ctx, op, &machine) if err != nil { if errors.Is(err, netboot.ErrNotYetDownloaded) { @@ -641,11 +643,13 @@ func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, ma transport := netbootSpec.TargetTransport() configurationSource := targetConfigurationSource(netbootSpec) networkMode := targetNetworkMode(netbootSpec) + if session != nil { transport = session.Spec.Boot.Transport configurationSource = session.Spec.Boot.ConfigurationSource networkMode = session.Spec.Boot.NetworkMode } + if transport == v1alpha3.NetbootTransportHTTP { if configurationSource == v1alpha3.NetbootConfigurationSourceDHCP { return pc.SetBootOverride(ctx, redfish.BootTargetUefiHTTP, redfish.BootOnce) @@ -671,17 +675,21 @@ func (r *Reconciler) httpBootConfig(machine *v1alpha3.Machine, session *v1alpha3 bootURL string err error ) + if session != nil { if r.SessionHTTPBootURL == nil { return "", redfish.StaticIPv4Config{}, fmt.Errorf("session HTTP boot URL resolver is not configured") } + bootURL, err = r.SessionHTTPBootURL(session) } else { if r.HTTPBootURL == nil { return "", redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot URL resolver is not configured") } + bootURL, err = r.HTTPBootURL(machine) } + if err != nil { return "", redfish.StaticIPv4Config{}, err } @@ -704,6 +712,7 @@ func httpBootStaticNetworkConfig(machine *v1alpha3.Machine, session *v1alpha3.Ne } else if machine.Spec.Netboot() != nil { leases = machine.Spec.Netboot().DHCPLeases } + if len(leases) == 0 { return redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot requires at least one static lease in spec.host.netboot.dhcpLeases") } @@ -804,6 +813,7 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma if session != nil { transport = session.Spec.Boot.Transport } + if transport == v1alpha3.NetbootTransportHTTP { if _, _, err := r.httpBootConfig(machine, session); err != nil { if errors.Is(err, netboot.ErrNotYetDownloaded) { diff --git a/internal/metalman/machineops/controller_test.go b/internal/metalman/machineops/controller_test.go index b06055442..8e05a852f 100644 --- a/internal/metalman/machineops/controller_test.go +++ b/internal/metalman/machineops/controller_test.go @@ -465,6 +465,7 @@ func TestConfigureRepaveBootSupportsIndependentBootAxes(t *testing.T) { t.Parallel() const bootURL = "https://boot.example.com/v1/netboot/sessions/session/capability/artifacts/bootx64.efi" + tests := []struct { name string transport v1alpha3.NetbootTransport @@ -522,6 +523,7 @@ func TestConfigureRepaveBootSupportsIndependentBootAxes(t *testing.T) { power := &recordingPowerClient{} client, err := power.ForMachine(t.Context(), &v1alpha3.Machine{ObjectMeta: metav1.ObjectMeta{Name: "machine"}}) require.NoError(t, err) + resolved := false reconciler := &Reconciler{SessionHTTPBootURL: func(*v1alpha3.NetbootSession) (string, error) { resolved = true @@ -1538,10 +1540,12 @@ func markTargetCondition(t *testing.T, c client.Client, opName, machineName, con var op v1alpha3.MachineOperation require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: opName}, &op)) + for i := range op.Status.Targets { if op.Status.Targets[i].MachineRef != machineName { continue } + apimeta.SetStatusCondition(&op.Status.Targets[i].Conditions, metav1.Condition{ Type: conditionType, Status: status, @@ -1553,6 +1557,7 @@ func markTargetCondition(t *testing.T, c client.Client, opName, machineName, con return } + t.Fatalf("operation %s has no target %s", opName, machineName) } diff --git a/internal/metalman/machineops/session_manager.go b/internal/metalman/machineops/session_manager.go index c1281e0cd..b33a11d3e 100644 --- a/internal/metalman/machineops/session_manager.go +++ b/internal/metalman/machineops/session_manager.go @@ -67,7 +67,7 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph netbootSpec := machine.Spec.Netboot() if netbootSpec == nil { - return nil, errors.New("Machine has no netboot configuration") + return nil, errors.New("machine has no netboot configuration") } var endpoint v1alpha3.NetbootEndpoint @@ -76,6 +76,7 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph } architecture := netbootSpec.TargetArchitecture() + machineDigest := m.Cache.DigestForArchitecture(netbootSpec.Image, architecture) if machineDigest == "" { return nil, fmt.Errorf("%w: machine image %q for architecture %q", netboot.ErrNotYetDownloaded, netbootSpec.Image, architecture) @@ -83,6 +84,7 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph netbootRef := netbootSpec.NetbootImage netbootPullSecret := netbootSpec.NetbootPullSecretRef + if netbootRef == "" { netbootRef = m.DefaultNetbootRef netbootPullSecret = m.DefaultNetbootPullSecret @@ -92,24 +94,29 @@ func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alph if netbootDigest == "" { return nil, fmt.Errorf("%w: netboot image %q for architecture %q", netboot.ErrNotYetDownloaded, netbootRef, architecture) } + metadata, err := m.Cache.MetadataForArchitecture(netbootDigest, architecture) if err != nil { return nil, fmt.Errorf("read netboot image metadata: %w", err) } + firmwareArtifact := firmwareArtifactForTransport(metadata, netbootSpec.TargetTransport()) if firmwareArtifact == "" { return nil, fmt.Errorf("netboot image %q has no firmware artifact for %s", netbootRef, netbootSpec.TargetTransport()) } now := m.now() + userData, err := m.resolveUserData(ctx, netbootSpec) if err != nil { return nil, err } + clusterInfo := netboot.ClusterInfo{} if m.Cluster != nil { clusterInfo = m.Cluster.ClusterInfo() } + session := &v1alpha3.NetbootSession{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -185,6 +192,7 @@ func (m *KubernetesSessionManager) resolveUserData(ctx context.Context, spec *v1 } ref := spec.CloudInit.UserDataConfigMapRef + var configMap corev1.ConfigMap if err := m.Client.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, &configMap); err != nil { return "", fmt.Errorf("get cloud-init user-data ConfigMap %s/%s: %w", ref.Namespace, ref.Name, err) @@ -194,9 +202,11 @@ func (m *KubernetesSessionManager) resolveUserData(ctx context.Context, spec *v1 if key == "" { key = "user-data" } + if value, ok := configMap.Data[key]; ok { return value, nil } + if value, ok := configMap.BinaryData[key]; ok { return string(value), nil } @@ -221,6 +231,7 @@ func (m *KubernetesSessionManager) refreshEndpointReadinessWithEndpoint(ctx cont now := m.now() ready := endpoint.Status.ObservedGeneration == endpoint.Generation && apimeta.IsStatusConditionTrue(endpoint.Status.Conditions, "Ready") + session.Status.Phase = v1alpha3.NetbootSessionPhasePreparing if ready { session.Status.Phase = v1alpha3.NetbootSessionPhaseReady @@ -303,11 +314,13 @@ func sessionArtifacts(firmwareArtifact string) []v1alpha3.NetbootSessionArtifact } unique := make([]v1alpha3.NetbootSessionArtifact, 0, len(artifacts)) + names := make(map[string]struct{}, len(artifacts)) for _, artifact := range artifacts { if _, exists := names[artifact.Name]; exists { continue } + names[artifact.Name] = struct{}{} unique = append(unique, artifact) } @@ -319,6 +332,7 @@ func firmwareArtifactForTransport(metadata *netboot.ImageMetadata, transport v1a if metadata == nil { return "" } + if transport == v1alpha3.NetbootTransportHTTP { return netboot.HTTPBootPathFromMetadata(metadata) } diff --git a/internal/metalman/machineops/session_manager_test.go b/internal/metalman/machineops/session_manager_test.go index 06db36c40..7c784937f 100644 --- a/internal/metalman/machineops/session_manager_test.go +++ b/internal/metalman/machineops/session_manager_test.go @@ -45,6 +45,7 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { cache := netboot.NewOCICache(t.TempDir()) machineDigest := "sha256:" + stringOf('a', 64) netbootDigest := "sha256:" + stringOf('b', 64) + cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, machineDigest) cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, netbootDigest) require.NoError(t, writeSessionMetadata(cache, netbootDigest, "http/bootx64.efi")) @@ -86,6 +87,7 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { Source: "NetbootImage", Path: "/disk/http/bootx64.efi", }) + for _, name := range []string{ "vmlinuz", "initrd", "init.cpio", "grub/grub.cfg", "grubx64.efi", "cloud-init/meta-data", "cloud-init/user-data", "cloud-init/vendor-data", "cloud-init/network-config", @@ -100,6 +102,7 @@ func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { return false }, "session artifact list is missing %s", name) } + require.True(t, session.Spec.ExpiresAt.Time.Equal(fixedNow().Add(24*time.Hour))) machine.Spec.PXE.Image = "ghcr.io/test/changed:v2" diff --git a/internal/metalman/machineops/session_status.go b/internal/metalman/machineops/session_status.go index 4b205f831..80a4e4a8d 100644 --- a/internal/metalman/machineops/session_status.go +++ b/internal/metalman/machineops/session_status.go @@ -33,24 +33,30 @@ func (r *SessionStatusRecorder) RecordCondition(ctx context.Context, sessionName if err := r.Client.Get(ctx, client.ObjectKey{Name: sessionName}, &session); err != nil { return fmt.Errorf("get NetbootSession %s: %w", sessionName, err) } + if session.UID != sessionUID { return fmt.Errorf("NetbootSession %s identity changed", sessionName) } condition.ObservedGeneration = session.Generation condition.LastTransitionTime = r.now() + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var latest v1alpha3.NetbootSession if err := r.Client.Get(ctx, client.ObjectKey{Name: sessionName}, &latest); err != nil { return err } + if latest.UID != sessionUID { return fmt.Errorf("NetbootSession %s identity changed", sessionName) } + if apimeta.IsStatusConditionTrue(latest.Status.Conditions, condition.Type) { return nil } + apimeta.SetStatusCondition(&latest.Status.Conditions, condition) + if latest.Status.Phase == v1alpha3.NetbootSessionPhaseReady { latest.Status.Phase = v1alpha3.NetbootSessionPhaseActive } @@ -70,8 +76,10 @@ func (r *SessionStatusRecorder) recordTargetCondition(ctx context.Context, sessi if apierrors.IsNotFound(err) { return nil } + return err } + if operation.UID != session.Spec.Operation.UID { return fmt.Errorf("MachineOperation %s identity changed", operation.Name) } @@ -81,9 +89,11 @@ func (r *SessionStatusRecorder) recordTargetCondition(ctx context.Context, sessi if target.Input == nil || target.Input.NetbootSessionRef == nil || target.Input.NetbootSessionRef.Name != session.Name || target.Input.NetbootSessionRef.UID != session.UID { continue } + if apimeta.IsStatusConditionTrue(target.Conditions, condition.Type) { return nil } + condition.ObservedGeneration = target.ObservedGeneration apimeta.SetStatusCondition(&target.Conditions, condition) diff --git a/internal/metalman/netboot/capability.go b/internal/metalman/netboot/capability.go index 2e438d65b..944434c73 100644 --- a/internal/metalman/netboot/capability.go +++ b/internal/metalman/netboot/capability.go @@ -37,9 +37,11 @@ func NewCapabilitySigner(key []byte, keyID string, now func() time.Time) (*Capab if len(key) < minimumCapabilityKeySize { return nil, fmt.Errorf("capability HMAC key must be at least %d bytes", minimumCapabilityKeySize) } + if strings.TrimSpace(keyID) == "" { return nil, errors.New("capability key ID is required") } + if now == nil { now = time.Now } @@ -58,12 +60,14 @@ func (s *CapabilitySigner) Sign(session *v1alpha3.NetbootSession) (string, error SessionUID: string(session.UID), Expires: session.Spec.ExpiresAt.Unix(), } + payload, err := json.Marshal(claims) if err != nil { return "", fmt.Errorf("marshal capability claims: %w", err) } encoded := base64.RawURLEncoding.EncodeToString(payload) + return encoded + "." + s.signature(encoded), nil } @@ -77,13 +81,16 @@ func (s *CapabilitySigner) Verify(session *v1alpha3.NetbootSession, capability s if err != nil { return errors.New("invalid capability") } + var claims capabilityClaims if err := json.Unmarshal(payload, &claims); err != nil { return errors.New("invalid capability") } + if session == nil || claims.KeyID != s.keyID || claims.Session != session.Name || claims.SessionUID != string(session.UID) || claims.Expires != session.Spec.ExpiresAt.Unix() { return errors.New("invalid capability") } + if !s.now().Before(time.Unix(claims.Expires, 0)) { return errors.New("expired capability") } diff --git a/internal/metalman/netboot/edge_auth.go b/internal/metalman/netboot/edge_auth.go index 5cadcf322..da9fe57b1 100644 --- a/internal/metalman/netboot/edge_auth.go +++ b/internal/metalman/netboot/edge_auth.go @@ -26,6 +26,7 @@ func (a *TokenReviewEdgeAuthenticator) Authenticate(ctx context.Context, request if a == nil || a.Client == nil || strings.TrimSpace(a.ServiceAccountName) == "" { return false } + token, ok := strings.CutPrefix(request.Header.Get("Authorization"), "Bearer ") if !ok || strings.TrimSpace(token) == "" { return false @@ -42,6 +43,7 @@ func (a *TokenReviewEdgeAuthenticator) Authenticate(ctx context.Context, request if !strings.HasPrefix(review.Status.User.Username, prefix) { return false } + parts := strings.Split(strings.TrimPrefix(review.Status.User.Username, prefix), ":") return len(parts) == 2 && parts[1] == a.ServiceAccountName diff --git a/internal/metalman/netboot/edge_auth_test.go b/internal/metalman/netboot/edge_auth_test.go index b6be81e2e..64638080b 100644 --- a/internal/metalman/netboot/edge_auth_test.go +++ b/internal/metalman/netboot/edge_auth_test.go @@ -20,6 +20,7 @@ func TestTokenReviewEdgeAuthenticatorRequiresAudienceAndServiceAccount(t *testin clientset := fake.NewClientset() clientset.PrependReactor("create", "tokenreviews", func(action clienttesting.Action) (bool, runtime.Object, error) { create := action.(clienttesting.CreateAction) + review := create.GetObject().(*authenticationv1.TokenReview) if len(review.Spec.Audiences) != 1 || review.Spec.Audiences[0] != EdgeTokenAudience { t.Errorf("audiences = %v", review.Spec.Audiences) diff --git a/internal/metalman/netboot/netboot.go b/internal/metalman/netboot/netboot.go index 4621920e7..4c9b046e9 100644 --- a/internal/metalman/netboot/netboot.go +++ b/internal/metalman/netboot/netboot.go @@ -454,12 +454,15 @@ func renderTemplate(tmplStr string, data templateData) ([]byte, error) { if data.ArtifactBaseURL == "" { data.ArtifactBaseURL = serveURL } + if data.BootImageWrittenURL == "" { data.BootImageWrittenURL = serveURL + "/pxe/disable" } + if data.CloudInitURL == "" { data.CloudInitURL = serveURL + "/cloudinit/log" } + if data.InstallLogURL == "" { data.InstallLogURL = serveURL + "/unbounded-agent/install-log" } diff --git a/internal/metalman/netboot/oci_cache.go b/internal/metalman/netboot/oci_cache.go index cdd83a5a0..8f53137dd 100644 --- a/internal/metalman/netboot/oci_cache.go +++ b/internal/metalman/netboot/oci_cache.go @@ -202,6 +202,7 @@ func (c *OCICache) ResolveDigestPathForArchitecture(digest, architecture, reqPat if digest == "" { return "", false, fmt.Errorf("image digest is required") } + if !c.IsCachedForArchitecture(digest, architecture) { return "", false, fmt.Errorf("%w: image digest %q for architecture %q is not cached", ErrNotYetDownloaded, digest, normalizeArchitecture(architecture)) } diff --git a/internal/metalman/netboot/session_dhcp_test.go b/internal/metalman/netboot/session_dhcp_test.go index bd24de6ac..a337c6699 100644 --- a/internal/metalman/netboot/session_dhcp_test.go +++ b/internal/metalman/netboot/session_dhcp_test.go @@ -34,16 +34,20 @@ func TestSessionDHCPDecisionUsesReadyImmutableSession(t *testing.T) { if response.Code != http.StatusOK { t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) } + var decision dhcp.Decision if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { t.Fatal(err) } + if decision.Lease.IPv4 != "10.0.1.20" { t.Errorf("lease IP = %q", decision.Lease.IPv4) } + if decision.Transport != v1alpha3.NetbootTransportHTTP { t.Errorf("transport = %q", decision.Transport) } + wantPrefix := "https://boot.example/v1/netboot/sessions/session-1/" if len(decision.BootFile) <= len(wantPrefix) || decision.BootFile[:len(wantPrefix)] != wantPrefix { t.Errorf("boot file = %q, want prefix %q", decision.BootFile, wantPrefix) @@ -66,8 +70,10 @@ func TestSessionDHCPDecisionRequiresEdgeAuthentication(t *testing.T) { request = httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) request.Header.Set("Authorization", "Bearer edge-token") + response = httptest.NewRecorder() server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusOK { t.Fatalf("authenticated status = %d, want %d", response.Code, http.StatusOK) } @@ -105,10 +111,12 @@ func TestSessionDHCPDecisionOmitsBootFileForRedfishConfiguration(t *testing.T) { if response.Code != http.StatusOK { t.Fatalf("status = %d, want %d", response.Code, http.StatusOK) } + var decision dhcp.Decision if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { t.Fatal(err) } + if decision.BootFile != "" { t.Errorf("boot file = %q, want empty", decision.BootFile) } @@ -137,13 +145,16 @@ func newSessionDHCPTestServer(t *testing.T, sessions ...*v1alpha3.NetbootSession if err := v1alpha3.AddToScheme(scheme); err != nil { t.Fatal(err) } + objects := make([]runtime.Object, len(sessions)) for i := range sessions { objects[i] = sessions[i] } + clientObjects := make([]runtime.Object, 0, len(objects)) clientObjects = append(clientObjects, objects...) client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(clientObjects...).Build() + signer, err := netboot.NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test", func() time.Time { return time.Unix(1000, 0) }) diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go index 2f6aa038a..548c5597f 100644 --- a/internal/metalman/netboot/session_http.go +++ b/internal/metalman/netboot/session_http.go @@ -57,13 +57,16 @@ func SessionArtifactURL(signer *CapabilitySigner, session *v1alpha3.NetbootSessi if signer == nil || session == nil { return "", errors.New("capability signer and session are required") } + if _, ok := sessionArtifact(session, artifactName); !ok { return "", fmt.Errorf("artifact %q is not listed by session %s", artifactName, session.Name) } + cleanArtifact := strings.TrimPrefix(artifactName, "/") if cleanArtifact == "" || pathpkg.Clean(cleanArtifact) != cleanArtifact || strings.HasPrefix(artifactName, "/") { return "", fmt.Errorf("invalid artifact name %q", artifactName) } + baseURL, err := SessionBaseURL(signer, session) if err != nil { return "", err @@ -78,6 +81,7 @@ func SessionBaseURL(signer *CapabilitySigner, session *v1alpha3.NetbootSession) if signer == nil || session == nil { return "", errors.New("capability signer and session are required") } + capability, err := signer.Sign(session) if err != nil { return "", err @@ -107,6 +111,7 @@ func (s *SessionHTTPServer) handleDHCPDecision(w http.ResponseWriter, r *http.Re http.Error(w, "session server unavailable", http.StatusServiceUnavailable) return } + if s.EdgeAuthenticator != nil && !s.EdgeAuthenticator.Authenticate(r.Context(), r) { http.Error(w, "unauthorized", http.StatusUnauthorized) return @@ -114,6 +119,7 @@ func (s *SessionHTTPServer) handleDHCPDecision(w http.ResponseWriter, r *http.Re endpoint := r.PathValue("endpoint") mac := strings.ToLower(r.PathValue("mac")) + var sessions v1alpha3.NetbootSessionList if err := s.Client.List(r.Context(), &sessions); err != nil { http.Error(w, "loading sessions", http.StatusServiceUnavailable) @@ -121,11 +127,13 @@ func (s *SessionHTTPServer) handleDHCPDecision(w http.ResponseWriter, r *http.Re } matches := make([]*v1alpha3.NetbootSession, 0, 1) + for i := range sessions.Items { session := &sessions.Items[i] if session.Spec.Endpoint.Name != endpoint || s.Capabilities.IsExpired(session) || (session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive) { continue } + for _, lease := range session.Spec.Boot.DHCPLeases { if strings.EqualFold(lease.MAC, mac) { matches = append(matches, session) @@ -133,47 +141,57 @@ func (s *SessionHTTPServer) handleDHCPDecision(w http.ResponseWriter, r *http.Re } } } + if len(matches) == 0 { http.NotFound(w, r) return } + if len(matches) != 1 { http.Error(w, "multiple ready sessions match DHCP client", http.StatusConflict) return } session := matches[0] + var lease *v1alpha3.DHCPLease + for i := range session.Spec.Boot.DHCPLeases { if strings.EqualFold(session.Spec.Boot.DHCPLeases[i].MAC, mac) { lease = &session.Spec.Boot.DHCPLeases[i] break } } + if lease == nil { http.NotFound(w, r) return } bootFile := "" + if session.Spec.Boot.ConfigurationSource == v1alpha3.NetbootConfigurationSourceDHCP { var err error + bootFile, err = SessionArtifactURL(s.Capabilities, session, session.Spec.Boot.FirmwareArtifact) if err != nil { http.Error(w, "building firmware URL", http.StatusServiceUnavailable) return } + if session.Spec.Boot.Transport == v1alpha3.NetbootTransportTFTP { capability, err := s.Capabilities.Sign(session) if err != nil { http.Error(w, "building firmware capability", http.StatusServiceUnavailable) return } + bootFile = pathpkg.Join("v1/netboot/sessions", session.Name, capability, "artifacts", session.Spec.Boot.FirmwareArtifact) } } w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(struct { Lease v1alpha3.DHCPLease `json:"lease"` Transport v1alpha3.NetbootTransport `json:"transport"` @@ -188,6 +206,7 @@ func (s *SessionHTTPServer) handleCallback(w http.ResponseWriter, r *http.Reques if !ok { return } + if s.StatusRecorder == nil { http.Error(w, "session status unavailable", http.StatusServiceUnavailable) return @@ -204,6 +223,7 @@ func (s *SessionHTTPServer) handleCallback(w http.ResponseWriter, r *http.Reques http.NotFound(w, r) return } + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, metav1.Condition{ Type: conditionType, Status: metav1.ConditionTrue, @@ -223,11 +243,13 @@ func (s *SessionHTTPServer) handleSessionCloudInit(w http.ResponseWriter, r *htt http.Error(w, "reading cloud-init event", http.StatusBadRequest) return } + var event cloudInitEvent if err := json.Unmarshal(body, &event); err != nil { http.Error(w, "invalid cloud-init event", http.StatusBadRequest) return } + condition := buildCloudInitCondition(&event, session.Spec.Machine.Generation) if condition != nil { condition.Type = v1alpha3.NetbootSessionConditionCloudInitDone @@ -236,6 +258,7 @@ func (s *SessionHTTPServer) handleSessionCloudInit(w http.ResponseWriter, r *htt return } } + w.WriteHeader(http.StatusNoContent) } @@ -244,11 +267,13 @@ func (s *SessionHTTPServer) handleInstallLog(w http.ResponseWriter, r *http.Requ if !ok { return } + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) if err != nil { http.Error(w, "reading install log", http.StatusBadRequest) return } + slog.Warn("unbounded-agent install log", "session", session.Name, "body", strings.TrimSpace(string(body))) w.WriteHeader(http.StatusNoContent) } @@ -258,6 +283,7 @@ func (s *SessionHTTPServer) handleAttest(w http.ResponseWriter, r *http.Request) if !ok { return } + if s.Attestation == nil || s.StatusRecorder == nil { http.Error(w, "session attestation unavailable", http.StatusServiceUnavailable) return @@ -269,9 +295,12 @@ func (s *SessionHTTPServer) handleAttest(w http.ResponseWriter, r *http.Request) http.NotFound(w, r) return } + http.Error(w, "loading Machine", http.StatusServiceUnavailable) + return } + if machine.UID != session.Spec.Machine.UID { http.Error(w, "Machine identity changed", http.StatusConflict) return @@ -279,10 +308,12 @@ func (s *SessionHTTPServer) handleAttest(w http.ResponseWriter, r *http.Request) response := newBufferedResponseWriter() s.Attestation.AttestMachine(response, r, &machine) + if response.status < http.StatusOK || response.status >= http.StatusMultipleChoices { response.writeTo(w) return } + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, metav1.Condition{ Type: v1alpha3.NetbootSessionConditionAttested, Status: metav1.ConditionTrue, @@ -292,6 +323,7 @@ func (s *SessionHTTPServer) handleAttest(w http.ResponseWriter, r *http.Request) http.Error(w, "recording session status", http.StatusServiceUnavailable) return } + response.writeTo(w) } @@ -311,14 +343,18 @@ func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Reques http.NotFound(w, r) return } + if artifact.Source == "Session" { if artifact.Name != "cloud-init/user-data" { http.NotFound(w, r) return } + http.ServeContent(w, r, artifact.Name, session.CreationTimestamp.Time, strings.NewReader(session.Spec.Provisioning.UserData)) + return } + image, ok := sessionArtifactImage(session, artifact.Source) if !ok { http.NotFound(w, r) @@ -330,24 +366,31 @@ func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Reques http.NotFound(w, r) return } + diskPath, isTemplate, err := s.Cache.ResolveDigestPathForArchitecture(image.Digest, session.Spec.Boot.Architecture, reqPath) if err != nil { if errors.Is(err, ErrNotYetDownloaded) { w.Header().Set("Retry-After", "5") http.Error(w, "artifact unavailable", http.StatusServiceUnavailable) + return } + http.NotFound(w, r) + return } + if isTemplate { data, err := s.renderSessionTemplate(diskPath, session) if err != nil { http.Error(w, "rendering artifact", http.StatusServiceUnavailable) return } + http.ServeContent(w, r, artifact.Name, session.CreationTimestamp.Time, bytes.NewReader(data)) s.recordFirmwareDownloaded(r.Context(), session, artifact.Name) + return } @@ -359,6 +402,7 @@ func (s *SessionHTTPServer) recordFirmwareDownloaded(ctx context.Context, sessio if s.StatusRecorder == nil || artifactName != session.Spec.Boot.FirmwareArtifact { return } + if err := s.StatusRecorder.RecordCondition(ctx, session.Name, session.UID, metav1.Condition{ Type: v1alpha3.NetbootSessionConditionBootLoaderDownloaded, Status: metav1.ConditionTrue, @@ -374,10 +418,12 @@ func (s *SessionHTTPServer) renderSessionTemplate(templatePath string, session * if err != nil { return nil, fmt.Errorf("reading template: %w", err) } + baseURL, err := SessionBaseURL(s.Capabilities, session) if err != nil { return nil, err } + machine := sessionMachine(session) cluster := session.Spec.Provisioning.Cluster agentConfig := provision.BuildAgentConfig(provision.BuildAgentConfigParams{ @@ -391,23 +437,29 @@ func (s *SessionHTTPServer) renderSessionTemplate(templatePath string, session * ProviderLabels: session.Spec.Provisioning.ProviderLabels, AttestURL: baseURL, }) + agentConfigJSON, err := json.MarshalIndent(agentConfig, " ", " ") if err != nil { return nil, fmt.Errorf("marshaling agent config: %w", err) } + data := newTemplateData(machine, ClusterInfo{ApiserverURL: cluster.APIServerURL, CACertBase64: cluster.CACertBase64}, baseURL, string(agentConfigJSON), "", true) + data.ArtifactBaseURL, err = JoinServeURLPath(baseURL, "artifacts") if err != nil { return nil, err } + data.BootImageWrittenURL, err = JoinServeURLPath(baseURL, "callbacks/boot-image-written") if err != nil { return nil, err } + data.CloudInitURL, err = JoinServeURLPath(baseURL, "callbacks/cloud-init") if err != nil { return nil, err } + data.InstallLogURL, err = JoinServeURLPath(baseURL, "logs/agent-install") if err != nil { return nil, err @@ -446,13 +498,17 @@ func (s *SessionHTTPServer) authorizeSession(w http.ResponseWriter, r *http.Requ http.NotFound(w, r) return nil, false } + http.Error(w, "loading session", http.StatusServiceUnavailable) + return nil, false } + if err := s.Capabilities.Verify(&session, r.PathValue("capability")); err != nil { http.Error(w, "unauthorized", http.StatusUnauthorized) return nil, false } + if session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive { http.Error(w, "session unavailable", http.StatusServiceUnavailable) return nil, false @@ -500,7 +556,9 @@ func (w *bufferedResponseWriter) writeTo(destination http.ResponseWriter) { for key, values := range w.header { destination.Header()[key] = append([]string(nil), values...) } + destination.WriteHeader(w.status) + if _, err := destination.Write(w.body.Bytes()); err != nil { slog.Warn("writing buffered attestation response", "err", err) } diff --git a/internal/metalman/netboot/session_http_test.go b/internal/metalman/netboot/session_http_test.go index 98f011581..de31688ae 100644 --- a/internal/metalman/netboot/session_http_test.go +++ b/internal/metalman/netboot/session_http_test.go @@ -26,6 +26,7 @@ func TestSessionHTTPServesImmutableArtifactWithCapabilityAndRange(t *testing.T) t.Parallel() const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + cache := setupOCICache(t, "unused.example/image:latest", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", map[string][]byte{ "disk.img.gz": []byte("0123456789"), }) @@ -33,6 +34,7 @@ func TestSessionHTTPServesImmutableArtifactWithCapabilityAndRange(t *testing.T) "disk.img.gz": []byte("mutable-tag-content"), })) cache.SetDigest("unused.example/image:latest", "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") + session := testNetbootSession("session-a", digest) client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { @@ -45,6 +47,7 @@ func TestSessionHTTPServesImmutableArtifactWithCapabilityAndRange(t *testing.T) handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() request := httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-a/"+capability+"/artifacts/disk.img.gz", nil) request.Header.Set("Range", "bytes=2-5") + response := httptest.NewRecorder() handler.ServeHTTP(response, request) @@ -57,6 +60,7 @@ func TestSessionHTTPRejectsInvalidExpiredAndUnlistedCapabilities(t *testing.T) { t.Parallel() const digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + cache := setupOCICache(t, "unused.example/image:latest", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", map[string][]byte{ "disk.img.gz": []byte("disk"), "secret": []byte("secret"), @@ -68,6 +72,7 @@ func TestSessionHTTPRejectsInvalidExpiredAndUnlistedCapabilities(t *testing.T) { require.NoError(t, err) capability, err := signer.Sign(session) require.NoError(t, err) + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() for name, test := range map[string]struct { @@ -105,6 +110,7 @@ func TestSessionHTTPRecordsAuthenticatedSessionCallback(t *testing.T) { require.NoError(t, err) capability, err := signer.Sign(session) require.NoError(t, err) + recorder := &recordingSessionConditionRecorder{} handler := (&SessionHTTPServer{Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, StatusRecorder: recorder}).Handler() @@ -129,6 +135,7 @@ func TestSessionHTTPRecordsCloudInitCompletionOnlyForFinalSuccess(t *testing.T) require.NoError(t, err) capability, err := signer.Sign(session) require.NoError(t, err) + recorder := &recordingSessionConditionsRecorder{} handler := (&SessionHTTPServer{Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, StatusRecorder: recorder}).Handler() path := "/v1/netboot/sessions/session-cloud-init/" + capability + "/callbacks/cloud-init" @@ -147,6 +154,7 @@ func TestSessionHTTPRecordsCloudInitCompletionOnlyForFinalSuccess(t *testing.T) require.Equal(t, metav1.ConditionFalse, recorder.conditions[0].Status) require.Equal(t, metav1.ConditionFalse, recorder.conditions[1].Status) require.Equal(t, metav1.ConditionTrue, recorder.conditions[2].Status) + for _, condition := range recorder.conditions { require.Equal(t, v1alpha3.NetbootSessionConditionCloudInitDone, condition.Type) } @@ -164,6 +172,7 @@ func TestSessionHTTPAttestsExactSessionMachineAndRecordsMilestone(t *testing.T) require.NoError(t, err) capability, err := signer.Sign(session) require.NoError(t, err) + attester := &recordingSessionAttester{} recorder := &recordingSessionConditionRecorder{} handler := (&SessionHTTPServer{ @@ -187,6 +196,7 @@ func TestSessionHTTPRecordsFirmwareDownloadForExactSession(t *testing.T) { t.Parallel() const digest = "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + cache := setupOCICache(t, "unused.example/netboot:latest", strings.TrimPrefix(digest, "sha256:"), map[string][]byte{ "bootx64.efi": []byte("firmware"), }) @@ -202,6 +212,7 @@ func TestSessionHTTPRecordsFirmwareDownloadForExactSession(t *testing.T) { require.NoError(t, err) capability, err := signer.Sign(session) require.NoError(t, err) + recorder := &recordingSessionConditionRecorder{} response := httptest.NewRecorder() @@ -239,6 +250,7 @@ func TestSessionHTTPRendersBootArtifactsFromImmutableSnapshot(t *testing.T) { t.Parallel() const digest = "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + cache := NewOCICache(t.TempDir()) diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.PXEArchitectureAMD64) require.NoError(t, os.MkdirAll(filepath.Join(diskDir, "grub"), 0o755)) @@ -291,16 +303,20 @@ func TestSessionHTTPRendersCapabilityScopedCallbacks(t *testing.T) { t.Parallel() const digest = "sha256:abababababababababababababababababababababababababababababababab" + cache := NewOCICache(t.TempDir()) diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.PXEArchitectureAMD64) require.NoError(t, os.MkdirAll(filepath.Join(diskDir, "cloud-init"), 0o755)) + for _, artifact := range []string{"grub.cfg", "vendor-data"} { source, err := os.ReadFile(filepath.Join("..", "..", "..", "images", "netboot", "assets", artifact+".tmpl")) require.NoError(t, err) + destination := filepath.Join(diskDir, artifact+".tmpl") if artifact == "vendor-data" { destination = filepath.Join(diskDir, "cloud-init", artifact+".tmpl") } + require.NoError(t, os.WriteFile(destination, source, 0o600)) } @@ -317,6 +333,7 @@ func TestSessionHTTPRendersCapabilityScopedCallbacks(t *testing.T) { require.NoError(t, err) capability, err := signer.Sign(session) require.NoError(t, err) + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() capabilityBase := "https://boot.example.com/v1/netboot/sessions/session-callbacks/" + capability @@ -350,6 +367,7 @@ type recordingSessionAttester struct { func (a *recordingSessionAttester) AttestMachine(w http.ResponseWriter, _ *http.Request, machine *v1alpha3.Machine) { a.machine = machine.DeepCopy() + w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"credentialBlob":"YQ=="}`)) } diff --git a/internal/metalman/netboot/tftp.go b/internal/metalman/netboot/tftp.go index eac1a56cd..a3e989039 100644 --- a/internal/metalman/netboot/tftp.go +++ b/internal/metalman/netboot/tftp.go @@ -48,6 +48,7 @@ func (t *TFTPServer) Start(ctx context.Context) error { if port == 0 { port = 69 } + addr := net.JoinHostPort(t.BindAddr, fmt.Sprint(port)) conn, err := net.ListenPacket("udp", addr) @@ -69,19 +70,23 @@ func (t *TFTPServer) readHandler(filename string, rf io.ReaderFrom) error { ctx := context.Background() ip := rf.(tftp.OutgoingTransfer).RemoteAddr().IP.String() //nolint:errcheck // Type is guaranteed by the tftp library. filename = strings.TrimPrefix(filename, "/") + log := slog.With("proto", "tftp", "filename", filename, "ip", ip) if t.Backend != nil { if !validSessionArtifactPath(filename) { return fmt.Errorf("invalid session artifact path %q", filename) } + reader, err := t.Backend.Open(ctx, filename) if err != nil { return fmt.Errorf("opening backend artifact: %w", err) } defer reader.Close() //nolint:errcheck // Backend stream is no longer needed. + if _, err := rf.ReadFrom(reader); err != nil { return fmt.Errorf("transferring backend artifact: %w", err) } + if recorder, ok := t.Backend.(TFTPBackendStatusRecorder); ok { if err := recorder.RecordBootLoaderDownloaded(ctx, filename); err != nil { return fmt.Errorf("recording backend transfer: %w", err) diff --git a/internal/metalman/netboot/tftp_backend.go b/internal/metalman/netboot/tftp_backend.go index 0d65de583..70e41ce1b 100644 --- a/internal/metalman/netboot/tftp_backend.go +++ b/internal/metalman/netboot/tftp_backend.go @@ -28,9 +28,11 @@ func NewHTTPArtifactBackend(backendURL string, client *http.Client) (*HTTPArtifa if err != nil { return nil, fmt.Errorf("parsing artifact backend URL: %w", err) } + if (backend.Scheme != "http" && backend.Scheme != "https") || backend.Host == "" { return nil, errors.New("artifact backend URL must use HTTP or HTTPS and include a host") } + if client == nil { client = http.DefaultClient } @@ -58,15 +60,18 @@ func (b *HTTPArtifactBackend) RecordBootLoaderDownloaded(ctx context.Context, fi requestURL := *b.backendURL requestURL.Path = pathpkg.Join(requestURL.Path, pathpkg.Join(parts[:5]...), "callbacks", "boot-loader-downloaded") + request, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), nil) if err != nil { return fmt.Errorf("creating TFTP milestone request: %w", err) } + response, err := b.client.Do(request) if err != nil { return fmt.Errorf("reporting TFTP milestone: %w", err) } defer response.Body.Close() //nolint:errcheck // Response body is not reused. + if response.StatusCode != http.StatusNoContent { return fmt.Errorf("reporting TFTP milestone: backend returned %s", response.Status) } @@ -88,16 +93,20 @@ func (r *resumingArtifactReader) Read(buffer []byte) (int, error) { for { n, err := r.body.Read(buffer) r.offset += int64(n) + truncated := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) if !truncated || r.offset > r.end { return n, err } + if n > 0 { return n, nil } + if r.attempts >= maxTFTPArtifactBackendAttempts { return 0, io.ErrUnexpectedEOF } + if err := r.open(r.offset, r.end); err != nil { if r.attempts >= maxTFTPArtifactBackendAttempts { return 0, err @@ -121,30 +130,37 @@ func (r *resumingArtifactReader) open(start, end int64) error { if err != nil { return err } + if end >= 0 { request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) } + response, err := r.client.Do(request) if err != nil { return fmt.Errorf("requesting TFTP artifact backend: %w", err) } + wantStatus := http.StatusOK if end >= 0 { wantStatus = http.StatusPartialContent } + if response.StatusCode != wantStatus || response.ContentLength <= 0 { response.Body.Close() //nolint:errcheck // Invalid response. return fmt.Errorf("TFTP artifact backend returned %s", response.Status) } + if end < 0 { end = response.ContentLength - 1 } else if response.ContentLength != end-start+1 || response.Header.Get("Content-Range") != fmt.Sprintf("bytes %d-%d/%d", start, end, end+1) { response.Body.Close() //nolint:errcheck // Invalid range response. return errors.New("TFTP artifact backend returned a mismatched range") } + if r.body != nil { r.body.Close() //nolint:errcheck // Previous response reached EOF. } + r.body = response.Body r.end = end diff --git a/internal/metalman/netboot/tftp_backend_test.go b/internal/metalman/netboot/tftp_backend_test.go index da36a7f26..388b0e1fc 100644 --- a/internal/metalman/netboot/tftp_backend_test.go +++ b/internal/metalman/netboot/tftp_backend_test.go @@ -27,9 +27,11 @@ func TestTFTPServerFetchesTokenizedSessionArtifactFromBackend(t *testing.T) { if err := server.readHandler(filename, transfer); err != nil { t.Fatal(err) } + if backend.filename != filename { t.Errorf("backend filename = %q, want %q", backend.filename, filename) } + if got := transfer.String(); got != "firmware" { t.Errorf("transfer = %q", got) } @@ -39,10 +41,12 @@ func TestTFTPServerRejectsLegacySourceIPFilenameWithBackend(t *testing.T) { t.Parallel() backend := &recordingTFTPBackend{data: []byte("firmware")} + server := &TFTPServer{Backend: backend} if err := server.readHandler("shimx64.efi", &memoryOutgoingTransfer{}); err == nil { t.Fatal("expected legacy filename to be rejected") } + if backend.filename != "" { t.Errorf("backend filename = %q, want empty", backend.filename) } @@ -58,6 +62,7 @@ func TestTFTPServerReportsCompletedSessionTransfer(t *testing.T) { if err := server.readHandler(filename, &memoryOutgoingTransfer{}); err != nil { t.Fatal(err) } + if backend.completed != filename { t.Errorf("completed filename = %q, want %q", backend.completed, filename) } @@ -105,7 +110,9 @@ func TestHTTPArtifactBackendResumesTruncatedResponse(t *testing.T) { t.Parallel() const artifact = "immutable-firmware" + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch requests.Add(1) { case 1: @@ -115,6 +122,7 @@ func TestHTTPArtifactBackendResumesTruncatedResponse(t *testing.T) { if got := r.Header.Get("Range"); got != "bytes=9-17" { t.Errorf("Range = %q", got) } + w.Header().Set("Content-Length", "9") w.Header().Set("Content-Range", "bytes 9-17/18") w.WriteHeader(http.StatusPartialContent) @@ -129,15 +137,18 @@ func TestHTTPArtifactBackendResumesTruncatedResponse(t *testing.T) { if err != nil { t.Fatal(err) } + reader, err := backend.Open(t.Context(), "v1/netboot/sessions/session/capability/artifacts/shimx64.efi") if err != nil { t.Fatal(err) } defer reader.Close() //nolint:errcheck // Test cleanup. + data, err := io.ReadAll(reader) if err != nil { t.Fatal(err) } + if got := string(data); got != artifact { t.Errorf("artifact = %q, want %q", got, artifact) } @@ -147,7 +158,9 @@ func TestHTTPArtifactBackendRetriesFailedResumeRequest(t *testing.T) { t.Parallel() const artifact = "immutable-firmware" + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch requests.Add(1) { case 1: @@ -159,11 +172,13 @@ func TestHTTPArtifactBackendRetriesFailedResumeRequest(t *testing.T) { t.Errorf("hijacking failed resume request: %v", err) return } + _ = conn.Close() case 3: if got := r.Header.Get("Range"); got != "bytes=9-17" { t.Errorf("Range = %q", got) } + w.Header().Set("Content-Length", "9") w.Header().Set("Content-Range", "bytes 9-17/18") w.WriteHeader(http.StatusPartialContent) @@ -178,18 +193,22 @@ func TestHTTPArtifactBackendRetriesFailedResumeRequest(t *testing.T) { if err != nil { t.Fatal(err) } + reader, err := backend.Open(t.Context(), "v1/netboot/sessions/session/capability/artifacts/shimx64.efi") if err != nil { t.Fatal(err) } defer reader.Close() //nolint:errcheck // Test cleanup. + data, err := io.ReadAll(reader) if err != nil { t.Fatal(err) } + if got := string(data); got != artifact { t.Errorf("artifact = %q, want %q", got, artifact) } + if got := requests.Load(); got != 3 { t.Errorf("backend requests = %d, want 3", got) } @@ -199,8 +218,10 @@ func TestHTTPArtifactBackendReportsSessionBootLoaderMilestone(t *testing.T) { t.Parallel() var callbackPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { callbackPath = r.URL.Path + w.WriteHeader(http.StatusNoContent) })) defer server.Close() @@ -209,10 +230,12 @@ func TestHTTPArtifactBackendReportsSessionBootLoaderMilestone(t *testing.T) { if err != nil { t.Fatal(err) } + artifactPath := "v1/netboot/sessions/session/capability/artifacts/shimx64.efi" if err := backend.RecordBootLoaderDownloaded(t.Context(), artifactPath); err != nil { t.Fatal(err) } + if want := "/v1/netboot/sessions/session/capability/callbacks/boot-loader-downloaded"; callbackPath != want { t.Errorf("callback path = %q, want %q", callbackPath, want) } diff --git a/internal/net/nodeagent/external.go b/internal/net/nodeagent/external.go index ef3b18b3d..4ad1ef17d 100644 --- a/internal/net/nodeagent/external.go +++ b/internal/net/nodeagent/external.go @@ -23,9 +23,11 @@ func externalGatewayConfig(options ExternalGatewayOptions) (*config, error) { if options.NodeName == "" { return nil, fmt.Errorf("external gateway node name is required") } + if options.RuntimeDir == "" { return nil, fmt.Errorf("external gateway runtime directory is required") } + if !filepath.IsAbs(options.RuntimeDir) { return nil, fmt.Errorf("external gateway runtime directory must be absolute") } @@ -53,9 +55,11 @@ func RunExternalGateway(ctx context.Context, options ExternalGatewayOptions) err if err != nil { return err } + if err := os.MkdirAll(cfg.WireGuardDir, 0o700); err != nil { return fmt.Errorf("create WireGuard runtime directory: %w", err) } + if err := os.MkdirAll(cfg.CNIConfDir, 0o700); err != nil { return fmt.Errorf("create CNI runtime directory: %w", err) } diff --git a/internal/net/nodeagent/site_watch_reconcile_test.go b/internal/net/nodeagent/site_watch_reconcile_test.go index db6fc00b0..9f48d3bad 100644 --- a/internal/net/nodeagent/site_watch_reconcile_test.go +++ b/internal/net/nodeagent/site_watch_reconcile_test.go @@ -109,6 +109,7 @@ func TestWaitForSiteMembershipImmediate(t *testing.T) { func TestWaitForSiteMembershipAcceptsSiteLessGateway(t *testing.T) { sliceInformer := newTestInformer() gatewayPoolInformer := newTestInformer() + pool := &unboundednetv1alpha1.GatewayPool{ ObjectMeta: metav1.ObjectMeta{Name: "external"}, Spec: unboundednetv1alpha1.GatewayPoolSpec{NodeSelector: map[string]string{"external": "true"}}, @@ -124,6 +125,7 @@ func TestWaitForSiteMembershipAcceptsSiteLessGateway(t *testing.T) { if err != nil { t.Fatalf("waitForSiteMembership returned error: %v", err) } + if got != "" { t.Fatalf("site-less gateway membership = %q, want empty", got) } @@ -174,6 +176,7 @@ func TestSiteLessGatewayDisablesCNIManagement(t *testing.T) { if manageCNIForMembership(siteInformer, "", true) { t.Fatal("site-less gateway must not wait for PodCIDRs or write CNI configuration") } + if !manageCNIForMembership(siteInformer, "", false) { t.Fatal("ordinary node without resolved membership must retain the default CNI policy") } diff --git a/internal/operator/components/metalman/metalman.go b/internal/operator/components/metalman/metalman.go index 9b8d05244..7e3fe1f1b 100644 --- a/internal/operator/components/metalman/metalman.go +++ b/internal/operator/components/metalman/metalman.go @@ -57,6 +57,7 @@ func (Component) Reconcile(ctx context.Context, env *component.Env, site *unboun if err := env.ApplyManifestFS(ctx, machinamanifests.Manifests, mutateSupportObject); err != nil { return component.Failed(err) } + if err := ensureCapabilitySecret(ctx, env, site); err != nil { return component.Failed(err) } @@ -71,6 +72,7 @@ func (Component) Reconcile(ctx context.Context, env *component.Env, site *unboun return component.Failed(err) } } + if err := reconcileEndpointEdges(ctx, env, site); err != nil { return component.Failed(err) } @@ -174,6 +176,7 @@ const ( func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config, role string, replicas int32) *appsv1.Deployment { image := cfg.Image("metalman") + name := DeploymentName(site.Name) if role == metalmanServerRole { name = ServerName(site.Name) @@ -211,11 +214,14 @@ func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg componen maxSurge := intstr.FromInt32(1) maxUnavailable := intstr.FromInt32(0) ports := []corev1.ContainerPort{{Name: "health", ContainerPort: 8081, Protocol: corev1.ProtocolTCP}} + serviceAccountName := "metalman-controller" if role == metalmanServerRole { serviceAccountName = "metalman-server" + ports = append(ports, corev1.ContainerPort{Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}) } + probe := func(path string) *corev1.Probe { return &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ Path: path, @@ -291,6 +297,7 @@ func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg componen func ensureCapabilitySecret(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) error { secret := &corev1.Secret{} + key := client.ObjectKey{Namespace: env.Namespace, Name: CapabilitySecretName(site.Name)} if err := env.Client.Get(ctx, key, secret); err == nil { return nil @@ -373,61 +380,76 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou if err := env.Client.List(ctx, &endpoints); err != nil { return fmt.Errorf("list NetbootEndpoints: %w", err) } + sort.Slice(endpoints.Items, func(i, j int) bool { return endpoints.Items[i].Name < endpoints.Items[j].Name }) desiredDeployments := map[string]struct{}{} desiredServices := map[string]struct{}{} desiredTLSSecrets := map[string]struct{}{} + for i := range endpoints.Items { endpoint := &endpoints.Items[i] if endpoint.Spec.SiteRef != site.Name { continue } + deployment, service, err := endpointEdgeObjects(endpoint, site, env.Namespace, env.Config) if err != nil { return fmt.Errorf("build edge for NetbootEndpoint %s: %w", endpoint.Name, err) } + if deployment != nil { if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { secret, err := mirroredTLSSecret(ctx, env.Client, endpoint, site, env.Namespace) if err != nil { return fmt.Errorf("mirror TLS Secret for NetbootEndpoint %s: %w", endpoint.Name, err) } + if err := env.ApplyObject(ctx, secret); err != nil { return err } + desiredTLSSecrets[secret.Name] = struct{}{} deployment.Spec.Template.Annotations = map[string]string{ edgeTLSChecksumAnnotation: tlsSecretChecksum(secret), } } + if err := env.ApplyObject(ctx, deployment); err != nil { return err } + liveDeployment := &appsv1.Deployment{} if err := env.Client.Get(ctx, client.ObjectKeyFromObject(deployment), liveDeployment); err != nil { return fmt.Errorf("get managed edge Deployment %s: %w", deployment.Name, err) } + if err := reconcileManagedEndpointStatus(ctx, env.Client, endpoint, liveDeployment); err != nil { return fmt.Errorf("update NetbootEndpoint %s status: %w", endpoint.Name, err) } + desiredDeployments[deployment.Name] = struct{}{} } + if service != nil { if err := env.ApplyObject(ctx, service); err != nil { return err } + desiredServices[service.Name] = struct{}{} } } + match := client.MatchingLabels{ "app.kubernetes.io/component": metalmanEdgeRole, unboundedv1alpha3.MachineSiteLabelKey: site.Name, } + var deployments appsv1.DeploymentList if err := env.Client.List(ctx, &deployments, client.InNamespace(env.Namespace), match); err != nil { return fmt.Errorf("list managed edge Deployments: %w", err) } + for i := range deployments.Items { if _, ok := desiredDeployments[deployments.Items[i].Name]; !ok { if err := env.DeleteIfExists(ctx, &deployments.Items[i]); err != nil { @@ -435,10 +457,12 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou } } } + var services corev1.ServiceList if err := env.Client.List(ctx, &services, client.InNamespace(env.Namespace), match); err != nil { return fmt.Errorf("list managed edge Services: %w", err) } + for i := range services.Items { if _, ok := desiredServices[services.Items[i].Name]; !ok { if err := env.DeleteIfExists(ctx, &services.Items[i]); err != nil { @@ -446,10 +470,12 @@ func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unbou } } } + var tlsSecrets corev1.SecretList if err := env.Client.List(ctx, &tlsSecrets, client.InNamespace(env.Namespace), match); err != nil { return fmt.Errorf("list managed edge TLS Secrets: %w", err) } + for i := range tlsSecrets.Items { if _, ok := desiredTLSSecrets[tlsSecrets.Items[i].Name]; !ok { if err := env.DeleteIfExists(ctx, &tlsSecrets.Items[i]); err != nil { @@ -470,6 +496,7 @@ func reconcileManagedEndpointStatus( if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeExternalL2 { return nil } + if deployment == nil { return fmt.Errorf("managed endpoint has no Deployment") } @@ -479,8 +506,10 @@ func reconcileManagedEndpointStatus( if err := kubeClient.Get(ctx, client.ObjectKey{Name: endpoint.Name}, current); err != nil { return err } + base := current.DeepCopy() current.Status.ObservedGeneration = current.Generation + condition := metav1.Condition{ Type: "Ready", Status: metav1.ConditionFalse, @@ -493,6 +522,7 @@ func reconcileManagedEndpointStatus( condition.Reason = "EdgeAvailable" condition.Message = "managed edge Deployment is available" } + apimeta.SetStatusCondition(¤t.Status.Conditions, condition) return kubeClient.Status().Patch(ctx, current, client.MergeFrom(base)) @@ -518,12 +548,16 @@ func mirroredTLSSecret( if endpoint.Spec.TLS.SecretRef == nil { return nil, fmt.Errorf("TLS secretRef is required") } + ref := endpoint.Spec.TLS.SecretRef + source := &corev1.Secret{} if err := kubeClient.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, source); err != nil { return nil, fmt.Errorf("get source Secret %s/%s: %w", ref.Namespace, ref.Name, err) } + certificate, certOK := source.Data[corev1.TLSCertKey] + privateKey, keyOK := source.Data[corev1.TLSPrivateKeyKey] if !certOK || !keyOK { return nil, fmt.Errorf("source Secret %s/%s must contain %s and %s", ref.Namespace, ref.Name, corev1.TLSCertKey, corev1.TLSPrivateKeyKey) @@ -552,6 +586,7 @@ func requestsForTLSSecret(ctx context.Context, kubeClient client.Client, secret } sites := map[string]struct{}{} + for i := range endpoints.Items { ref := endpoints.Items[i].Spec.TLS.SecretRef if endpoints.Items[i].Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret && ref != nil && @@ -559,13 +594,16 @@ func requestsForTLSSecret(ctx context.Context, kubeClient client.Client, secret sites[endpoints.Items[i].Spec.SiteRef] = struct{}{} } } + names := make([]string, 0, len(sites)) for site := range sites { if site != "" { names = append(names, site) } } + sort.Strings(names) + requests := make([]ctrl.Request, 0, len(names)) for _, site := range names { requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKey{Name: site}}) @@ -583,6 +621,7 @@ func endpointEdgeObjects( if endpoint.Spec.SiteRef != site.Name { return nil, nil, fmt.Errorf("endpoint site %q does not match %q", endpoint.Spec.SiteRef, site.Name) } + if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeExternalL2 { return nil, nil, nil } @@ -613,7 +652,9 @@ func endpointEdgeObjects( if endpoint.Spec.ManagedL2 == nil { return nil, nil, fmt.Errorf("managedL2 configuration is required") } + replicas = 1 + args = append(args, "--bind-address="+endpoint.Spec.ManagedL2.Address, "--dhcp-enabled", @@ -626,10 +667,12 @@ func endpointEdgeObjects( corev1.ContainerPort{Name: "dhcp", ContainerPort: 67, Protocol: corev1.ProtocolUDP}, corev1.ContainerPort{Name: "tftp", ContainerPort: 69, Protocol: corev1.ProtocolUDP}, ) + affinity, err := requiredNodeAffinity(endpoint.Spec.ManagedL2.NodeSelector) if err != nil { return nil, nil, err } + podSpec.HostNetwork = true podSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet podSpec.Affinity = affinity @@ -643,6 +686,7 @@ func endpointEdgeObjects( } expirationSeconds := int64(3600) + container := corev1.Container{ Name: "metalman", Image: cfg.Image("metalman"), @@ -664,6 +708,7 @@ func endpointEdgeObjects( }}}, }}}} } + if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { args = append(args, "--tls-cert-file=/var/run/secrets/metalman-tls/tls.crt", @@ -677,6 +722,7 @@ func endpointEdgeObjects( Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: EdgeTLSSecretName(endpoint.Name)}}, }) } + podSpec.Containers = []corev1.Container{container} deployment := &appsv1.Deployment{ @@ -698,15 +744,18 @@ func endpointEdgeObjects( if endpoint.Spec.Type != unboundedv1alpha3.NetbootEndpointTypeHTTP { return deployment, nil, nil } + serviceType := endpoint.Spec.HTTP.ServiceType if serviceType == "" { serviceType = corev1.ServiceTypeClusterIP } + servicePort := corev1.ServicePort{Name: "http", Port: 8880, TargetPort: intstr.FromInt32(8880), Protocol: corev1.ProtocolTCP} if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { servicePort.Name = "https" servicePort.Port = 443 } + service := &corev1.Service{ TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, ObjectMeta: metav1.ObjectMeta{ @@ -743,14 +792,18 @@ func endpointEdgeLabels(endpoint *unboundedv1alpha3.NetbootEndpoint, site string func requiredNodeAffinity(selector metav1.LabelSelector) (*corev1.Affinity, error) { requirements := make([]corev1.NodeSelectorRequirement, 0, len(selector.MatchLabels)+len(selector.MatchExpressions)) + keys := make([]string, 0, len(selector.MatchLabels)) for key := range selector.MatchLabels { keys = append(keys, key) } + sort.Strings(keys) + for _, key := range keys { requirements = append(requirements, corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{selector.MatchLabels[key]}}) } + for _, expression := range selector.MatchExpressions { operator := corev1.NodeSelectorOperator(expression.Operator) switch operator { @@ -758,6 +811,7 @@ func requiredNodeAffinity(selector metav1.LabelSelector) (*corev1.Affinity, erro default: return nil, fmt.Errorf("unsupported node selector operator %q", expression.Operator) } + requirements = append(requirements, corev1.NodeSelectorRequirement{Key: expression.Key, Operator: operator, Values: expression.Values}) } diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index a0bbf6ce0..072f943d1 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -83,6 +83,7 @@ func TestRBACSeparatesControllerAndServerIdentities(t *testing.T) { if err != nil { t.Fatalf("read Metalman RBAC template: %v", err) } + manifest := string(data) for _, required := range []string{ @@ -98,24 +99,28 @@ func TestRBACSeparatesControllerAndServerIdentities(t *testing.T) { } serverNamespaceRole := manifest[strings.Index(manifest, "kind: Role\nmetadata:\n name: metalman-server"):] + serverNamespaceRole = serverNamespaceRole[:strings.Index(serverNamespaceRole, "\n---")] if !strings.Contains(serverNamespaceRole, "resources: [\"serviceaccounts/token\"]") { t.Fatalf("metalman-server Role lacks bootstrap-token issuance:\n%s", serverNamespaceRole) } serverClusterRole := manifest[strings.Index(manifest, "kind: ClusterRole\nmetadata:\n name: metalman-server"):] + serverClusterRole = serverClusterRole[:strings.Index(serverClusterRole, "\n---")] if !strings.Contains(serverClusterRole, "resources: [\"tokenreviews\"]") { t.Fatalf("metalman-server ClusterRole lacks TokenReview permission:\n%s", serverClusterRole) } controllerRole := manifest[strings.Index(manifest, "kind: ClusterRole\nmetadata:\n name: metalman-controller\nrules:"):] + controllerRole = controllerRole[:strings.Index(controllerRole, "\n---")] if strings.Contains(controllerRole, "resources: [\"tokenreviews\"]") { t.Fatalf("metalman-controller retains server TokenReview permission:\n%s", controllerRole) } kubeSystemBinding := manifest[strings.Index(manifest, "kind: RoleBinding\nmetadata:\n name: metalman-controller\n namespace: kube-system"):] + kubeSystemBinding = kubeSystemBinding[:strings.Index(kubeSystemBinding, "\n---")] if !strings.Contains(kubeSystemBinding, "name: metalman-server") { t.Fatalf("kube-system metadata binding excludes metalman-server:\n%s", kubeSystemBinding) @@ -127,6 +132,7 @@ func TestOperatorRBACCanReconcileNetbootEndpoints(t *testing.T) { if err != nil { t.Fatalf("read operator RBAC template: %v", err) } + manifest := string(data) for _, required := range []string{ "resources: [\"netbootendpoints\"]", @@ -162,18 +168,22 @@ func TestReconcileManagedEndpointStatusFromDeploymentAvailability(t *testing.T) if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, endpoint, deployment); err != nil { t.Fatalf("reconcileManagedEndpointStatus: %v", err) } + updated := &unboundedv1alpha3.NetbootEndpoint{} if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { t.Fatalf("get endpoint: %v", err) } + if updated.Status.ObservedGeneration != endpoint.Generation { t.Fatalf("observed generation = %d, want %d", updated.Status.ObservedGeneration, endpoint.Generation) } + ready := findCondition(updated.Status.Conditions, "Ready") if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != "EdgeAvailable" { t.Fatalf("Ready condition = %#v", ready) } - if updated.Status.Claim == nil || updated.Status.Claim.HolderIdentity != "edge-pod-1" || updated.Status.Claim.RenewedAt.Time.Unix() != now.Time.Unix() { + + if updated.Status.Claim == nil || updated.Status.Claim.HolderIdentity != "edge-pod-1" || updated.Status.Claim.RenewedAt.Unix() != now.Unix() { t.Fatalf("claim was not preserved: %#v", updated.Status.Claim) } @@ -181,12 +191,15 @@ func TestReconcileManagedEndpointStatusFromDeploymentAvailability(t *testing.T) if err := kubeClient.Status().Update(t.Context(), deployment); err != nil { t.Fatalf("update Deployment status: %v", err) } + if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, updated, deployment); err != nil { t.Fatalf("reconcile unavailable endpoint: %v", err) } + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { t.Fatalf("get unavailable endpoint: %v", err) } + ready = findCondition(updated.Status.Conditions, "Ready") if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "EdgeUnavailable" { t.Fatalf("unavailable Ready condition = %#v", ready) @@ -208,10 +221,12 @@ func TestReconcileManagedEndpointStatusLeavesExternalClaimUntouched(t *testing.T if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, endpoint, nil); err != nil { t.Fatalf("reconcileManagedEndpointStatus: %v", err) } + updated := &unboundedv1alpha3.NetbootEndpoint{} if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { t.Fatal(err) } + if updated.Status.ObservedGeneration != 1 || updated.Status.Conditions[0].Reason != "ExternalEdgeReady" { t.Fatalf("external endpoint status changed: %#v", updated.Status) } @@ -316,16 +331,20 @@ func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { if controller.Spec.Replicas == nil || *controller.Spec.Replicas != 1 { t.Fatalf("controller replicas = %v, want 1", controller.Spec.Replicas) } + assertOrdinaryPodNetworking(t, &controller.Spec.Template.Spec) controllerContainer := controller.Spec.Template.Spec.Containers[0] if got := controllerContainer.Args; len(got) < 2 || got[0] != "controller" || got[1] != "--site=rack-a" { t.Fatalf("controller args = %#v", got) } + if controller.Spec.Template.Spec.ServiceAccountName != "metalman-controller" { t.Fatalf("controller service account = %q", controller.Spec.Template.Spec.ServiceAccountName) } + assertCapabilityKeyMount(t, &controller.Spec.Template.Spec, &controllerContainer, "rack-a") + for _, port := range controllerContainer.Ports { if port.Name == "http" || port.Name == "dhcp" || port.Name == "tftp" { t.Fatalf("controller exposes data-plane port %#v", port) @@ -336,27 +355,35 @@ func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { if server.Spec.Replicas == nil || *server.Spec.Replicas != 2 { t.Fatalf("server replicas = %v, want 2", server.Spec.Replicas) } + assertOrdinaryPodNetworking(t, &server.Spec.Template.Spec) serverContainer := server.Spec.Template.Spec.Containers[0] if got := serverContainer.Args; len(got) < 3 || got[0] != "server" || got[1] != "--site=rack-a" || got[2] != "--cache-dir=/var/cache/metalman" { t.Fatalf("server args = %#v", got) } + if server.Spec.Template.Spec.ServiceAccountName != "metalman-server" { t.Fatalf("server service account = %q", server.Spec.Template.Spec.ServiceAccountName) } + assertCapabilityKeyMount(t, &server.Spec.Template.Spec, &serverContainer, "rack-a") + if !hasContainerPort(serverContainer.Ports, "http", 8880) { t.Fatalf("server ports = %#v, want HTTP 8880", serverContainer.Ports) } + assertWorkloadHealthAndResources(t, &serverContainer) + if len(server.Spec.Template.Spec.TopologySpreadConstraints) != 1 { t.Fatalf("server topology spread constraints = %#v", server.Spec.Template.Spec.TopologySpreadConstraints) } + spread := server.Spec.Template.Spec.TopologySpreadConstraints[0] if spread.TopologyKey != corev1.LabelHostname || spread.MaxSkew != 1 || spread.WhenUnsatisfiable != corev1.ScheduleAnyway { t.Fatalf("server topology spread = %#v", spread) } + if got := server.Spec.Strategy.RollingUpdate; got == nil || got.MaxUnavailable == nil || got.MaxUnavailable.IntValue() != 0 { t.Fatalf("server maxUnavailable = %#v, want 0", got) } @@ -365,9 +392,11 @@ func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { if service.Name != ServerName("rack-a") { t.Fatalf("service name = %q, want %q", service.Name, ServerName("rack-a")) } + if service.Spec.Selector["app.kubernetes.io/name"] != "metalman-server" || service.Spec.Selector[unboundedv1alpha3.MachineSiteLabelKey] != "rack-a" { t.Fatalf("service selector = %#v", service.Spec.Selector) } + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != 8880 || service.Spec.Ports[0].TargetPort.IntValue() != 8880 { t.Fatalf("service ports = %#v", service.Spec.Ports) } @@ -376,6 +405,7 @@ func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { if pdb.Spec.MinAvailable == nil || pdb.Spec.MinAvailable.IntValue() != 1 { t.Fatalf("PDB minAvailable = %#v, want 1", pdb.Spec.MinAvailable) } + if pdb.Spec.Selector == nil || pdb.Spec.Selector.MatchLabels["app.kubernetes.io/name"] != "metalman-server" { t.Fatalf("PDB selector = %#v", pdb.Spec.Selector) } @@ -404,20 +434,25 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { if err != nil { t.Fatalf("endpointEdgeObjects: %v", err) } + if deployment == nil || service != nil { t.Fatalf("managed L2 objects = deployment %v, service %v", deployment, service) } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { t.Fatalf("managed L2 replicas = %v, want 1", deployment.Spec.Replicas) } + pod := &deployment.Spec.Template.Spec if !pod.HostNetwork || pod.DNSPolicy != corev1.DNSClusterFirstWithHostNet { t.Fatalf("managed L2 networking = hostNetwork %v, dnsPolicy %q", pod.HostNetwork, pod.DNSPolicy) } + if pod.Affinity == nil || pod.Affinity.NodeAffinity == nil || pod.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { t.Fatalf("managed L2 edge lacks required placement: %#v", pod.Affinity) } + container := &pod.Containers[0] for _, arg := range []string{ "edge", @@ -432,10 +467,13 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { t.Fatalf("managed L2 args %#v lack %q", container.Args, arg) } } + if pod.ServiceAccountName != "metalman-edge" { t.Fatalf("managed L2 service account = %q", pod.ServiceAccountName) } + assertProjectedEdgeToken(t, pod, container) + if !hasContainerPort(container.Ports, "http", 8880) || !hasContainerPort(container.Ports, "dhcp", 67) || !hasContainerPort(container.Ports, "tftp", 69) { @@ -462,17 +500,22 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { if err != nil { t.Fatalf("endpointEdgeObjects: %v", err) } + if deployment == nil || service == nil { t.Fatalf("HTTP objects = deployment %v, service %v", deployment, service) } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 2 { t.Fatalf("HTTP replicas = %v, want 2", deployment.Spec.Replicas) } + assertOrdinaryPodNetworking(t, &deployment.Spec.Template.Spec) + args := deployment.Spec.Template.Spec.Containers[0].Args if containsString(args, "--dhcp-enabled") || containsString(args, "--tftp-enabled") { t.Fatalf("HTTP edge enables L2 protocols: %#v", args) } + if service.Spec.Type != corev1.ServiceTypeNodePort || service.Spec.Selector[netbootEndpointLabel] != endpoint.Name { t.Fatalf("HTTP edge Service = %#v", service.Spec) } @@ -501,6 +544,7 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { if err != nil { t.Fatalf("endpointEdgeObjects: %v", err) } + container := &deployment.Spec.Template.Spec.Containers[0] for _, arg := range []string{ "--tls-cert-file=/var/run/secrets/metalman-tls/tls.crt", @@ -510,14 +554,17 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { t.Fatalf("Secret TLS args %#v lack %q", container.Args, arg) } } + if len(deployment.Spec.Template.Spec.Volumes) != 1 || deployment.Spec.Template.Spec.Volumes[0].Secret == nil || deployment.Spec.Template.Spec.Volumes[0].Secret.SecretName != EdgeTLSSecretName(endpoint.Name) { t.Fatalf("Secret TLS volumes = %#v", deployment.Spec.Template.Spec.Volumes) } + if len(container.VolumeMounts) != 1 || container.VolumeMounts[0].MountPath != "/var/run/secrets/metalman-tls" || !container.VolumeMounts[0].ReadOnly { t.Fatalf("Secret TLS mounts = %#v", container.VolumeMounts) } + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Name != "https" || service.Spec.Ports[0].Port != 443 { t.Fatalf("Secret TLS Service ports = %#v", service.Spec.Ports) } @@ -536,6 +583,7 @@ func TestEndpointEdgeWorkloadMatrix(t *testing.T) { if err != nil { t.Fatalf("endpointEdgeObjects: %v", err) } + if deployment != nil || service != nil { t.Fatalf("external L2 objects = deployment %v, service %v", deployment, service) } @@ -576,21 +624,27 @@ func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { t.Fatalf("reconcileEndpointEdges: %v", err) } + mirror := &corev1.Secret{} + key := client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeTLSSecretName(endpoint.Name)} if err := env.Client.Get(t.Context(), key, mirror); err != nil { t.Fatalf("get mirrored TLS Secret: %v", err) } + if got := string(mirror.Data[corev1.TLSCertKey]); got != wantCert { t.Fatalf("mirrored certificate = %q, want %q", got, wantCert) } + if got := string(mirror.Data[corev1.TLSPrivateKeyKey]); got != "private-key-v1" { t.Fatalf("mirrored private key = %q", got) } + deployment := &appsv1.Deployment{} if err := env.Client.Get(t.Context(), client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeName(endpoint.Name)}, deployment); err != nil { t.Fatalf("get TLS edge Deployment: %v", err) } + checksum := deployment.Spec.Template.Annotations[edgeTLSChecksumAnnotation] if checksum == "" { t.Fatal("TLS edge pod template lacks certificate checksum") @@ -600,10 +654,12 @@ func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { } firstChecksum := assertMirror("certificate-v1") + source.Data[corev1.TLSCertKey] = []byte("certificate-v2") if err := env.Client.Update(t.Context(), source); err != nil { t.Fatalf("update source TLS Secret: %v", err) } + if secondChecksum := assertMirror("certificate-v2"); secondChecksum == firstChecksum { t.Fatalf("TLS edge checksum did not change after certificate rotation: %q", secondChecksum) } @@ -611,6 +667,7 @@ func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { if err := env.Client.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, endpoint); err != nil { t.Fatalf("refresh endpoint: %v", err) } + endpoint.Spec.TLS = unboundedv1alpha3.NetbootEndpointTLS{ Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, Mode: unboundedv1alpha3.NetbootEndpointTLSExternal, @@ -618,9 +675,11 @@ func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { if err := env.Client.Update(t.Context(), endpoint); err != nil { t.Fatalf("update endpoint TLS mode: %v", err) } + if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { t.Fatalf("reconcile external TLS endpoint: %v", err) } + err := env.Client.Get(t.Context(), client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeTLSSecretName(endpoint.Name)}, &corev1.Secret{}) if !apierrors.IsNotFound(err) { t.Fatalf("mirrored TLS Secret still exists: %v", err) @@ -684,6 +743,7 @@ func TestReconcileEndpointEdgesDeletesStaleManagedWorkloads(t *testing.T) { if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { t.Fatalf("reconcileEndpointEdges: %v", err) } + for _, object := range []client.Object{staleDeployment, staleService} { err := env.Client.Get(t.Context(), client.ObjectKeyFromObject(object), object) if !apierrors.IsNotFound(err) { @@ -706,10 +766,12 @@ func TestEnsureCapabilitySecretPreservesExistingKey(t *testing.T) { } key := client.ObjectKey{Namespace: component.DefaultNamespace, Name: CapabilitySecretName("rack-a")} + first := &corev1.Secret{} if err := env.Client.Get(t.Context(), key, first); err != nil { t.Fatalf("get first capability Secret: %v", err) } + if len(first.Data[capabilitySecretKey]) != 32 { t.Fatalf("capability key length = %d, want 32", len(first.Data[capabilitySecretKey])) } @@ -722,6 +784,7 @@ func TestEnsureCapabilitySecretPreservesExistingKey(t *testing.T) { if err := env.Client.Get(t.Context(), key, second); err != nil { t.Fatalf("get second capability Secret: %v", err) } + if string(second.Data[capabilitySecretKey]) != string(first.Data[capabilitySecretKey]) { t.Fatal("capability key changed across reconciliation") } @@ -754,6 +817,7 @@ func TestDeploymentAllowsZeroReplicas(t *testing.T) { // The split roles have fixed availability semantics; the former Site-level // replica knob no longer scales the controller and data plane together. site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}} + d := serverDeployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) if d.Spec.Replicas == nil || *d.Spec.Replicas != 2 { t.Fatalf("server replicas = %v, want 2", d.Spec.Replicas) @@ -792,6 +856,7 @@ func assertOrdinaryPodNetworking(t *testing.T, podSpec *corev1.PodSpec) { if podSpec.HostNetwork { t.Fatal("pod unexpectedly uses host networking") } + if podSpec.Affinity != nil && podSpec.Affinity.NodeAffinity != nil && podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { t.Fatalf("pod unexpectedly has required node affinity: %#v", podSpec.Affinity) @@ -832,9 +897,11 @@ func assertWorkloadHealthAndResources(t *testing.T, container *corev1.Container) t.Fatalf("%s probe = %#v, want HTTP probe on 8081", name, probe) } } + if container.LivenessProbe.HTTPGet.Path != "/healthz" || container.ReadinessProbe.HTTPGet.Path != "/readyz" { t.Fatalf("probe paths = %q, %q", container.LivenessProbe.HTTPGet.Path, container.ReadinessProbe.HTTPGet.Path) } + if container.Resources.Requests.Cpu().IsZero() || container.Resources.Requests.Memory().IsZero() || container.Resources.Limits.Cpu().IsZero() || container.Resources.Limits.Memory().IsZero() { t.Fatalf("resources are incomplete: %#v", container.Resources) @@ -881,10 +948,12 @@ func assertProjectedEdgeToken(t *testing.T, pod *corev1.PodSpec, container *core if mount.Name != "edge-token" || mount.MountPath != "/var/run/secrets/metalman" || !mount.ReadOnly { continue } + for _, volume := range pod.Volumes { if volume.Name != mount.Name || volume.Projected == nil || len(volume.Projected.Sources) != 1 { continue } + token := volume.Projected.Sources[0].ServiceAccountToken if token != nil && token.Audience == "metalman-edge" && token.Path == "token" { return @@ -894,41 +963,3 @@ func assertProjectedEdgeToken(t *testing.T, pod *corev1.PodSpec, container *core t.Fatalf("missing audience-bound projected edge token: mounts=%#v volumes=%#v", container.VolumeMounts, pod.Volumes) } - -func assertSiteAffinity(t *testing.T, affinity *corev1.Affinity, siteName string) { - t.Helper() - - if affinity == nil || affinity.NodeAffinity == nil || affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { - t.Fatalf("missing node affinity: %#v", affinity) - } - - terms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - if len(terms) != 2 { - t.Fatalf("node selector terms len = %d, want 2: %#v", len(terms), terms) - } - - want := map[string]bool{component.SiteLabelKey: false, component.DeprecatedSiteLabelKey: false} - - for _, term := range terms { - if len(term.MatchExpressions) != 1 { - t.Fatalf("term must have one expression: %#v", term) - } - - expr := term.MatchExpressions[0] - if expr.Operator != corev1.NodeSelectorOpIn || len(expr.Values) != 1 || expr.Values[0] != siteName { - t.Fatalf("unexpected site affinity expression: %#v", expr) - } - - if _, ok := want[expr.Key]; !ok { - t.Fatalf("unexpected site affinity key %q", expr.Key) - } - - want[expr.Key] = true - } - - for key, seen := range want { - if !seen { - t.Fatalf("site affinity missing key %q", key) - } - } -} diff --git a/internal/operator/migrate_test.go b/internal/operator/migrate_test.go index 607f308ae..92b82727b 100644 --- a/internal/operator/migrate_test.go +++ b/internal/operator/migrate_test.go @@ -283,6 +283,7 @@ func TestLegacyMetalmanDetectionIgnoresRemovedDHCPFlag(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r := newReaper(t, metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", tt.args...)) + got, err := r.legacyMetalmanExistsForSite(t.Context(), "edge") if err != nil || !got { t.Fatalf("legacyMetalmanExistsForSite = %t, err=%v; want true", got, err)