diff --git a/api/machina/v1alpha3/machine_types.go b/api/machina/v1alpha3/machine_types.go index e590f5481..f4735005c 100644 --- a/api/machina/v1alpha3/machine_types.go +++ b/api/machina/v1alpha3/machine_types.go @@ -354,7 +354,11 @@ type RedfishSpec struct { PasswordRef SecretKeySelector `json:"passwordRef"` } -// PXESpec defines PXE boot configuration for a Machine. +// +kubebuilder:validation:XValidation:rule="!(self.transport == 'TFTP' && self.configurationSource == 'Redfish')",message="TFTP transport requires DHCP configuration" +// +kubebuilder:validation:XValidation:rule="self.networkMode != 'Static' || self.configurationSource == 'Redfish'",message="static networking requires Redfish configuration" +// +kubebuilder:validation:XValidation:rule="self.configurationSource != 'Redfish' || has(self.redfish)",message="Redfish configuration requires redfish connection details" + +// PXESpec defines network boot configuration for a Machine. type PXESpec struct { // Image is an OCI image reference containing the machine disk image. // The image must contain /disk/disk.img.gz. @@ -389,13 +393,24 @@ type PXESpec struct { // +optional NetbootPullSecretRef *NamespacedSecretReference `json:"netbootPullSecretRef,omitempty"` - // BootProtocol selects how metalman should trigger network boot for - // repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - // HTTP boot with a URL derived from the netboot image metadata. - // +kubebuilder:validation:Enum=PXE;HTTP - // +kubebuilder:default=PXE + // Transport selects the firmware boot artifact transport. + // +kubebuilder:default=TFTP + // +optional + Transport NetbootTransport `json:"transport,omitempty"` + + // ConfigurationSource selects how firmware receives its boot target. + // +kubebuilder:default=DHCP + // +optional + ConfigurationSource NetbootConfigurationSource `json:"configurationSource,omitempty"` + + // NetworkMode selects how firmware configures the provisioning network. + // +kubebuilder:default=DHCP // +optional - BootProtocol string `json:"bootProtocol,omitempty"` + NetworkMode NetbootNetworkMode `json:"networkMode,omitempty"` + + // EndpointRef names the NetbootEndpoint that serves this Machine. + // +kubebuilder:validation:MinLength=1 + EndpointRef string `json:"endpointRef"` // DHCPLeases defines static IPv4 provisioning network settings. PXE boot // uses them as DHCP leases. HTTP boot uses the first entry to configure the @@ -420,18 +435,39 @@ type PXESpec struct { } const ( - // PXEBootProtocolPXE uses DHCP/TFTP PXE boot. - PXEBootProtocolPXE = "PXE" - // PXEBootProtocolHTTP uses Redfish UEFI HTTP boot. - PXEBootProtocolHTTP = "HTTP" // PXEArchitectureAMD64 is the x86_64 target architecture for PXE boot. PXEArchitectureAMD64 = "amd64" // PXEArchitectureARM64 is the aarch64 target architecture for PXE boot. PXEArchitectureARM64 = "arm64" // DefaultPXEArchitecture is used when host.netboot.architecture is omitted. DefaultPXEArchitecture = PXEArchitectureAMD64 - // DefaultPXEBootProtocol is used when host.netboot.bootProtocol is omitted. - DefaultPXEBootProtocol = PXEBootProtocolPXE +) + +// NetbootTransport is the protocol firmware uses to fetch boot artifacts. +// +kubebuilder:validation:Enum=TFTP;HTTP +type NetbootTransport string + +const ( + NetbootTransportTFTP NetbootTransport = "TFTP" + NetbootTransportHTTP NetbootTransport = "HTTP" +) + +// NetbootConfigurationSource supplies the firmware boot target. +// +kubebuilder:validation:Enum=DHCP;Redfish +type NetbootConfigurationSource string + +const ( + NetbootConfigurationSourceDHCP NetbootConfigurationSource = "DHCP" + NetbootConfigurationSourceRedfish NetbootConfigurationSource = "Redfish" +) + +// NetbootNetworkMode configures the firmware provisioning interface. +// +kubebuilder:validation:Enum=DHCP;Static +type NetbootNetworkMode string + +const ( + NetbootNetworkModeDHCP NetbootNetworkMode = "DHCP" + NetbootNetworkModeStatic NetbootNetworkMode = "Static" ) // TargetArchitecture returns the effective PXE target architecture. @@ -443,13 +479,13 @@ func (p *PXESpec) TargetArchitecture() string { return p.Architecture } -// TargetBootProtocol returns the effective network boot protocol. -func (p *PXESpec) TargetBootProtocol() string { - if p == nil || p.BootProtocol == "" { - return DefaultPXEBootProtocol +// TargetTransport returns the effective firmware boot transport. +func (p *PXESpec) TargetTransport() NetbootTransport { + if p == nil || p.Transport == "" { + return NetbootTransportTFTP } - return p.BootProtocol + return p.Transport } // CloudInitSpec defines cloud-init customization for PXE-booted machines. diff --git a/api/machina/v1alpha3/machine_validation_test.go b/api/machina/v1alpha3/machine_validation_test.go index 113e924f0..567617a67 100644 --- a/api/machina/v1alpha3/machine_validation_test.go +++ b/api/machina/v1alpha3/machine_validation_test.go @@ -51,6 +51,33 @@ func TestMachineProviderOwnershipSchema(t *testing.T) { }) } +func TestMachineNetbootSchema(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("../../../deploy/machina/crd/unbounded-cloud.io_machines.yaml") + if err != nil { + t.Fatalf("read Machine CRD: %v", err) + } + + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("parse Machine CRD: %v", err) + } + + netbootSchema := crd.Spec.Versions[0].Schema.OpenAPIV3Schema. + Properties["spec"].Properties["host"].Properties["netboot"] + + for _, field := range []string{"transport", "configurationSource", "networkMode", "endpointRef"} { + if _, ok := netbootSchema.Properties[field]; !ok { + t.Errorf("netboot schema is missing %q", field) + } + } + + if _, ok := netbootSchema.Properties["bootProtocol"]; ok { + t.Error("netboot schema still exposes bootProtocol") + } +} + func assertSchemaValidations(t *testing.T, schema apiextensionsv1.JSONSchemaProps, want map[string]string) { t.Helper() diff --git a/api/machina/v1alpha3/machineoperation_types.go b/api/machina/v1alpha3/machineoperation_types.go index 0412ab7a9..f854a22b1 100644 --- a/api/machina/v1alpha3/machineoperation_types.go +++ b/api/machina/v1alpha3/machineoperation_types.go @@ -164,6 +164,11 @@ type ProviderOperationStatus struct { // one MachineOperation target. It is persisted before provider execution so // retries do not silently adopt later desired-state changes. type MachineOperationTargetInput struct { + // NetbootSessionRef identifies the immutable provisioning session assigned + // to this HostReplace target. + // +optional + NetbootSessionRef *NetbootSessionReference `json:"netbootSessionRef,omitempty"` + // ProviderRef identifies the exact provider-owned resource referenced by // host.external.machineRef when the operation target was initialized. // +optional diff --git a/api/machina/v1alpha3/netboot_validation_test.go b/api/machina/v1alpha3/netboot_validation_test.go new file mode 100644 index 000000000..48f2e330f --- /dev/null +++ b/api/machina/v1alpha3/netboot_validation_test.go @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha3 + +import ( + "os" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func TestNetbootEndpointSchema(t *testing.T) { + t.Parallel() + + crd := readCRD(t, "../../../deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml") + if crd.Spec.Scope != apiextensionsv1.ClusterScoped { + t.Errorf("scope = %q, want %q", crd.Spec.Scope, apiextensionsv1.ClusterScoped) + } + + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + for _, field := range []string{"siteRef", "type", "externalURL", "tls", "managedL2", "http"} { + if _, ok := spec.Properties[field]; !ok { + t.Errorf("endpoint spec is missing %q", field) + } + } + + assertSchemaValidations(t, spec, map[string]string{ + "self.tls.trust != 'Public' || (self.externalURL.startsWith('https://') && self.tls.mode != 'Disabled')": "public endpoints require HTTPS", + "self.type == 'ManagedL2' ? has(self.managedL2) : !has(self.managedL2)": "managedL2 configuration must be set only for ManagedL2 endpoints", + "self.type == 'HTTP' ? has(self.http) : !has(self.http)": "http configuration must be set only for HTTP endpoints", + }) + assertSchemaValidations(t, spec.Properties["tls"], map[string]string{ + "self.mode == 'Secret' ? has(self.secretRef) : !has(self.secretRef)": "secretRef must be set only when TLS mode is Secret", + }) + + status := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] + for _, field := range []string{"observedGeneration", "claim", "conditions"} { + if _, ok := status.Properties[field]; !ok { + t.Errorf("endpoint status is missing %q", field) + } + } +} + +func TestNetbootSessionSchema(t *testing.T) { + t.Parallel() + + crd := readCRD(t, "../../../deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml") + if crd.Spec.Scope != apiextensionsv1.ClusterScoped { + t.Errorf("scope = %q, want %q", crd.Spec.Scope, apiextensionsv1.ClusterScoped) + } + + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + for _, field := range []string{"machine", "operation", "endpoint", "boot", "provisioning", "artifacts", "expiresAt"} { + if _, ok := spec.Properties[field]; !ok { + t.Errorf("session spec is missing %q", field) + } + } + + assertSchemaValidations(t, spec, map[string]string{ + "self == oldSelf": "netboot session spec is immutable", + }) + boot := spec.Properties["boot"] + + firmware, ok := boot.Properties["firmwareArtifact"] + if !ok { + t.Fatal("session boot snapshot is missing firmwareArtifact") + } + + if firmware.MinLength == nil || *firmware.MinLength != 1 { + t.Error("session boot firmwareArtifact must be non-empty") + } + + provisioning := spec.Properties["provisioning"] + for _, field := range []string{"cluster", "kubernetes", "agent", "providerLabels", "userData"} { + if _, ok := provisioning.Properties[field]; !ok { + t.Errorf("session provisioning snapshot is missing %q", field) + } + } + + artifactSource := spec.Properties["artifacts"].Properties["files"].Items.Schema.Properties["source"] + requireEnumValue(t, artifactSource.Enum, "Session") + + status := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"] + for _, field := range []string{"phase", "conditions"} { + if _, ok := status.Properties[field]; !ok { + t.Errorf("session status is missing %q", field) + } + } +} + +func requireEnumValue(t *testing.T, values []apiextensionsv1.JSON, want string) { + t.Helper() + + for _, value := range values { + var got string + if err := yaml.Unmarshal(value.Raw, &got); err == nil && got == want { + return + } + } + + t.Errorf("enum does not contain %q", want) +} + +func TestMachineOperationTargetInputHasNetbootSessionRef(t *testing.T) { + t.Parallel() + + crd := readCRD(t, "../../../deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml") + + input := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"]. + Properties["targets"].Items.Schema.Properties["input"] + if _, ok := input.Properties["netbootSessionRef"]; !ok { + t.Error("operation target input is missing netbootSessionRef") + } +} + +func readCRD(t *testing.T, path string) apiextensionsv1.CustomResourceDefinition { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read CRD: %v", err) + } + + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("parse CRD: %v", err) + } + + return crd +} diff --git a/api/machina/v1alpha3/netbootendpoint_types.go b/api/machina/v1alpha3/netbootendpoint_types.go new file mode 100644 index 000000000..f0f766d73 --- /dev/null +++ b/api/machina/v1alpha3/netbootendpoint_types.go @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &NetbootEndpoint{}, &NetbootEndpointList{}) + metav1.AddToGroupVersion(s, GroupVersion) + + return nil + }) +} + +// NetbootEndpointType identifies where an endpoint edge runs. +// +kubebuilder:validation:Enum=ManagedL2;ExternalL2;HTTP +type NetbootEndpointType string + +const ( + NetbootEndpointTypeManagedL2 NetbootEndpointType = "ManagedL2" + NetbootEndpointTypeExternalL2 NetbootEndpointType = "ExternalL2" + NetbootEndpointTypeHTTP NetbootEndpointType = "HTTP" +) + +// NetbootEndpointTrust identifies the network trust boundary of an endpoint. +// +kubebuilder:validation:Enum=TrustedLAN;Public +type NetbootEndpointTrust string + +const ( + NetbootEndpointTrustTrustedLAN NetbootEndpointTrust = "TrustedLAN" + NetbootEndpointTrustPublic NetbootEndpointTrust = "Public" +) + +// NetbootEndpointTLSMode identifies where TLS is terminated. +// +kubebuilder:validation:Enum=Disabled;Secret;External +type NetbootEndpointTLSMode string + +const ( + NetbootEndpointTLSDisabled NetbootEndpointTLSMode = "Disabled" + NetbootEndpointTLSSecret NetbootEndpointTLSMode = "Secret" + NetbootEndpointTLSExternal NetbootEndpointTLSMode = "External" +) + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=nbe +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Site",type="string",JSONPath=".spec.siteRef" +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.type" +// +kubebuilder:printcolumn:name="URL",type="string",JSONPath=".spec.externalURL" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// NetbootEndpoint declares a stable client-facing netboot endpoint and its edge +// placement. The endpoint URL is snapshotted into each NetbootSession. +type NetbootEndpoint struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetbootEndpointSpec `json:"spec,omitempty"` + Status NetbootEndpointStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetbootEndpointList contains a list of NetbootEndpoint resources. +type NetbootEndpointList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetbootEndpoint `json:"items"` +} + +// +kubebuilder:validation:XValidation:rule="self.tls.trust != 'Public' || (self.externalURL.startsWith('https://') && self.tls.mode != 'Disabled')",message="public endpoints require HTTPS" +// +kubebuilder:validation:XValidation:rule="self.type == 'ManagedL2' ? has(self.managedL2) : !has(self.managedL2)",message="managedL2 configuration must be set only for ManagedL2 endpoints" +// +kubebuilder:validation:XValidation:rule="self.type == 'HTTP' ? has(self.http) : !has(self.http)",message="http configuration must be set only for HTTP endpoints" + +// NetbootEndpointSpec defines a stable edge endpoint. +type NetbootEndpointSpec struct { + // SiteRef names the Site whose Machines may use this endpoint. + // +kubebuilder:validation:MinLength=1 + SiteRef string `json:"siteRef"` + + // Type identifies how the endpoint edge is operated. + Type NetbootEndpointType `json:"type"` + + // ExternalURL is the stable base URL advertised to firmware and installers. + // +kubebuilder:validation:Pattern=`^https?://` + ExternalURL string `json:"externalURL"` + + // TLS defines the endpoint trust boundary and TLS termination mode. + TLS NetbootEndpointTLS `json:"tls"` + + // ManagedL2 configures an operator-managed edge on a provisioning LAN. + // +optional + ManagedL2 *NetbootManagedL2Spec `json:"managedL2,omitempty"` + + // HTTP configures an operator-managed HTTP edge Service. + // +optional + HTTP *NetbootHTTPEndpointSpec `json:"http,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="self.mode == 'Secret' ? has(self.secretRef) : !has(self.secretRef)",message="secretRef must be set only when TLS mode is Secret" + +// NetbootEndpointTLS defines transport security for an endpoint. +type NetbootEndpointTLS struct { + // Trust identifies whether the endpoint is confined to a trusted LAN or is + // reachable across an untrusted network. + Trust NetbootEndpointTrust `json:"trust"` + + // Mode identifies where TLS is terminated. + Mode NetbootEndpointTLSMode `json:"mode"` + + // SecretRef references the serving certificate when mode is Secret. + // +optional + SecretRef *NamespacedSecretReference `json:"secretRef,omitempty"` +} + +// NetbootManagedL2Spec places a managed edge on a provisioning network. +type NetbootManagedL2Spec struct { + // NodeSelector selects nodes attached to the provisioning network. + NodeSelector metav1.LabelSelector `json:"nodeSelector"` + + // Interface is the host interface used for DHCP and TFTP. + // +kubebuilder:validation:MinLength=1 + Interface string `json:"interface"` + + // Address is the stable address advertised by DHCP and used by the edge. + // +kubebuilder:validation:MinLength=1 + Address string `json:"address"` +} + +// NetbootHTTPEndpointSpec configures the Service for an HTTP-only edge. +type NetbootHTTPEndpointSpec struct { + // ServiceType controls how the edge Service is exposed. + // +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer + // +kubebuilder:default=ClusterIP + // +optional + ServiceType corev1.ServiceType `json:"serviceType,omitempty"` +} + +// NetbootEndpointStatus reports edge ownership and readiness. +type NetbootEndpointStatus struct { + // ObservedGeneration is the latest spec generation processed by the edge. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Claim identifies the edge currently responsible for this endpoint. + // +optional + Claim *NetbootEndpointClaim `json:"claim,omitempty"` + + // Conditions report endpoint readiness and degradation. + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// NetbootEndpointClaim records the current endpoint edge claimant. +type NetbootEndpointClaim struct { + // HolderIdentity uniquely identifies the claiming edge process. + HolderIdentity string `json:"holderIdentity"` + + // RenewedAt records the latest successful claim heartbeat. + RenewedAt metav1.Time `json:"renewedAt"` +} diff --git a/api/machina/v1alpha3/netbootsession_types.go b/api/machina/v1alpha3/netbootsession_types.go new file mode 100644 index 000000000..416c8911b --- /dev/null +++ b/api/machina/v1alpha3/netbootsession_types.go @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha3 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &NetbootSession{}, &NetbootSessionList{}) + metav1.AddToGroupVersion(s, GroupVersion) + + return nil + }) +} + +// NetbootSessionPhase is the durable lifecycle phase of a provisioning session. +// +kubebuilder:validation:Enum=Pending;Preparing;Ready;Active;Complete;Failed;Expired +type NetbootSessionPhase string + +const ( + NetbootSessionPhasePending NetbootSessionPhase = "Pending" + NetbootSessionPhasePreparing NetbootSessionPhase = "Preparing" + NetbootSessionPhaseReady NetbootSessionPhase = "Ready" + NetbootSessionPhaseActive NetbootSessionPhase = "Active" + NetbootSessionPhaseComplete NetbootSessionPhase = "Complete" + NetbootSessionPhaseFailed NetbootSessionPhase = "Failed" + NetbootSessionPhaseExpired NetbootSessionPhase = "Expired" +) + +// Condition types for NetbootSession. +const ( + NetbootSessionConditionPrepared = "Prepared" + NetbootSessionConditionEndpointReady = "EndpointReady" + NetbootSessionConditionBootLoaderDownloaded = "BootLoaderDownloaded" + NetbootSessionConditionBootImageWritten = "BootImageWritten" + NetbootSessionConditionCloudInitDone = "CloudInitDone" + NetbootSessionConditionAttested = "Attested" +) + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=nbs +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".spec.machine.name" +// +kubebuilder:printcolumn:name="Endpoint",type="string",JSONPath=".spec.endpoint.name" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Expires",type="date",JSONPath=".spec.expiresAt" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// NetbootSession is an immutable provisioning contract for one +// MachineOperation target. Runtime progress is recorded only in status. +type NetbootSession struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetbootSessionSpec `json:"spec,omitempty"` + Status NetbootSessionStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NetbootSessionList contains a list of NetbootSession resources. +type NetbootSessionList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetbootSession `json:"items"` +} + +// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="netboot session spec is immutable" + +// NetbootSessionSpec snapshots all input needed to provision one target. +type NetbootSessionSpec struct { + // Machine identifies the exact Machine revision being provisioned. + Machine NetbootSessionObjectSnapshot `json:"machine"` + + // Operation identifies the owning MachineOperation. + Operation NetbootSessionObjectSnapshot `json:"operation"` + + // Endpoint snapshots the selected endpoint and advertised URL. + Endpoint NetbootSessionEndpointSnapshot `json:"endpoint"` + + // Boot snapshots firmware and provisioning network configuration. + Boot NetbootSessionBoot `json:"boot"` + + // Provisioning snapshots inputs used to render installer and first-boot + // configuration. + Provisioning NetbootSessionProvisioning `json:"provisioning"` + + // Artifacts identifies immutable OCI sources and files for this session. + Artifacts NetbootSessionArtifacts `json:"artifacts"` + + // ExpiresAt is the last time new requests may use this session. + ExpiresAt metav1.Time `json:"expiresAt"` +} + +// NetbootSessionObjectSnapshot identifies an exact Kubernetes object revision. +type NetbootSessionObjectSnapshot struct { + // Name is the cluster-scoped object name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // UID detects deletion and recreation of the object. + UID types.UID `json:"uid"` + + // Generation records the observed desired-state generation. + // +kubebuilder:validation:Minimum=1 + Generation int64 `json:"generation"` +} + +// NetbootSessionEndpointSnapshot identifies the endpoint selected for a session. +type NetbootSessionEndpointSnapshot struct { + // Name is the NetbootEndpoint name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // UID detects deletion and recreation of the endpoint. + UID types.UID `json:"uid"` + + // ExternalURL is the immutable base URL advertised for this session. + // +kubebuilder:validation:Pattern=`^https?://` + ExternalURL string `json:"externalURL"` +} + +// NetbootSessionBoot snapshots firmware and provisioning network settings. +type NetbootSessionBoot struct { + Transport NetbootTransport `json:"transport"` + ConfigurationSource NetbootConfigurationSource `json:"configurationSource"` + NetworkMode NetbootNetworkMode `json:"networkMode"` + + // FirmwareArtifact is the named immutable artifact advertised to firmware. + // +kubebuilder:validation:MinLength=1 + FirmwareArtifact string `json:"firmwareArtifact"` + + // Architecture selects the boot artifact platform. + // +kubebuilder:validation:Enum=amd64;arm64 + Architecture string `json:"architecture"` + + // DHCPLeases snapshots the target's provisioning network settings. + // +optional + DHCPLeases []DHCPLease `json:"dhcpLeases,omitempty"` + + // TargetDisk is the block device written by the installer. + // +optional + TargetDisk string `json:"targetDisk,omitempty"` +} + +// NetbootSessionProvisioning snapshots installer and first-boot inputs. +type NetbootSessionProvisioning struct { + Cluster NetbootSessionCluster `json:"cluster"` + + // Kubernetes contains the target's immutable kubelet configuration. + // +optional + Kubernetes *KubernetesSpec `json:"kubernetes,omitempty"` + + // Agent contains the immutable agent installation configuration. + // +optional + Agent *AgentSpec `json:"agent,omitempty"` + + // ProviderLabels are merged into the rendered kubelet labels. + // +optional + ProviderLabels map[string]string `json:"providerLabels,omitempty"` + + // UserData is the resolved cloud-init user-data content. + UserData string `json:"userData"` +} + +// NetbootSessionCluster snapshots cluster connection inputs used by the agent. +type NetbootSessionCluster struct { + APIServerURL string `json:"apiServerURL"` + CACertBase64 string `json:"caCertBase64"` + DNS string `json:"dns"` + + // KubernetesVersion is the cluster version used when the Machine does not + // specify one. + KubernetesVersion string `json:"kubernetesVersion"` +} + +// NetbootSessionArtifacts snapshots immutable OCI sources and artifact paths. +type NetbootSessionArtifacts struct { + MachineImage NetbootSessionImage `json:"machineImage"` + NetbootImage NetbootSessionImage `json:"netbootImage"` + + // Files lists the named files an edge may request for this session. + // +listType=map + // +listMapKey=name + Files []NetbootSessionArtifact `json:"files"` +} + +// NetbootSessionImage identifies an OCI image resolved to an immutable digest. +type NetbootSessionImage struct { + // Reference is the source repository reference used to resolve the image. + // +kubebuilder:validation:MinLength=1 + Reference string `json:"reference"` + + // Digest is the immutable OCI manifest digest. + // +kubebuilder:validation:Pattern=`^sha256:[a-f0-9]{64}$` + Digest string `json:"digest"` + + // PullSecretRef references registry credentials without copying them into + // the session. + // +optional + PullSecretRef *NamespacedSecretReference `json:"pullSecretRef,omitempty"` +} + +// NetbootSessionArtifact maps a public artifact name to an immutable image path. +type NetbootSessionArtifact struct { + // Name is the stable route name used by edges and rendered templates. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Source selects the OCI image containing this file, or Session for + // content snapshotted directly into the session. + // +kubebuilder:validation:Enum=MachineImage;NetbootImage;Session + Source string `json:"source"` + + // Path is the absolute path within the unpacked OCI image. + // +kubebuilder:validation:Pattern=`^/` + Path string `json:"path"` + + // Size is the expected file size when known. + // +optional + // +kubebuilder:validation:Minimum=0 + Size *int64 `json:"size,omitempty"` +} + +// NetbootSessionStatus reports preparation and target-scoped milestones. +type NetbootSessionStatus struct { + // Phase is the current durable lifecycle phase. + // +optional + Phase NetbootSessionPhase `json:"phase,omitempty"` + + // CapabilityID identifies the signing key and capability generation without + // persisting a bearer capability. + // +optional + CapabilityID string `json:"capabilityID,omitempty"` + + // Conditions contain preparation, endpoint readiness, and target-scoped + // provisioning milestones. + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// NetbootSessionReference identifies the immutable session assigned to an +// operation target. +type NetbootSessionReference struct { + // Name is the NetbootSession name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // UID detects deletion and recreation of the session. + UID types.UID `json:"uid"` +} diff --git a/api/machina/v1alpha3/site_types.go b/api/machina/v1alpha3/site_types.go index a07907167..e368f4e7d 100644 --- a/api/machina/v1alpha3/site_types.go +++ b/api/machina/v1alpha3/site_types.go @@ -155,16 +155,6 @@ type MachinaComponentSpec struct { // MetalmanComponentSpec configures Metalman for a site. type MetalmanComponentSpec struct { SiteComponentSpec `json:",inline"` - - // DHCPAutoInterface lets Metalman choose the DHCP interface automatically. - // +optional - DHCPAutoInterface *bool `json:"dhcpAutoInterface,omitempty"` - - // Replicas is the desired number of Metalman replicas. Defaults to 1 when - // omitted. - // +kubebuilder:validation:Minimum=0 - // +optional - Replicas *int32 `json:"replicas,omitempty"` } // StorageComponentSpec configures unbounded-storage for a site. Storage daemon diff --git a/api/machina/v1alpha3/site_types_test.go b/api/machina/v1alpha3/site_types_test.go index 6cbf7a22b..bd4aa26c9 100644 --- a/api/machina/v1alpha3/site_types_test.go +++ b/api/machina/v1alpha3/site_types_test.go @@ -4,15 +4,43 @@ package v1alpha3 import ( + "os" "testing" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" ) +func TestMetalmanComponentSchemaExposesOnlyEnablement(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("../../../deploy/machina/crd/unbounded-cloud.io_sites.yaml") + if err != nil { + t.Fatalf("read Site CRD: %v", err) + } + + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("parse Site CRD: %v", err) + } + + metalman := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"].Properties["components"].Properties["metalman"] + if _, ok := metalman.Properties["enabled"]; !ok { + t.Error("Metalman component schema lacks enabled") + } + + for _, field := range []string{"dhcpAutoInterface", "replicas"} { + if _, ok := metalman.Properties[field]; ok { + t.Errorf("Metalman component schema still exposes %q", field) + } + } +} + func TestSiteResourceAndAddToScheme(t *testing.T) { gr := Resource("sites") if gr.Group != GroupVersion.Group || gr.Resource != "sites" { @@ -40,7 +68,6 @@ func TestDeepCopySiteAndList(t *testing.T) { enabled := true priority := int32(10) detectMultiplier := int32(3) - replicas := int32(3) receive := intstr.FromString("300ms") transmit := intstr.FromInt(400) @@ -72,8 +99,6 @@ func TestDeepCopySiteAndList(t *testing.T) { }, Metalman: &MetalmanComponentSpec{ SiteComponentSpec: SiteComponentSpec{Enabled: &enabled}, - DHCPAutoInterface: &enabled, - Replicas: &replicas, }, }, }, @@ -98,7 +123,6 @@ func TestDeepCopySiteAndList(t *testing.T) { site.Spec.NodeCidrs[0] = "10.99.0.0/16" site.Spec.PodCidrAssignments[0].CidrBlocks[0] = "10.250.0.0/16" site.Spec.HealthCheckSettings.DetectMultiplier = ptrInt32(9) - site.Spec.Components.Metalman.Replicas = ptrInt32(5) site.Status.Conditions[0].Status = metav1.ConditionFalse if copied.Spec.NodeCidrs[0] != "10.0.0.0/16" { @@ -113,10 +137,6 @@ func TestDeepCopySiteAndList(t *testing.T) { t.Fatalf("expected deep-copied health check settings to be isolated") } - if copied.Spec.Components.Metalman.Replicas == nil || *copied.Spec.Components.Metalman.Replicas != 3 { - t.Fatalf("expected deep-copied Metalman replicas to be isolated") - } - if copied.Status.Conditions[0].Status != metav1.ConditionTrue { t.Fatalf("expected deep-copied component condition to be isolated") } diff --git a/api/machina/v1alpha3/zz_generated.deepcopy.go b/api/machina/v1alpha3/zz_generated.deepcopy.go index 137751bce..b0a0b54b4 100644 --- a/api/machina/v1alpha3/zz_generated.deepcopy.go +++ b/api/machina/v1alpha3/zz_generated.deepcopy.go @@ -961,6 +961,11 @@ func (in *MachineOperationStatus) DeepCopy() *MachineOperationStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachineOperationTargetInput) DeepCopyInto(out *MachineOperationTargetInput) { *out = *in + if in.NetbootSessionRef != nil { + in, out := &in.NetbootSessionRef, &out.NetbootSessionRef + *out = new(NetbootSessionReference) + **out = **in + } if in.ProviderRef != nil { in, out := &in.ProviderRef, &out.ProviderRef *out = new(ProviderMachineSnapshot) @@ -1118,16 +1123,6 @@ func (in *MachineStatus) DeepCopy() *MachineStatus { func (in *MetalmanComponentSpec) DeepCopyInto(out *MetalmanComponentSpec) { *out = *in in.SiteComponentSpec.DeepCopyInto(&out.SiteComponentSpec) - if in.DHCPAutoInterface != nil { - in, out := &in.DHCPAutoInterface, &out.DHCPAutoInterface - *out = new(bool) - **out = **in - } - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetalmanComponentSpec. @@ -1155,6 +1150,467 @@ func (in *NamespacedSecretReference) DeepCopy() *NamespacedSecretReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpoint) DeepCopyInto(out *NetbootEndpoint) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpoint. +func (in *NetbootEndpoint) DeepCopy() *NetbootEndpoint { + if in == nil { + return nil + } + out := new(NetbootEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootEndpoint) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointClaim) DeepCopyInto(out *NetbootEndpointClaim) { + *out = *in + in.RenewedAt.DeepCopyInto(&out.RenewedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointClaim. +func (in *NetbootEndpointClaim) DeepCopy() *NetbootEndpointClaim { + if in == nil { + return nil + } + out := new(NetbootEndpointClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointList) DeepCopyInto(out *NetbootEndpointList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetbootEndpoint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointList. +func (in *NetbootEndpointList) DeepCopy() *NetbootEndpointList { + if in == nil { + return nil + } + out := new(NetbootEndpointList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootEndpointList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointSpec) DeepCopyInto(out *NetbootEndpointSpec) { + *out = *in + in.TLS.DeepCopyInto(&out.TLS) + if in.ManagedL2 != nil { + in, out := &in.ManagedL2, &out.ManagedL2 + *out = new(NetbootManagedL2Spec) + (*in).DeepCopyInto(*out) + } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(NetbootHTTPEndpointSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointSpec. +func (in *NetbootEndpointSpec) DeepCopy() *NetbootEndpointSpec { + if in == nil { + return nil + } + out := new(NetbootEndpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointStatus) DeepCopyInto(out *NetbootEndpointStatus) { + *out = *in + if in.Claim != nil { + in, out := &in.Claim, &out.Claim + *out = new(NetbootEndpointClaim) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointStatus. +func (in *NetbootEndpointStatus) DeepCopy() *NetbootEndpointStatus { + if in == nil { + return nil + } + out := new(NetbootEndpointStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootEndpointTLS) DeepCopyInto(out *NetbootEndpointTLS) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(NamespacedSecretReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootEndpointTLS. +func (in *NetbootEndpointTLS) DeepCopy() *NetbootEndpointTLS { + if in == nil { + return nil + } + out := new(NetbootEndpointTLS) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootHTTPEndpointSpec) DeepCopyInto(out *NetbootHTTPEndpointSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootHTTPEndpointSpec. +func (in *NetbootHTTPEndpointSpec) DeepCopy() *NetbootHTTPEndpointSpec { + if in == nil { + return nil + } + out := new(NetbootHTTPEndpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootManagedL2Spec) DeepCopyInto(out *NetbootManagedL2Spec) { + *out = *in + in.NodeSelector.DeepCopyInto(&out.NodeSelector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootManagedL2Spec. +func (in *NetbootManagedL2Spec) DeepCopy() *NetbootManagedL2Spec { + if in == nil { + return nil + } + out := new(NetbootManagedL2Spec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSession) DeepCopyInto(out *NetbootSession) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSession. +func (in *NetbootSession) DeepCopy() *NetbootSession { + if in == nil { + return nil + } + out := new(NetbootSession) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootSession) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionArtifact) DeepCopyInto(out *NetbootSessionArtifact) { + *out = *in + if in.Size != nil { + in, out := &in.Size, &out.Size + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionArtifact. +func (in *NetbootSessionArtifact) DeepCopy() *NetbootSessionArtifact { + if in == nil { + return nil + } + out := new(NetbootSessionArtifact) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionArtifacts) DeepCopyInto(out *NetbootSessionArtifacts) { + *out = *in + in.MachineImage.DeepCopyInto(&out.MachineImage) + in.NetbootImage.DeepCopyInto(&out.NetbootImage) + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = make([]NetbootSessionArtifact, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionArtifacts. +func (in *NetbootSessionArtifacts) DeepCopy() *NetbootSessionArtifacts { + if in == nil { + return nil + } + out := new(NetbootSessionArtifacts) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionBoot) DeepCopyInto(out *NetbootSessionBoot) { + *out = *in + if in.DHCPLeases != nil { + in, out := &in.DHCPLeases, &out.DHCPLeases + *out = make([]DHCPLease, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionBoot. +func (in *NetbootSessionBoot) DeepCopy() *NetbootSessionBoot { + if in == nil { + return nil + } + out := new(NetbootSessionBoot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionCluster) DeepCopyInto(out *NetbootSessionCluster) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionCluster. +func (in *NetbootSessionCluster) DeepCopy() *NetbootSessionCluster { + if in == nil { + return nil + } + out := new(NetbootSessionCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionEndpointSnapshot) DeepCopyInto(out *NetbootSessionEndpointSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionEndpointSnapshot. +func (in *NetbootSessionEndpointSnapshot) DeepCopy() *NetbootSessionEndpointSnapshot { + if in == nil { + return nil + } + out := new(NetbootSessionEndpointSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionImage) DeepCopyInto(out *NetbootSessionImage) { + *out = *in + if in.PullSecretRef != nil { + in, out := &in.PullSecretRef, &out.PullSecretRef + *out = new(NamespacedSecretReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionImage. +func (in *NetbootSessionImage) DeepCopy() *NetbootSessionImage { + if in == nil { + return nil + } + out := new(NetbootSessionImage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionList) DeepCopyInto(out *NetbootSessionList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetbootSession, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionList. +func (in *NetbootSessionList) DeepCopy() *NetbootSessionList { + if in == nil { + return nil + } + out := new(NetbootSessionList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetbootSessionList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionObjectSnapshot) DeepCopyInto(out *NetbootSessionObjectSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionObjectSnapshot. +func (in *NetbootSessionObjectSnapshot) DeepCopy() *NetbootSessionObjectSnapshot { + if in == nil { + return nil + } + out := new(NetbootSessionObjectSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionProvisioning) DeepCopyInto(out *NetbootSessionProvisioning) { + *out = *in + out.Cluster = in.Cluster + if in.Kubernetes != nil { + in, out := &in.Kubernetes, &out.Kubernetes + *out = new(KubernetesSpec) + (*in).DeepCopyInto(*out) + } + if in.Agent != nil { + in, out := &in.Agent, &out.Agent + *out = new(AgentSpec) + (*in).DeepCopyInto(*out) + } + if in.ProviderLabels != nil { + in, out := &in.ProviderLabels, &out.ProviderLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionProvisioning. +func (in *NetbootSessionProvisioning) DeepCopy() *NetbootSessionProvisioning { + if in == nil { + return nil + } + out := new(NetbootSessionProvisioning) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionReference) DeepCopyInto(out *NetbootSessionReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionReference. +func (in *NetbootSessionReference) DeepCopy() *NetbootSessionReference { + if in == nil { + return nil + } + out := new(NetbootSessionReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionSpec) DeepCopyInto(out *NetbootSessionSpec) { + *out = *in + out.Machine = in.Machine + out.Operation = in.Operation + out.Endpoint = in.Endpoint + in.Boot.DeepCopyInto(&out.Boot) + in.Provisioning.DeepCopyInto(&out.Provisioning) + in.Artifacts.DeepCopyInto(&out.Artifacts) + in.ExpiresAt.DeepCopyInto(&out.ExpiresAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionSpec. +func (in *NetbootSessionSpec) DeepCopy() *NetbootSessionSpec { + if in == nil { + return nil + } + out := new(NetbootSessionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetbootSessionStatus) DeepCopyInto(out *NetbootSessionStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetbootSessionStatus. +func (in *NetbootSessionStatus) DeepCopy() *NetbootSessionStatus { + if in == nil { + return nil + } + out := new(NetbootSessionStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PXESpec) DeepCopyInto(out *PXESpec) { *out = *in diff --git a/cmd/kubectl-unbounded/app/cmd_site.go b/cmd/kubectl-unbounded/app/cmd_site.go index 0a184aaa4..8ca5fd547 100644 --- a/cmd/kubectl-unbounded/app/cmd_site.go +++ b/cmd/kubectl-unbounded/app/cmd_site.go @@ -14,7 +14,8 @@ func siteCommandGroup() *cobra.Command { } cmd.AddCommand( - siteInitCommand()) + siteInitCommand(), + siteBootstrapNetbootCommand(nil)) return cmd } diff --git a/cmd/kubectl-unbounded/app/machine_ops.go b/cmd/kubectl-unbounded/app/machine_ops.go index a6e4fd012..1937303d2 100644 --- a/cmd/kubectl-unbounded/app/machine_ops.go +++ b/cmd/kubectl-unbounded/app/machine_ops.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" ) // ANSI color/style codes for terminal output. @@ -32,6 +33,7 @@ func buildScheme() *runtime.Scheme { s := runtime.NewScheme() utilruntime.Must(corev1.AddToScheme(s)) utilruntime.Must(v1alpha3.AddToScheme(s)) + utilruntime.Must(netv1alpha1.AddToScheme(s)) return s } diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go new file mode 100644 index 000000000..5df145dd0 --- /dev/null +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot.go @@ -0,0 +1,1467 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package app + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "sync" + "syscall" + "time" + + "github.com/spf13/cobra" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/kube" + "github.com/Azure/unbounded/internal/net/nodeagent" + "github.com/Azure/unbounded/internal/unbounded" +) + +const ( + defaultBootstrapNetbootTimeout = 30 * time.Minute + defaultBootstrapPollInterval = time.Second +) + +type siteBootstrapNetbootHandler struct { + site string + machine string + interfaceName string + address string + gatewayExternalAddress string + endpointName string + httpPort int + kubeconfigPath string + namespace string + metalmanBinary string + timeout time.Duration + routedCIDRs []string + resources client.Client + kubeClient kubernetes.Interface + restConfig *rest.Config + pollInterval time.Duration + dependencies bootstrapNetbootDependencies +} + +type bootstrapNetbootDependencies struct { + resolveBinary func() (string, error) + preflightNetwork func() error + localPort func() (int, error) + portForward func(context.Context, int) (*bootstrapPortForward, error) + edgeToken func(context.Context) (*bootstrapEdgeToken, error) + startEdge func(string, []string) (bootstrapEdgeProcess, error) + dialEdge func(context.Context, string) error + preflightGateway func() error + gatewayRuntimeDir func() (string, error) + startGateway func(context.Context, nodeagent.ExternalGatewayOptions) (bootstrapEdgeProcess, error) +} + +type bootstrapNetbootState struct { + originalEndpointRef string +} + +type bootstrapPortForwardAttempt interface { + Done() <-chan error + Stop() +} + +type bootstrapPortForwardStarter func( + ctx context.Context, + podName string, + localPort int, + remotePort int, +) (bootstrapPortForwardAttempt, error) + +type bootstrapPortForward struct { + url string + cancel context.CancelFunc + done chan struct{} +} + +type bootstrapEdgeToken struct { + path string + cancel context.CancelFunc + done chan struct{} + once sync.Once +} + +type bootstrapEdgeProcess interface { + Done() <-chan struct{} + Err() error + Stop(ctx context.Context) error +} + +type commandBootstrapEdgeProcess struct { + cmd *exec.Cmd + done chan struct{} + stopOnce sync.Once + mu sync.Mutex + err error +} + +type embeddedBootstrapGatewayProcess struct { + cancel context.CancelFunc + done chan struct{} + once sync.Once + mu sync.Mutex + err error +} + +func (p *commandBootstrapEdgeProcess) Done() <-chan struct{} { + return p.done +} + +func (p *commandBootstrapEdgeProcess) Err() error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.err +} + +func (p *commandBootstrapEdgeProcess) Stop(ctx context.Context) error { + var signalErr error + + p.stopOnce.Do(func() { + signalErr = p.cmd.Process.Signal(syscall.SIGTERM) + }) + + if signalErr != nil && !errors.Is(signalErr, os.ErrProcessDone) { + return fmt.Errorf("stop Metalman edge: %w", signalErr) + } + + select { + case <-p.done: + err := p.Err() + if err != nil && !isSignalExit(err) { + return fmt.Errorf("wait for Metalman edge: %w", err) + } + + return nil + case <-ctx.Done(): + if err := p.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { + return fmt.Errorf("kill Metalman edge: %w", err) + } + + <-p.done + + return ctx.Err() + } +} + +func (p *embeddedBootstrapGatewayProcess) Done() <-chan struct{} { return p.done } + +func (p *embeddedBootstrapGatewayProcess) Err() error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.err +} + +func (p *embeddedBootstrapGatewayProcess) Stop(ctx context.Context) error { + p.once.Do(p.cancel) + + select { + case <-p.done: + return p.Err() + case <-ctx.Done(): + return ctx.Err() + } +} + +func (t *bootstrapEdgeToken) Path() string { + return t.path +} + +func (t *bootstrapEdgeToken) Close() error { + var err error + + t.once.Do(func() { + t.cancel() + <-t.done + err = os.RemoveAll(filepath.Dir(t.path)) + }) + + return err +} + +func (f *bootstrapPortForward) URL() string { + return f.url +} + +func (f *bootstrapPortForward) Close() error { + f.cancel() + <-f.done + + return nil +} + +func siteBootstrapNetbootCommand(handler *siteBootstrapNetbootHandler) *cobra.Command { + if handler == nil { + handler = &siteBootstrapNetbootHandler{} + } + + cmd := &cobra.Command{ + Use: "bootstrap-netboot SITE", + Short: "Temporarily serve netboot from this machine until the first Site node is Ready", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + handler.site = args[0] + + return handler.execute(cmd.Context()) + }, + } + + cmd.Flags().StringVar(&handler.machine, "machine", "", "Machine whose Node readiness completes bootstrap") + cmd.Flags().StringVar(&handler.interfaceName, "interface", "", "Provisioning network interface for DHCP and TFTP") + cmd.Flags().StringVar(&handler.address, "address", "", "Address on the provisioning interface advertised to clients") + cmd.Flags().StringVar(&handler.endpointName, "endpoint-name", "", "Ephemeral NetbootEndpoint name (generated when omitted)") + cmd.Flags().IntVar(&handler.httpPort, "http-port", 8880, "Local HTTP artifact port") + cmd.Flags().StringVar(&handler.kubeconfigPath, "kubeconfig", "", "Path to kubeconfig file") + cmd.Flags().StringVar(&handler.namespace, "namespace", unbounded.SystemNamespace(), "Namespace containing Metalman workloads") + cmd.Flags().StringVar(&handler.metalmanBinary, "metalman-binary", "", "Path to the metalman binary") + cmd.Flags().StringVar(&handler.gatewayExternalAddress, "gateway-external-address", "", "WireGuard address reachable by remote gateway peers (defaults to --address)") + cmd.Flags().DurationVar(&handler.timeout, "timeout", defaultBootstrapNetbootTimeout, "Maximum time to wait for the designated Node to become Ready") + cmd.Flags().StringSliceVar(&handler.routedCIDRs, "routed-cidr", nil, "CIDR routed through an ephemeral external gateway (repeatable)") + + if err := cmd.MarkFlagRequired("machine"); err != nil { + panic(fmt.Sprintf("mark machine flag required: %v", err)) + } + + if err := cmd.MarkFlagRequired("interface"); err != nil { + panic(fmt.Sprintf("mark interface flag required: %v", err)) + } + + if err := cmd.MarkFlagRequired("address"); err != nil { + panic(fmt.Sprintf("mark address flag required: %v", err)) + } + + return cmd +} + +func (h *siteBootstrapNetbootHandler) execute(ctx context.Context) (retErr error) { + if h.timeout <= 0 { + h.timeout = defaultBootstrapNetbootTimeout + } + + if h.endpointName == "" { + h.endpointName = "bootstrap-" + h.machine + } + + if h.gatewayExternalAddress == "" { + h.gatewayExternalAddress = h.address + } + + if err := h.initializeDependencies(); err != nil { + return err + } + + binary, err := h.dependencies.resolveBinary() + if err != nil { + return err + } + + if err := h.dependencies.preflightNetwork(); err != nil { + return err + } + + var gatewayRuntimeDir string + + if len(h.routedCIDRs) > 0 { + if err := h.dependencies.preflightGateway(); err != nil { + return err + } + + gatewayRuntimeDir, err = h.dependencies.gatewayRuntimeDir() + if err != nil { + return fmt.Errorf("create external gateway runtime directory: %w", err) + } + + defer func() { retErr = errors.Join(retErr, os.RemoveAll(gatewayRuntimeDir)) }() + } + + if err := h.initializeClients(); err != nil { + return err + } + + h.initializeRuntimeDependencies() + + runCtx, cancel := context.WithTimeout(ctx, h.timeout) + defer cancel() + + state, err := h.prepareClusterResources(runCtx) + if err != nil { + return err + } + + resourcesPrepared := true + + var ( + forward *bootstrapPortForward + token *bootstrapEdgeToken + process bootstrapEdgeProcess + gatewayProcess bootstrapEdgeProcess + gatewayPrepared bool + ) + + defer func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cleanupCancel() + + if resourcesPrepared { + retErr = errors.Join(retErr, h.restoreClusterResources(cleanupCtx, state)) + } + + if process != nil { + retErr = errors.Join(retErr, process.Stop(cleanupCtx)) + } + + if gatewayProcess != nil { + retErr = errors.Join(retErr, gatewayProcess.Stop(cleanupCtx)) + } + + if forward != nil { + retErr = errors.Join(retErr, forward.Close()) + } + + if token != nil { + retErr = errors.Join(retErr, token.Close()) + } + + if gatewayPrepared { + retErr = errors.Join(retErr, h.cleanupGatewayResources(cleanupCtx)) + } + }() + + if len(h.routedCIDRs) > 0 { + if err := h.prepareGatewayResources(runCtx); err != nil { + return err + } + + gatewayPrepared = true + + gatewayProcess, err = h.dependencies.startGateway(runCtx, nodeagent.ExternalGatewayOptions{ + NodeName: h.endpointName, RuntimeDir: gatewayRuntimeDir, RESTConfig: h.restConfig, + }) + if err != nil { + return fmt.Errorf("start external gateway dataplane: %w", err) + } + + if err := h.waitForGatewayReady(runCtx, gatewayProcess); err != nil { + return err + } + } + + if err := h.waitForMetalman(runCtx); err != nil { + return err + } + + localPort, err := h.dependencies.localPort() + if err != nil { + return fmt.Errorf("select local port for Metalman server: %w", err) + } + + forward, err = h.dependencies.portForward(runCtx, localPort) + if err != nil { + return err + } + + token, err = h.dependencies.edgeToken(runCtx) + if err != nil { + return err + } + + process, err = h.dependencies.startEdge(binary, h.edgeArguments(forward.URL(), token.Path())) + if err != nil { + return err + } + + if err := h.waitForEdgeReady(runCtx, process, h.dependencies.dialEdge); err != nil { + return err + } + + if err := h.claimEndpoint(runCtx, "kubectl-unbounded/"+h.endpointName); err != nil { + return err + } + + var machine v1alpha3.Machine + if err := h.resources.Get(runCtx, client.ObjectKey{Name: h.machine}, &machine); err != nil { + return fmt.Errorf("get designated Machine %s: %w", h.machine, err) + } + + return h.waitForNodeReadyAndProcesses(runCtx, designatedNodeName(&machine), process, gatewayProcess) +} + +func (h *siteBootstrapNetbootHandler) initializeDependencies() error { + if h.dependencies.resolveBinary == nil { + h.dependencies.resolveBinary = func() (string, error) { + return h.resolveMetalmanBinary(exec.LookPath) + } + } + + if h.dependencies.preflightNetwork == nil { + h.dependencies.preflightNetwork = h.validateProvisioningInterface + } + + if h.dependencies.localPort == nil { + h.dependencies.localPort = availableLoopbackPort + } + + if h.dependencies.startEdge == nil { + h.dependencies.startEdge = func(binary string, args []string) (bootstrapEdgeProcess, error) { + return startBootstrapEdgeProcess(binary, args, os.Stdout, os.Stderr) + } + } + + if h.dependencies.dialEdge == nil { + h.dependencies.dialEdge = dialBootstrapEdge + } + + if h.dependencies.preflightGateway == nil { + h.dependencies.preflightGateway = h.validateExternalGateway + } + + if h.dependencies.gatewayRuntimeDir == nil { + h.dependencies.gatewayRuntimeDir = func() (string, error) { + return os.MkdirTemp("/run", "unbounded-netboot-") + } + } + + if h.dependencies.startGateway == nil { + h.dependencies.startGateway = startEmbeddedBootstrapGateway + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) initializeClients() error { + if h.resources != nil && h.kubeClient != nil && h.restConfig != nil { + return nil + } + + kubeClient, config, err := kube.ClientAndConfigFromFile(getKubeconfigPath(h.kubeconfigPath)) + if err != nil { + return fmt.Errorf("create Kubernetes client for netboot bootstrap: %w", err) + } + + resources, err := client.New(config, client.Options{Scheme: buildScheme()}) + if err != nil { + return fmt.Errorf("create resource client for netboot bootstrap: %w", err) + } + + h.kubeClient = kubeClient + h.resources = resources + h.restConfig = config + + return nil +} + +func (h *siteBootstrapNetbootHandler) initializeRuntimeDependencies() { + if h.dependencies.portForward == nil { + starter := newSPDYBootstrapPortForwardStarter(h.kubeClient, h.restConfig, h.namespace) + h.dependencies.portForward = func(ctx context.Context, localPort int) (*bootstrapPortForward, error) { + return newBootstrapPortForward( + ctx, + h.kubeClient, + h.namespace, + "metalman-server-"+h.site, + localPort, + 8880, + starter, + h.pollInterval, + ) + } + } + + if h.dependencies.edgeToken == nil { + h.dependencies.edgeToken = func(ctx context.Context) (*bootstrapEdgeToken, error) { + return newBootstrapEdgeToken(ctx, h.kubeClient, h.namespace, "", 0) + } + } +} + +func (h *siteBootstrapNetbootHandler) validateProvisioningInterface() error { + interfaceInfo, err := net.InterfaceByName(h.interfaceName) + if err != nil { + return fmt.Errorf("find provisioning interface %s: %w", h.interfaceName, err) + } + + wanted := net.ParseIP(h.address) + if wanted == nil || wanted.To4() == nil { + return fmt.Errorf("provisioning address %q must be an IPv4 address", h.address) + } + + addresses, err := interfaceInfo.Addrs() + if err != nil { + return fmt.Errorf("list addresses on provisioning interface %s: %w", h.interfaceName, err) + } + + for _, address := range addresses { + ip, _, parseErr := net.ParseCIDR(address.String()) + if parseErr == nil && ip.Equal(wanted) { + return nil + } + } + + return fmt.Errorf("provisioning interface %s does not own address %s", h.interfaceName, h.address) +} + +func (h *siteBootstrapNetbootHandler) validateExternalGateway() error { + if err := h.validateRoutedCIDRs(); err != nil { + return err + } + + if os.Geteuid() != 0 { + return fmt.Errorf("external gateway dataplane requires root privileges") + } + + if ip := net.ParseIP(h.gatewayExternalAddress); ip == nil || ip.To4() == nil { + return fmt.Errorf("gateway external address %q must be an IPv4 address", h.gatewayExternalAddress) + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) validateRoutedCIDRs() error { + for _, routedCIDR := range h.routedCIDRs { + if _, _, err := net.ParseCIDR(routedCIDR); err != nil { + return fmt.Errorf("invalid routed CIDR %q: %w", routedCIDR, err) + } + } + + return nil +} + +func availableLoopbackPort() (int, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer listener.Close() //nolint:errcheck // Best-effort cleanup after reserving the port. + + tcpAddress, ok := listener.Addr().(*net.TCPAddr) + if !ok { + return 0, fmt.Errorf("loopback listener returned address type %T", listener.Addr()) + } + + return tcpAddress.Port, nil +} + +func dialBootstrapEdge(ctx context.Context, address string) error { + dialer := net.Dialer{} + + connection, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return err + } + + return connection.Close() +} + +func (h *siteBootstrapNetbootHandler) prepareClusterResources(ctx context.Context) (bootstrapNetbootState, error) { + var site v1alpha3.Site + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.site}, &site); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("get Site %s: %w", h.site, err) + } + + enabled := true + + if site.Spec.Components.Metalman == nil { + site.Spec.Components.Metalman = &v1alpha3.MetalmanComponentSpec{} + } + + site.Spec.Components.Metalman.Enabled = &enabled + + if err := h.resources.Update(ctx, &site); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("enable Metalman for Site %s: %w", h.site, err) + } + + var machine v1alpha3.Machine + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.machine}, &machine); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("get Machine %s: %w", h.machine, err) + } + + if machine.Labels[v1alpha3.MachineSiteLabelKey] != h.site { + return bootstrapNetbootState{}, fmt.Errorf("machine %s does not belong to Site %s", h.machine, h.site) + } + + netboot := machine.Spec.Netboot() + if netboot == nil { + return bootstrapNetbootState{}, fmt.Errorf("machine %s has no netboot configuration", h.machine) + } + + state := bootstrapNetbootState{originalEndpointRef: netboot.EndpointRef} + externalURL := (&url.URL{ + Scheme: "http", + Host: net.JoinHostPort(h.address, strconv.Itoa(h.httpPort)), + }).String() + endpoint := &v1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: v1alpha3.NetbootEndpointSpec{ + SiteRef: h.site, + Type: v1alpha3.NetbootEndpointTypeExternalL2, + ExternalURL: externalURL, + TLS: v1alpha3.NetbootEndpointTLS{ + Trust: v1alpha3.NetbootEndpointTrustTrustedLAN, + Mode: v1alpha3.NetbootEndpointTLSDisabled, + }, + }, + } + + if err := h.resources.Create(ctx, endpoint); err != nil { + return bootstrapNetbootState{}, fmt.Errorf("create NetbootEndpoint %s: %w", h.endpointName, err) + } + + netboot.EndpointRef = h.endpointName + if err := h.resources.Update(ctx, &machine); err != nil { + if deleteErr := h.resources.Delete(ctx, endpoint); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return bootstrapNetbootState{}, errors.Join( + fmt.Errorf("select bootstrap endpoint for Machine %s: %w", h.machine, err), + fmt.Errorf("delete NetbootEndpoint %s after Machine update failure: %w", h.endpointName, deleteErr), + ) + } + + return bootstrapNetbootState{}, fmt.Errorf("select bootstrap endpoint for Machine %s: %w", h.machine, err) + } + + return state, nil +} + +func (h *siteBootstrapNetbootHandler) restoreClusterResources(ctx context.Context, state bootstrapNetbootState) error { + var machine v1alpha3.Machine + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.machine}, &machine); err != nil { + return fmt.Errorf("get Machine %s for cleanup: %w", h.machine, err) + } + + if netboot := machine.Spec.Netboot(); netboot != nil && netboot.EndpointRef == h.endpointName { + netboot.EndpointRef = state.originalEndpointRef + + if err := h.resources.Update(ctx, &machine); err != nil { + return fmt.Errorf("restore Machine %s endpoint: %w", h.machine, err) + } + } + + endpoint := &v1alpha3.NetbootEndpoint{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + if err := h.resources.Delete(ctx, endpoint); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete NetbootEndpoint %s: %w", h.endpointName, err) + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) prepareGatewayResources(ctx context.Context) (retErr error) { + if err := h.validateRoutedCIDRs(); err != nil { + return err + } + + protocol := netv1alpha1.TunnelProtocolWireGuard + enabled := true + selector := map[string]string{"net.unbounded-cloud.io/bootstrap-gateway": h.endpointName} + + pool := &netv1alpha1.GatewayPool{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: netv1alpha1.GatewayPoolSpec{ + Type: "External", + NodeSelector: selector, + RoutedCidrs: append([]string(nil), h.routedCIDRs...), + TunnelProtocol: &protocol, + }, + } + if err := h.resources.Create(ctx, pool); err != nil { + return fmt.Errorf("create bootstrap GatewayPool %s: %w", h.endpointName, err) + } + + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, h.cleanupGatewayResources(context.WithoutCancel(ctx))) + } + }() + + assignment := &netv1alpha1.SiteGatewayPoolAssignment{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: netv1alpha1.SiteGatewayPoolAssignmentSpec{ + Enabled: &enabled, + Sites: []string{h.site}, + GatewayPools: []string{h.endpointName}, + TunnelProtocol: &protocol, + }, + } + if err := h.resources.Create(ctx, assignment); err != nil { + return fmt.Errorf("create bootstrap SiteGatewayPoolAssignment %s: %w", h.endpointName, err) + } + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: h.endpointName, + Labels: map[string]string{ + "net.unbounded-cloud.io/bootstrap-gateway": h.endpointName, + "net.unbounded-cloud.io/external-node": "true", + }, + }, + Spec: corev1.NodeSpec{ + Unschedulable: true, + Taints: []corev1.Taint{{ + Key: "net.unbounded-cloud.io/gateway-node", Value: "true", Effect: corev1.TaintEffectNoSchedule, + }}, + }, + } + + createdNode, err := h.kubeClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create bootstrap gateway Node %s: %w", h.endpointName, err) + } + + createdNode.Status.Addresses = []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: h.address}, + {Type: corev1.NodeExternalIP, Address: h.gatewayExternalAddress}, + } + if _, err := h.kubeClient.CoreV1().Nodes().UpdateStatus(ctx, createdNode, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("set bootstrap gateway Node %s addresses: %w", h.endpointName, err) + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) cleanupGatewayResources(ctx context.Context) error { + assignment := &netv1alpha1.SiteGatewayPoolAssignment{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + + assignmentErr := h.resources.Delete(ctx, assignment) + if apierrors.IsNotFound(assignmentErr) { + assignmentErr = nil + } + + if assignmentErr != nil { + assignmentErr = fmt.Errorf("delete bootstrap SiteGatewayPoolAssignment %s: %w", h.endpointName, assignmentErr) + } + + nodeErr := h.kubeClient.CoreV1().Nodes().Delete(ctx, h.endpointName, metav1.DeleteOptions{}) + if apierrors.IsNotFound(nodeErr) { + nodeErr = nil + } + + if nodeErr != nil { + nodeErr = fmt.Errorf("delete bootstrap gateway Node %s: %w", h.endpointName, nodeErr) + } + + pool := &netv1alpha1.GatewayPool{ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}} + + poolErr := h.resources.Delete(ctx, pool) + if apierrors.IsNotFound(poolErr) { + poolErr = nil + } + + if poolErr != nil { + poolErr = fmt.Errorf("delete bootstrap GatewayPool %s: %w", h.endpointName, poolErr) + } + + return errors.Join(assignmentErr, nodeErr, poolErr) +} + +func (h *siteBootstrapNetbootHandler) metalmanDeploymentsReady(ctx context.Context) (bool, error) { + for _, name := range []string{"metalman-controller-" + h.site, "metalman-server-" + h.site} { + deployment, err := h.kubeClient.AppsV1().Deployments(h.namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("get Deployment %s/%s: %w", h.namespace, name, err) + } + + if !deploymentRolloutComplete(deployment) { + return false, nil + } + } + + return true, nil +} + +func (h *siteBootstrapNetbootHandler) waitForMetalman(ctx context.Context) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + ready, err := h.metalmanDeploymentsReady(ctx) + if err != nil { + return err + } + + if ready { + return nil + } + + select { + case <-ctx.Done(): + return fmt.Errorf("wait for Metalman controller and server: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func (h *siteBootstrapNetbootHandler) edgeArguments(backendURL, tokenFile string) []string { + return []string{ + "edge", + "--backend-url=" + backendURL, + "--endpoint=" + h.endpointName, + "--bind-address=" + h.address, + "--http-port=" + strconv.Itoa(h.httpPort), + "--dhcp-enabled", + "--dhcp-interface=" + h.interfaceName, + "--dhcp-server-ip=" + h.address, + "--edge-token-file=" + tokenFile, + "--tftp-enabled", + "--tftp-bind-address=" + h.address, + } +} + +func (h *siteBootstrapNetbootHandler) resolveMetalmanBinary(lookPath func(string) (string, error)) (string, error) { + if h.metalmanBinary != "" { + info, err := os.Stat(h.metalmanBinary) + if err != nil { + return "", fmt.Errorf("metalman binary %q is not accessible: %w", h.metalmanBinary, err) + } + + if info.IsDir() || info.Mode()&0o111 == 0 { + return "", fmt.Errorf("metalman binary %q is not executable", h.metalmanBinary) + } + + return h.metalmanBinary, nil + } + + path, err := lookPath("metalman") + if err != nil { + return "", fmt.Errorf("find metalman executable: %w; install it or set --metalman-binary", err) + } + + return path, nil +} + +func (h *siteBootstrapNetbootHandler) waitForEdgeReady( + ctx context.Context, + process bootstrapEdgeProcess, + dial func(context.Context, string) error, +) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + + address := net.JoinHostPort(h.address, strconv.Itoa(h.httpPort)) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + if err := dial(ctx, address); err == nil { + return nil + } + + select { + case <-process.Done(): + if err := process.Err(); err != nil { + return fmt.Errorf("metalman edge exited before becoming ready: %w", err) + } + + return errors.New("metalman edge exited before becoming ready") + case <-ctx.Done(): + return fmt.Errorf("wait for Metalman edge listener: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func startBootstrapEdgeProcess(binary string, args []string, stdout, stderr io.Writer) (bootstrapEdgeProcess, error) { + cmd := exec.Command(binary, args...) + cmd.Stdout = stdout + + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start Metalman edge: %w", err) + } + + process := &commandBootstrapEdgeProcess{cmd: cmd, done: make(chan struct{})} + + go func() { + err := cmd.Wait() + + process.mu.Lock() + process.err = err + process.mu.Unlock() + close(process.done) + }() + + return process, nil +} + +func startEmbeddedBootstrapGateway( + ctx context.Context, + options nodeagent.ExternalGatewayOptions, +) (bootstrapEdgeProcess, error) { + runCtx, cancel := context.WithCancel(ctx) + process := &embeddedBootstrapGatewayProcess{cancel: cancel, done: make(chan struct{})} + + go func() { + err := nodeagent.RunExternalGateway(runCtx, options) + + process.mu.Lock() + process.err = err + process.mu.Unlock() + close(process.done) + }() + + return process, nil +} + +func isSignalExit(err error) bool { + var exitError *exec.ExitError + if !errors.As(err, &exitError) || exitError.ProcessState == nil { + return false + } + + status, ok := exitError.Sys().(syscall.WaitStatus) + + return ok && status.Signaled() +} + +func (h *siteBootstrapNetbootHandler) claimEndpoint(ctx context.Context, identity string) error { + var endpoint v1alpha3.NetbootEndpoint + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint); err != nil { + return fmt.Errorf("get NetbootEndpoint %s: %w", h.endpointName, err) + } + + now := metav1.Now() + endpoint.Status.ObservedGeneration = endpoint.Generation + endpoint.Status.Claim = &v1alpha3.NetbootEndpointClaim{ + HolderIdentity: identity, + RenewedAt: now, + } + metaSetStatusCondition(&endpoint.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + ObservedGeneration: endpoint.Generation, + Reason: "ExternalEdgeReady", + Message: "administrator bootstrap edge is ready", + LastTransitionTime: now, + }) + + if err := h.resources.Status().Update(ctx, &endpoint); err != nil { + return fmt.Errorf("claim NetbootEndpoint %s: %w", h.endpointName, err) + } + + return nil +} + +func (h *siteBootstrapNetbootHandler) waitForNodeReady(ctx context.Context, nodeName string) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + node, err := h.kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("get designated Node %s: %w", nodeName, err) + } + + if err == nil && nodeReady(node) { + return nil + } + + select { + case <-ctx.Done(): + return fmt.Errorf("wait for designated Node %s to become Ready: %w", nodeName, ctx.Err()) + case <-ticker.C: + } + } +} + +func (h *siteBootstrapNetbootHandler) waitForNodeReadyAndProcesses( + ctx context.Context, + nodeName string, + edge bootstrapEdgeProcess, + gateway bootstrapEdgeProcess, +) error { + waitCtx, cancel := context.WithCancel(ctx) + defer cancel() + + ready := make(chan error, 1) + + go func() { ready <- h.waitForNodeReady(waitCtx, nodeName) }() + + var gatewayDone <-chan struct{} + if gateway != nil { + gatewayDone = gateway.Done() + } + + select { + case err := <-ready: + return err + case <-edge.Done(): + return processExitError("metalman edge", nodeName, edge.Err()) + case <-gatewayDone: + return processExitError("external gateway", nodeName, gateway.Err()) + case <-ctx.Done(): + return fmt.Errorf("wait for designated Node %s while bootstrap dataplanes are running: %w", nodeName, ctx.Err()) + } +} + +func processExitError(name, nodeName string, err error) error { + if err != nil { + return fmt.Errorf("%s exited before designated Node %s became Ready: %w", name, nodeName, err) + } + + return fmt.Errorf("%s exited before designated Node %s became Ready", name, nodeName) +} + +func (h *siteBootstrapNetbootHandler) waitForGatewayReady(ctx context.Context, process bootstrapEdgeProcess) error { + interval := h.pollInterval + if interval <= 0 { + interval = defaultBootstrapPollInterval + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + ready, err := h.gatewayReady(ctx) + if err != nil { + return err + } + + if ready { + return nil + } + + select { + case <-process.Done(): + if err := process.Err(); err != nil { + return fmt.Errorf("external gateway exited before becoming ready: %w", err) + } + + return fmt.Errorf("external gateway exited before becoming ready") + case <-ctx.Done(): + return fmt.Errorf("wait for external gateway readiness: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func (h *siteBootstrapNetbootHandler) gatewayReady(ctx context.Context) (bool, error) { + node, err := h.kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("get bootstrap gateway Node %s: %w", h.endpointName, err) + } + + if node.Annotations["net.unbounded-cloud.io/wg-pubkey"] == "" { + return false, nil + } + + var pool netv1alpha1.GatewayPool + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool); err != nil { + return false, fmt.Errorf("get bootstrap GatewayPool %s: %w", h.endpointName, err) + } + + if pool.Status.NodeCount != 1 || !containsString(pool.Status.ConnectedSites, h.site) { + return false, nil + } + + matchedNode := false + + for _, poolNode := range pool.Status.Nodes { + if poolNode.Name == h.endpointName && poolNode.WireGuardPublicKey != "" { + matchedNode = true + break + } + } + + if !matchedNode { + return false, nil + } + + var gatewayNode netv1alpha1.GatewayPoolNode + if err := h.resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &gatewayNode); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + + return false, fmt.Errorf("get bootstrap GatewayPoolNode %s: %w", h.endpointName, err) + } + + return !gatewayNode.Status.LastUpdated.IsZero() && len(gatewayNode.Status.Routes) > 0, nil +} + +func containsString(values []string, wanted string) bool { + for _, value := range values { + if value == wanted { + return true + } + } + + return false +} + +func nodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + + return false +} + +func designatedNodeName(machine *v1alpha3.Machine) string { + if machine.Spec.Kubernetes != nil && machine.Spec.Kubernetes.NodeRef != nil && machine.Spec.Kubernetes.NodeRef.Name != "" { + return machine.Spec.Kubernetes.NodeRef.Name + } + + return machine.Name +} + +func metaSetStatusCondition(conditions *[]metav1.Condition, condition metav1.Condition) { + for i := range *conditions { + if (*conditions)[i].Type == condition.Type { + (*conditions)[i] = condition + + return + } + } + + *conditions = append(*conditions, condition) +} + +func newBootstrapPortForward( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + deploymentName string, + localPort int, + remotePort int, + start bootstrapPortForwardStarter, + retryInterval time.Duration, +) (*bootstrapPortForward, error) { + if retryInterval <= 0 { + retryInterval = defaultBootstrapPollInterval + } + + forwardCtx, cancel := context.WithCancel(ctx) + + podName, err := readyDeploymentPod(forwardCtx, kubeClient, namespace, deploymentName, "") + if err != nil { + cancel() + + return nil, err + } + + attempt, err := start(forwardCtx, podName, localPort, remotePort) + if err != nil { + cancel() + + return nil, fmt.Errorf("start port-forward to Pod %s: %w", podName, err) + } + + forward := &bootstrapPortForward{ + url: "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(localPort)), + cancel: cancel, + done: make(chan struct{}), + } + + go func() { + defer close(forward.done) + + current := attempt + + for { + select { + case <-forwardCtx.Done(): + current.Stop() + <-current.Done() + + return + case <-current.Done(): + } + + for { + select { + case <-forwardCtx.Done(): + current.Stop() + + return + case <-time.After(retryInterval): + } + + podName, err = readyDeploymentPod(forwardCtx, kubeClient, namespace, deploymentName, podName) + if err != nil { + continue + } + + current, err = start(forwardCtx, podName, localPort, remotePort) + if err == nil { + break + } + } + } + }() + + return forward, nil +} + +func readyDeploymentPod( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + deploymentName string, + excludedPod string, +) (string, error) { + deployment, err := kubeClient.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("get Deployment %s/%s for port-forward: %w", namespace, deploymentName, err) + } + + selector := metav1.FormatLabelSelector(deployment.Spec.Selector) + + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return "", fmt.Errorf("list Pods for Deployment %s/%s: %w", namespace, deploymentName, err) + } + + ready := make([]string, 0, len(pods.Items)) + excludedReady := false + + for i := range pods.Items { + pod := &pods.Items[i] + if pod.DeletionTimestamp == nil && podReady(pod) { + if pod.Name == excludedPod { + excludedReady = true + + continue + } + + ready = append(ready, pod.Name) + } + } + + if len(ready) == 0 && excludedReady { + ready = append(ready, excludedPod) + } + + if len(ready) == 0 { + return "", fmt.Errorf("deployment %s/%s has no Ready Pods", namespace, deploymentName) + } + + sort.Strings(ready) + + return ready[0], nil +} + +func podReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + + return false +} + +type spdyBootstrapPortForwardAttempt struct { + stop chan struct{} + done chan error + once sync.Once +} + +func (a *spdyBootstrapPortForwardAttempt) Done() <-chan error { + return a.done +} + +func (a *spdyBootstrapPortForwardAttempt) Stop() { + a.once.Do(func() { close(a.stop) }) +} + +func newSPDYBootstrapPortForwardStarter( + kubeClient kubernetes.Interface, + config *rest.Config, + namespace string, +) bootstrapPortForwardStarter { + return func( + ctx context.Context, + podName string, + localPort int, + remotePort int, + ) (bootstrapPortForwardAttempt, error) { + targetURL := kubeClient.CoreV1().RESTClient().Post(). + Resource("pods"). + Namespace(namespace). + Name(podName). + SubResource("portforward"). + URL() + + transport, upgrader, err := spdy.RoundTripperFor(config) + if err != nil { + return nil, fmt.Errorf("create SPDY transport: %w", err) + } + + stop := make(chan struct{}) + ready := make(chan struct{}) + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, targetURL) + + forwarder, err := portforward.NewOnAddresses( + dialer, + []string{"127.0.0.1"}, + []string{fmt.Sprintf("%d:%d", localPort, remotePort)}, + stop, + ready, + io.Discard, + io.Discard, + ) + if err != nil { + return nil, fmt.Errorf("create port-forward: %w", err) + } + + attempt := &spdyBootstrapPortForwardAttempt{stop: stop, done: make(chan error, 1)} + + go func() { attempt.done <- forwarder.ForwardPorts() }() + + select { + case <-ctx.Done(): + attempt.Stop() + + return nil, ctx.Err() + case err := <-attempt.done: + attempt.Stop() + + return nil, fmt.Errorf("port-forward exited before becoming ready: %w", err) + case <-ready: + return attempt, nil + case <-time.After(30 * time.Second): + attempt.Stop() + + return nil, fmt.Errorf("port-forward to Pod %s timed out", podName) + } + } +} + +func newBootstrapEdgeToken( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + tempRoot string, + refreshInterval time.Duration, +) (*bootstrapEdgeToken, error) { + directory, err := os.MkdirTemp(tempRoot, "unbounded-netboot-") + if err != nil { + return nil, fmt.Errorf("create edge token directory: %w", err) + } + + path := filepath.Join(directory, "edge-token") + if err := refreshBootstrapEdgeToken(ctx, kubeClient, namespace, path); err != nil { + if removeErr := os.RemoveAll(directory); removeErr != nil { + return nil, errors.Join(err, fmt.Errorf("remove edge token directory: %w", removeErr)) + } + + return nil, err + } + + if refreshInterval <= 0 { + refreshInterval = 20 * time.Minute + } + + tokenCtx, cancel := context.WithCancel(ctx) + credential := &bootstrapEdgeToken{ + path: path, + cancel: cancel, + done: make(chan struct{}), + } + + go func() { + defer close(credential.done) + + ticker := time.NewTicker(refreshInterval) + defer ticker.Stop() + + for { + select { + case <-tokenCtx.Done(): + return + case <-ticker.C: + if err := refreshBootstrapEdgeToken(tokenCtx, kubeClient, namespace, path); err != nil && tokenCtx.Err() == nil { + slog.WarnContext(tokenCtx, "refreshing Metalman edge token failed", "err", err) + } + } + } + }() + + return credential, nil +} + +func refreshBootstrapEdgeToken( + ctx context.Context, + kubeClient kubernetes.Interface, + namespace string, + path string, +) error { + expirationSeconds := int64(time.Hour / time.Second) + + response, err := kubeClient.CoreV1().ServiceAccounts(namespace).CreateToken( + ctx, + "metalman-edge", + &authenticationv1.TokenRequest{Spec: authenticationv1.TokenRequestSpec{ + Audiences: []string{"metalman-edge"}, + ExpirationSeconds: &expirationSeconds, + }}, + metav1.CreateOptions{}, + ) + if err != nil { + return fmt.Errorf("request metalman-edge token: %w", err) + } + + if response.Status.Token == "" { + return fmt.Errorf("request metalman-edge token: API returned an empty token") + } + + temporaryPath := path + ".new" + if err := os.WriteFile(temporaryPath, []byte(response.Status.Token), 0o600); err != nil { + return fmt.Errorf("write metalman-edge token: %w", err) + } + + if err := os.Rename(temporaryPath, path); err != nil { + if removeErr := os.Remove(temporaryPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return errors.Join( + fmt.Errorf("replace metalman-edge token: %w", err), + fmt.Errorf("remove temporary metalman-edge token: %w", removeErr), + ) + } + + return fmt.Errorf("replace metalman-edge token: %w", err) + } + + return nil +} diff --git a/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go new file mode 100644 index 000000000..6892ca67d --- /dev/null +++ b/cmd/kubectl-unbounded/app/site_bootstrap_netboot_test.go @@ -0,0 +1,895 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package app + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + clienttesting "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/internal/net/nodeagent" +) + +func TestSiteBootstrapNetbootCommandContract(t *testing.T) { + group := siteCommandGroup() + + cmd, _, err := group.Find([]string{"bootstrap-netboot"}) + require.NoError(t, err) + require.Equal(t, "bootstrap-netboot SITE", cmd.Use) + + for _, name := range []string{ + "machine", + "interface", + "address", + "endpoint-name", + "http-port", + "kubeconfig", + "namespace", + "metalman-binary", + "gateway-external-address", + "timeout", + "routed-cidr", + } { + require.NotNilf(t, cmd.Flags().Lookup(name), "missing --%s", name) + } + + for _, name := range []string{"machine", "interface", "address"} { + flag := cmd.Flags().Lookup(name) + require.Contains(t, flag.Annotations, "cobra_annotation_bash_completion_one_required_flag") + } +} + +func TestBootstrapNetbootPreparesAndRestoresClusterResources(t *testing.T) { + ctx := context.Background() + site := &v1alpha3.Site{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}, + Spec: v1alpha3.SiteSpec{ + Components: v1alpha3.SiteComponents{}, + }, + } + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-node", + Labels: map[string]string{v1alpha3.MachineSiteLabelKey: site.Name}, + }, + Spec: v1alpha3.MachineSpec{Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{ + EndpointRef: "permanent-edge", + }}}, + } + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).WithObjects(site, machine).Build() + h := &siteBootstrapNetbootHandler{ + site: site.Name, + machine: machine.Name, + address: "192.0.2.10", + httpPort: 8880, + endpointName: "bootstrap-first-node", + resources: resources, + } + + state, err := h.prepareClusterResources(ctx) + require.NoError(t, err) + require.Equal(t, "permanent-edge", state.originalEndpointRef) + + var gotSite v1alpha3.Site + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: site.Name}, &gotSite)) + require.NotNil(t, gotSite.Spec.Components.Metalman) + require.NotNil(t, gotSite.Spec.Components.Metalman.Enabled) + require.True(t, *gotSite.Spec.Components.Metalman.Enabled) + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, h.endpointName, gotMachine.Spec.Netboot().EndpointRef) + + var endpoint v1alpha3.NetbootEndpoint + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) + require.Equal(t, v1alpha3.NetbootEndpointTypeExternalL2, endpoint.Spec.Type) + require.Equal(t, site.Name, endpoint.Spec.SiteRef) + require.Equal(t, "http://192.0.2.10:8880", endpoint.Spec.ExternalURL) + require.Equal(t, v1alpha3.NetbootEndpointTrustTrustedLAN, endpoint.Spec.TLS.Trust) + + require.NoError(t, h.restoreClusterResources(ctx, state)) + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: site.Name}, &gotSite)) + require.True(t, *gotSite.Spec.Components.Metalman.Enabled) +} + +func TestBootstrapNetbootEndpointCollisionDoesNotMutateMachine(t *testing.T) { + ctx := context.Background() + enabled := true + site := &v1alpha3.Site{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}, + Spec: v1alpha3.SiteSpec{Components: v1alpha3.SiteComponents{ + Metalman: &v1alpha3.MetalmanComponentSpec{SiteComponentSpec: v1alpha3.SiteComponentSpec{Enabled: &enabled}}, + }}, + } + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "first-node", Labels: map[string]string{v1alpha3.MachineSiteLabelKey: site.Name}}, + Spec: v1alpha3.MachineSpec{Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{EndpointRef: "permanent-edge"}}}, + } + existing := &v1alpha3.NetbootEndpoint{ObjectMeta: metav1.ObjectMeta{Name: "bootstrap-first-node"}} + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).WithObjects(site, machine, existing).Build() + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, address: "192.0.2.10", httpPort: 8880, + endpointName: existing.Name, resources: resources, + } + + _, err := h.prepareClusterResources(ctx) + require.Error(t, err) + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) +} + +func TestBootstrapNetbootMetalmanReadinessRequiresCurrentControllerAndServers(t *testing.T) { + controller := readyDeployment("metalman-controller-rack-a", 1) + server := readyDeployment("metalman-server-rack-a", 2) + server.Status.ObservedGeneration-- + kubeClient := fake.NewSimpleClientset(controller, server) + h := &siteBootstrapNetbootHandler{site: "rack-a", namespace: "unbounded-system", kubeClient: kubeClient} + + ready, err := h.metalmanDeploymentsReady(context.Background()) + require.NoError(t, err) + require.False(t, ready) + + server.Status.ObservedGeneration = server.Generation + _, err = kubeClient.AppsV1().Deployments(h.namespace).UpdateStatus(context.Background(), server, metav1.UpdateOptions{}) + require.NoError(t, err) + + ready, err = h.metalmanDeploymentsReady(context.Background()) + require.NoError(t, err) + require.True(t, ready) +} + +func TestBootstrapNetbootWaitsUntilMetalmanRolloutCompletes(t *testing.T) { + controller := readyDeployment("metalman-controller-rack-a", 1) + server := readyDeployment("metalman-server-rack-a", 2) + server.Status.AvailableReplicas = 1 + kubeClient := fake.NewSimpleClientset(controller, server) + h := &siteBootstrapNetbootHandler{ + site: "rack-a", namespace: "unbounded-system", kubeClient: kubeClient, + pollInterval: time.Millisecond, + } + + go func() { + time.Sleep(5 * time.Millisecond) + + server.Status.AvailableReplicas = 2 + _, _ = kubeClient.AppsV1().Deployments(h.namespace).UpdateStatus(context.Background(), server, metav1.UpdateOptions{}) + }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + require.NoError(t, h.waitForMetalman(ctx)) +} + +func readyDeployment(name string, replicas int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "unbounded-system", Generation: 2}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, + }}, + }, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, + Replicas: replicas, + UpdatedReplicas: replicas, + AvailableReplicas: replicas, + }, + } +} + +func TestBootstrapNetbootEdgeArgumentsContainOnlyDataPlaneConfiguration(t *testing.T) { + h := &siteBootstrapNetbootHandler{ + endpointName: "bootstrap-first-node", + interfaceName: "eno1", + address: "192.0.2.10", + httpPort: 8880, + } + + args := h.edgeArguments("http://127.0.0.1:32123", "/run/token") + require.Equal(t, []string{ + "edge", + "--backend-url=http://127.0.0.1:32123", + "--endpoint=bootstrap-first-node", + "--bind-address=192.0.2.10", + "--http-port=8880", + "--dhcp-enabled", + "--dhcp-interface=eno1", + "--dhcp-server-ip=192.0.2.10", + "--edge-token-file=/run/token", + "--tftp-enabled", + "--tftp-bind-address=192.0.2.10", + }, args) + + for _, arg := range args { + require.NotContains(t, arg, "--site") + require.NotContains(t, arg, "--cache-dir") + require.NotContains(t, arg, "leader-elect") + } +} + +func TestBootstrapNetbootClaimsEndpointAndWaitsForDesignatedNode(t *testing.T) { + ctx := context.Background() + endpoint := &v1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "bootstrap-first-node", Generation: 3}, + Spec: v1alpha3.NetbootEndpointSpec{SiteRef: "rack-a"}, + } + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).WithStatusSubresource(endpoint).WithObjects(endpoint).Build() + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{v1alpha3.MachineSiteLabelKey: "rack-a"}, + }} + kubeClient := fake.NewSimpleClientset(node) + h := &siteBootstrapNetbootHandler{ + site: "rack-a", endpointName: endpoint.Name, resources: resources, kubeClient: kubeClient, + pollInterval: time.Millisecond, + } + + require.NoError(t, h.claimEndpoint(ctx, "bootstrap/123")) + + var claimed v1alpha3.NetbootEndpoint + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: endpoint.Name}, &claimed)) + require.Equal(t, endpoint.Generation, claimed.Status.ObservedGeneration) + require.Equal(t, "bootstrap/123", claimed.Status.Claim.HolderIdentity) + require.Equal(t, metav1.ConditionTrue, claimed.Status.Conditions[0].Status) + + waitCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + ready := make(chan error, 1) + + go func() { ready <- h.waitForNodeReady(waitCtx, node.Name) }() + + time.Sleep(5 * time.Millisecond) + + other := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "other", Labels: map[string]string{v1alpha3.MachineSiteLabelKey: "rack-a"}}, + Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}, + } + _, err := kubeClient.CoreV1().Nodes().Create(ctx, other, metav1.CreateOptions{}) + require.NoError(t, err) + + select { + case err := <-ready: + require.Failf(t, "wait returned for wrong Node", "error: %v", err) + case <-time.After(5 * time.Millisecond): + } + + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + _, err = kubeClient.CoreV1().Nodes().UpdateStatus(ctx, node, metav1.UpdateOptions{}) + require.NoError(t, err) + require.NoError(t, <-ready) +} + +func TestBootstrapNetbootDesignatedNodeNameUsesMachineNodeRef(t *testing.T) { + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "machine-name"}, + Spec: v1alpha3.MachineSpec{Kubernetes: &v1alpha3.KubernetesSpec{ + NodeRef: &v1alpha3.LocalObjectReference{Name: "node-name"}, + }}, + } + + require.Equal(t, "node-name", designatedNodeName(machine)) + machine.Spec.Kubernetes.NodeRef = nil + require.Equal(t, "machine-name", designatedNodeName(machine)) +} + +func TestBootstrapPortForwardReconnectsToReadyServerPod(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + deployment := readyDeployment("metalman-server-rack-a", 2) + deployment.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"app": "metalman-server"}} + unready := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "server-a", Namespace: deployment.Namespace, Labels: deployment.Spec.Selector.MatchLabels}, + } + ready := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "server-b", Namespace: deployment.Namespace, Labels: deployment.Spec.Selector.MatchLabels}, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, Status: corev1.ConditionTrue, + }}}, + } + replacement := ready.DeepCopy() + replacement.Name = "server-c" + kubeClient := fake.NewSimpleClientset(deployment, unready, ready, replacement) + starter := &fakeBootstrapPortForwardStarter{started: make(chan string, 4)} + + forward, err := newBootstrapPortForward( + ctx, + kubeClient, + deployment.Namespace, + deployment.Name, + 32123, + 8880, + starter.start, + time.Millisecond, + ) + require.NoError(t, err) + require.Equal(t, "http://127.0.0.1:32123", forward.URL()) + require.Equal(t, "server-b", <-starter.started) + + starter.fail("server-b", errors.New("connection lost")) + + select { + case podName := <-starter.started: + require.Equal(t, "server-c", podName) + case <-time.After(time.Second): + require.Fail(t, "port-forward did not reconnect") + } + + require.NoError(t, forward.Close()) + require.Equal(t, []int{32123, 32123}, starter.ports()) +} + +func TestBootstrapPortForwardClosesWhileWaitingToReconnect(t *testing.T) { + deployment := readyDeployment("metalman-server-rack-a", 1) + deployment.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"app": "metalman-server"}} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "server-a", Namespace: deployment.Namespace, Labels: deployment.Spec.Selector.MatchLabels}, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, Status: corev1.ConditionTrue, + }}}, + } + starter := &fakeBootstrapPortForwardStarter{started: make(chan string, 2)} + forward, err := newBootstrapPortForward( + context.Background(), + fake.NewSimpleClientset(deployment, pod), + deployment.Namespace, + deployment.Name, + 32123, + 8880, + starter.start, + time.Hour, + ) + require.NoError(t, err) + require.Equal(t, pod.Name, <-starter.started) + starter.fail(pod.Name, errors.New("connection lost")) + time.Sleep(time.Millisecond) + + closed := make(chan error, 1) + + go func() { closed <- forward.Close() }() + + select { + case err := <-closed: + require.NoError(t, err) + case <-time.After(time.Second): + require.Fail(t, "port-forward close blocked during reconnect") + } +} + +func TestBootstrapEdgeTokenUsesAudienceAndRotatesSecureFile(t *testing.T) { + ctx := context.Background() + kubeClient := fake.NewSimpleClientset() + + var mu sync.Mutex + + requests := 0 + + kubeClient.PrependReactor("create", "serviceaccounts", func(action clienttesting.Action) (bool, runtime.Object, error) { + create := action.(clienttesting.CreateAction) + request := create.GetObject().(*authenticationv1.TokenRequest) + require.Equal(t, []string{"metalman-edge"}, request.Spec.Audiences) + require.Equal(t, int64(3600), *request.Spec.ExpirationSeconds) + + mu.Lock() + requests++ + token := fmt.Sprintf("token-%d", requests) + mu.Unlock() + + return true, &authenticationv1.TokenRequest{ + Status: authenticationv1.TokenRequestStatus{Token: token}, + }, nil + }) + + credential, err := newBootstrapEdgeToken(ctx, kubeClient, "unbounded-system", t.TempDir(), time.Millisecond) + require.NoError(t, err) + + path := credential.Path() + require.Equal(t, "edge-token", filepath.Base(path)) + + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + require.Eventually(t, func() bool { + contents, readErr := os.ReadFile(path) + return readErr == nil && string(contents) != "token-1" + }, time.Second, time.Millisecond) + + require.NoError(t, credential.Close()) + + _, err = os.Stat(filepath.Dir(path)) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestBootstrapWaitsForEdgeListenerAndReportsEarlyExit(t *testing.T) { + h := &siteBootstrapNetbootHandler{address: "192.0.2.10", httpPort: 8880, pollInterval: time.Millisecond} + process := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + attempts := 0 + dial := func(_ context.Context, address string) error { + require.Equal(t, "192.0.2.10:8880", address) + + attempts++ + if attempts < 2 { + return errors.New("not listening") + } + + return nil + } + + require.NoError(t, h.waitForEdgeReady(context.Background(), process, dial)) + require.Equal(t, 2, attempts) + + failed := &fakeBootstrapEdgeProcess{done: make(chan struct{}), err: errors.New("bind: address already in use")} + close(failed.done) + err := h.waitForEdgeReady(context.Background(), failed, func(context.Context, string) error { + return errors.New("not listening") + }) + require.ErrorContains(t, err, "edge exited before becoming ready") + require.ErrorContains(t, err, "address already in use") +} + +func TestBootstrapPreflightResolvesMetalmanBeforeClusterMutation(t *testing.T) { + explicitPath := filepath.Join(t.TempDir(), "metalman") + require.NoError(t, os.WriteFile(explicitPath, []byte("binary"), 0o700)) + h := &siteBootstrapNetbootHandler{metalmanBinary: explicitPath} + lookedUp := "" + path, err := h.resolveMetalmanBinary(func(name string) (string, error) { + lookedUp = name + + return name, nil + }) + require.NoError(t, err) + require.Equal(t, explicitPath, path) + require.Empty(t, lookedUp) + + h.metalmanBinary = "" + path, err = h.resolveMetalmanBinary(func(name string) (string, error) { + lookedUp = name + + return "/usr/local/bin/metalman", nil + }) + require.NoError(t, err) + require.Equal(t, "metalman", lookedUp) + require.Equal(t, "/usr/local/bin/metalman", path) + + _, err = h.resolveMetalmanBinary(func(string) (string, error) { + return "", exec.ErrNotFound + }) + require.ErrorContains(t, err, "--metalman-binary") +} + +func TestBootstrapNetbootExecuteCleansUpAfterDesignatedNodeReady(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, true) + process := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + forward := testBootstrapPortForward(ctx) + token := testBootstrapEdgeToken(t, ctx) + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, pollInterval: time.Millisecond, + dependencies: bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + localPort: func() (int, error) { return 32123, nil }, + portForward: func(context.Context, int) (*bootstrapPortForward, error) { return forward, nil }, + edgeToken: func(context.Context) (*bootstrapEdgeToken, error) { return token, nil }, + startEdge: func(binary string, args []string) (bootstrapEdgeProcess, error) { + require.Equal(t, "/usr/bin/metalman", binary) + require.Contains(t, args, "--backend-url=http://127.0.0.1:32123") + + return process, nil + }, + dialEdge: func(context.Context, string) error { return nil }, + }, + } + + require.NoError(t, h.execute(ctx)) + require.True(t, process.stopped) + require.Equal(t, "bootstrap-first-node", h.endpointName) + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + + var endpoint v1alpha3.NetbootEndpoint + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) + + _, err := os.Stat(filepath.Dir(token.Path())) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestBootstrapNetbootExecuteReportsEarlyEdgeExitAndRollsBack(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, false) + process := &fakeBootstrapEdgeProcess{done: make(chan struct{}), err: errors.New("bind: address already in use")} + close(process.done) + + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, pollInterval: time.Millisecond, + dependencies: bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + localPort: func() (int, error) { return 32123, nil }, + portForward: func(ctx context.Context, _ int) (*bootstrapPortForward, error) { + return testBootstrapPortForward(ctx), nil + }, + edgeToken: func(ctx context.Context) (*bootstrapEdgeToken, error) { + return testBootstrapEdgeToken(t, ctx), nil + }, + startEdge: func(string, []string) (bootstrapEdgeProcess, error) { return process, nil }, + dialEdge: func(context.Context, string) error { return errors.New("not listening") }, + }, + } + + err := h.execute(ctx) + require.ErrorContains(t, err, "edge exited before becoming ready") + + var gotMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &gotMachine)) + require.Equal(t, "permanent-edge", gotMachine.Spec.Netboot().EndpointRef) + + var endpoint v1alpha3.NetbootEndpoint + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &endpoint)) +} + +func TestBootstrapNetbootExecuteRunsPreflightBeforeClusterMutation(t *testing.T) { + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, true) + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, + dependencies: bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "", errors.New("metalman missing") }, + preflightNetwork: func() error { require.Fail(t, "network preflight should not follow binary failure"); return nil }, + }, + } + + err := h.execute(context.Background()) + require.ErrorContains(t, err, "metalman missing") + + var gotSite v1alpha3.Site + require.NoError(t, resources.Get(context.Background(), client.ObjectKey{Name: site.Name}, &gotSite)) + require.Nil(t, gotSite.Spec.Components.Metalman) +} + +func TestBootstrapNetbootPreparesAndCleansExternalGatewayResources(t *testing.T) { + ctx := context.Background() + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).Build() + kubeClient := fake.NewSimpleClientset() + h := &siteBootstrapNetbootHandler{ + site: "rack-a", endpointName: "bootstrap-first-node", address: "192.0.2.10", + gatewayExternalAddress: "198.51.100.10", + routedCIDRs: []string{"10.40.0.0/16", "10.50.0.0/24"}, resources: resources, kubeClient: kubeClient, + } + + require.NoError(t, h.prepareGatewayResources(ctx)) + + var pool netv1alpha1.GatewayPool + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) + require.Equal(t, "External", pool.Spec.Type) + require.Equal(t, h.routedCIDRs, pool.Spec.RoutedCidrs) + require.Equal(t, map[string]string{"net.unbounded-cloud.io/bootstrap-gateway": h.endpointName}, pool.Spec.NodeSelector) + require.Equal(t, netv1alpha1.TunnelProtocolWireGuard, *pool.Spec.TunnelProtocol) + + var assignment netv1alpha1.SiteGatewayPoolAssignment + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &assignment)) + require.Equal(t, []string{h.site}, assignment.Spec.Sites) + require.Equal(t, []string{h.endpointName}, assignment.Spec.GatewayPools) + require.True(t, *assignment.Spec.Enabled) + require.Equal(t, netv1alpha1.TunnelProtocolWireGuard, *assignment.Spec.TunnelProtocol) + + node, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.NoError(t, err) + require.True(t, node.Spec.Unschedulable) + require.Equal(t, h.endpointName, node.Labels["net.unbounded-cloud.io/bootstrap-gateway"]) + require.Equal(t, "true", node.Labels["net.unbounded-cloud.io/external-node"]) + require.Contains(t, node.Spec.Taints, corev1.Taint{ + Key: "net.unbounded-cloud.io/gateway-node", Value: "true", Effect: corev1.TaintEffectNoSchedule, + }) + require.ElementsMatch(t, []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: h.address}, + {Type: corev1.NodeExternalIP, Address: h.gatewayExternalAddress}, + }, node.Status.Addresses) + + require.NoError(t, h.cleanupGatewayResources(ctx)) + _, err = kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.Error(t, err) + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &assignment)) + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) +} + +func TestBootstrapNetbootExecuteRunsOptInGatewayUntilNodeReady(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, true) + edgeProcess := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + gatewayProcess := &fakeBootstrapEdgeProcess{done: make(chan struct{})} + forward := testBootstrapPortForward(ctx) + token := testBootstrapEdgeToken(t, ctx) + runtimeDir := t.TempDir() + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + gatewayExternalAddress: "198.51.100.10", routedCIDRs: []string{"10.40.0.0/16"}, + httpPort: 8880, namespace: "unbounded-system", timeout: time.Second, + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, pollInterval: time.Millisecond, + } + h.dependencies = bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + preflightGateway: func() error { return nil }, + localPort: func() (int, error) { return 32123, nil }, + portForward: func(context.Context, int) (*bootstrapPortForward, error) { return forward, nil }, + edgeToken: func(context.Context) (*bootstrapEdgeToken, error) { return token, nil }, + startEdge: func(string, []string) (bootstrapEdgeProcess, error) { return edgeProcess, nil }, + dialEdge: func(context.Context, string) error { return nil }, + gatewayRuntimeDir: func() (string, error) { + return runtimeDir, nil + }, + startGateway: func(_ context.Context, options nodeagent.ExternalGatewayOptions) (bootstrapEdgeProcess, error) { + require.Equal(t, h.endpointName, options.NodeName) + require.Equal(t, runtimeDir, options.RuntimeDir) + require.Same(t, h.restConfig, options.RESTConfig) + markBootstrapGatewayReady(t, ctx, h, resources, kubeClient) + + return gatewayProcess, nil + }, + } + + require.NoError(t, h.execute(ctx)) + require.True(t, gatewayProcess.stopped) + require.True(t, edgeProcess.stopped) + + var pool netv1alpha1.GatewayPool + require.Error(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) + _, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.Error(t, err) +} + +func markBootstrapGatewayReady( + t *testing.T, + ctx context.Context, + h *siteBootstrapNetbootHandler, + resources client.Client, + kubeClient *fake.Clientset, +) { + t.Helper() + + node, err := kubeClient.CoreV1().Nodes().Get(ctx, h.endpointName, metav1.GetOptions{}) + require.NoError(t, err) + + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + + node.Annotations["net.unbounded-cloud.io/wg-pubkey"] = "public-key" + _, err = kubeClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + require.NoError(t, err) + + var pool netv1alpha1.GatewayPool + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: h.endpointName}, &pool)) + pool.Status.NodeCount = 1 + pool.Status.Nodes = []netv1alpha1.GatewayNodeInfo{{Name: h.endpointName, WireGuardPublicKey: "public-key"}} + pool.Status.ConnectedSites = []string{h.site} + require.NoError(t, resources.Update(ctx, &pool)) + + require.NoError(t, resources.Create(ctx, &netv1alpha1.GatewayPoolNode{ + ObjectMeta: metav1.ObjectMeta{Name: h.endpointName}, + Spec: netv1alpha1.GatewayNodeSpec{NodeName: h.endpointName, GatewayPool: h.endpointName}, + Status: netv1alpha1.GatewayNodeStatus{ + LastUpdated: metav1.Now(), + Routes: map[string]netv1alpha1.GatewayNodeRoute{ + "10.40.0.0/16": {Type: "RoutedCidr"}, + }, + }, + })) +} + +func TestBootstrapNetbootRejectsInvalidRoutedCIDRBeforeCreatingGateway(t *testing.T) { + resources := fakeclient.NewClientBuilder().WithScheme(buildScheme()).Build() + h := &siteBootstrapNetbootHandler{ + endpointName: "bootstrap-first-node", routedCIDRs: []string{"not-a-cidr"}, resources: resources, + kubeClient: fake.NewSimpleClientset(), + } + + err := h.prepareGatewayResources(context.Background()) + require.ErrorContains(t, err, "invalid routed CIDR") + + var pools netv1alpha1.GatewayPoolList + require.NoError(t, resources.List(context.Background(), &pools)) + require.Empty(t, pools.Items) +} + +func TestBootstrapNetbootExecuteRejectsInvalidRoutedCIDRBeforeClusterMutation(t *testing.T) { + ctx := context.Background() + site, machine, resources, kubeClient := bootstrapLifecycleFixture(t, false) + h := &siteBootstrapNetbootHandler{ + site: site.Name, machine: machine.Name, interfaceName: "eno1", address: "192.0.2.10", + routedCIDRs: []string{"not-a-cidr"}, httpPort: 8880, namespace: "unbounded-system", + resources: resources, kubeClient: kubeClient, restConfig: &rest.Config{}, + } + h.dependencies = bootstrapNetbootDependencies{ + resolveBinary: func() (string, error) { return "/usr/bin/metalman", nil }, + preflightNetwork: func() error { return nil }, + gatewayRuntimeDir: func() (string, error) { + t.Fatal("gateway runtime directory must not be created after invalid CIDR preflight") + + return "", nil + }, + } + + err := h.execute(ctx) + require.ErrorContains(t, err, "invalid routed CIDR") + + var actualMachine v1alpha3.Machine + require.NoError(t, resources.Get(ctx, client.ObjectKey{Name: machine.Name}, &actualMachine)) + require.Equal(t, "permanent-edge", actualMachine.Spec.Netboot().EndpointRef) + + var endpoints v1alpha3.NetbootEndpointList + require.NoError(t, resources.List(ctx, &endpoints)) + require.Empty(t, endpoints.Items) +} + +func bootstrapLifecycleFixture( + t *testing.T, + nodeReady bool, +) (*v1alpha3.Site, *v1alpha3.Machine, client.Client, *fake.Clientset) { + t.Helper() + + site := &v1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}} + machine := &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-node", Labels: map[string]string{v1alpha3.MachineSiteLabelKey: site.Name}, + }, + Spec: v1alpha3.MachineSpec{Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{ + EndpointRef: "permanent-edge", + }}}, + } + resources := fakeclient.NewClientBuilder(). + WithScheme(buildScheme()). + WithStatusSubresource(&v1alpha3.NetbootEndpoint{}). + WithObjects(site, machine). + Build() + controller := readyDeployment("metalman-controller-rack-a", 1) + server := readyDeployment("metalman-server-rack-a", 2) + + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}} + if nodeReady { + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + } + + return site, machine, resources, fake.NewSimpleClientset(controller, server, node) +} + +func testBootstrapPortForward(ctx context.Context) *bootstrapPortForward { + forwardCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + + go func() { + <-forwardCtx.Done() + close(done) + }() + + return &bootstrapPortForward{url: "http://127.0.0.1:32123", cancel: cancel, done: done} +} + +func testBootstrapEdgeToken(t *testing.T, ctx context.Context) *bootstrapEdgeToken { + t.Helper() + directory := t.TempDir() + path := filepath.Join(directory, "edge-token") + require.NoError(t, os.WriteFile(path, []byte("token"), 0o600)) + + tokenCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + + go func() { + <-tokenCtx.Done() + close(done) + }() + + return &bootstrapEdgeToken{path: path, cancel: cancel, done: done} +} + +type fakeBootstrapEdgeProcess struct { + done chan struct{} + err error + stopped bool +} + +func (f *fakeBootstrapEdgeProcess) Done() <-chan struct{} { + return f.done +} + +func (f *fakeBootstrapEdgeProcess) Err() error { return f.err } + +func (f *fakeBootstrapEdgeProcess) Stop(context.Context) error { + f.stopped = true + + return nil +} + +type fakeBootstrapPortForwardStarter struct { + mu sync.Mutex + attempts map[string]*fakeBootstrapPortForwardAttempt + local []int + started chan string +} + +func (f *fakeBootstrapPortForwardStarter) start(_ context.Context, podName string, localPort, remotePort int) (bootstrapPortForwardAttempt, error) { + if remotePort != 8880 { + return nil, fmt.Errorf("unexpected remote port %d", remotePort) + } + + attempt := &fakeBootstrapPortForwardAttempt{done: make(chan error, 1)} + + f.mu.Lock() + if f.attempts == nil { + f.attempts = map[string]*fakeBootstrapPortForwardAttempt{} + } + + f.attempts[podName] = attempt + f.local = append(f.local, localPort) + f.mu.Unlock() + + f.started <- podName + + return attempt, nil +} + +func (f *fakeBootstrapPortForwardStarter) fail(podName string, err error) { + f.mu.Lock() + attempt := f.attempts[podName] + f.mu.Unlock() + + attempt.done <- err +} + +func (f *fakeBootstrapPortForwardStarter) ports() []int { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]int(nil), f.local...) +} + +type fakeBootstrapPortForwardAttempt struct { + done chan error + once sync.Once +} + +func (f *fakeBootstrapPortForwardAttempt) Done() <-chan error { + return f.done +} + +func (f *fakeBootstrapPortForwardAttempt) Stop() { + f.once.Do(func() { close(f.done) }) +} diff --git a/cmd/metalman/README.md b/cmd/metalman/README.md index 696af5b43..c6b161d50 100644 --- a/cmd/metalman/README.md +++ b/cmd/metalman/README.md @@ -1,272 +1,110 @@ # metalman -Join bare metal nodes to a Kubernetes cluster using PXE and (optionally) Redfish. +Metalman provisions bare-metal Kubernetes nodes with DHCP/TFTP or UEFI HTTP +boot, optional Redfish control, and TPM 2.0 attestation. -A dedicated `metalman` binary is shipped as a container image for running the -PXE server inside a cluster. `unbounded-operator` deploys that image when a -Site enables the Metalman component. +## Runtime Roles -Run `metalman version` to print the binary version. - -## Usage - -```bash -# Create a Machine -kubectl apply -f - < [flags] -``` - -#### Deploying With Site Components - -Enable Metalman in the Site spec and let `unbounded-operator` create or update -the per-site Deployment running `metalman serve-pxe`: - -```yaml -apiVersion: unbounded-cloud.io/v1alpha3 -kind: Site -metadata: - name: rack-a -spec: - components: - metalman: - enabled: true - dhcpAutoInterface: true -``` - -The resulting Deployment (`metalman-controller-`) runs with host networking for DHCP. -It exposes ports 8880/tcp (HTTP), 8081/tcp (health), 67/udp (DHCP), and 69/udp (TFTP). - -The generated Deployment uses host networking, a `CriticalAddonsOnly` -toleration, DNS policy `ClusterFirstWithHostNet`, and a node selector -`unbounded-cloud.io/site=`. Resource requests are 100m CPU / 128Mi -memory with limits of 500m CPU / 256Mi memory. +Metalman is split into three roles: -#### DHCP Modes +| Command | Responsibility | Kubernetes access | Host networking | +|---------|----------------|-------------------|-----------------| +| `metalman controller` | MachineOperation, Redfish, immutable session creation, OCI preparation | Yes, leader-elected | No | +| `metalman server` | Session artifacts, callbacks, attestation, edge decision API | Yes | No | +| `metalman edge` | DHCP/TFTP on the provisioning LAN and HTTP proxying | No | Only when required for L2 | -The DHCP server operates in one of two modes depending on whether -`--dhcp-interface` is set: +Enabling `Site.spec.components.metalman.enabled` makes the operator deploy one +controller, two server replicas, a server Service, a PodDisruptionBudget, and a +shared capability-signing Secret. Endpoint resources determine edge placement. -- **Interface mode** (`--dhcp-interface=eth0`): Binds to a network interface - and listens for broadcast DHCP traffic. Use this when the controller is - directly attached to the provisioning network. +## Endpoint Types -- **Auto-interface mode** (`--dhcp-auto-interface`): Automatically detects - the network interface from the server bind address. Mutually exclusive - with `--dhcp-interface`. +- `ManagedL2` creates a host-network edge on selected nodes. It can own DHCP, + TFTP, and private HTTP on a directly attached provisioning network. +- `HTTP` creates replicated HTTP edge pods and a Service. Public endpoints must + use HTTPS, either from a TLS Secret or external TLS termination. +- `ExternalL2` describes an edge outside the cluster. No edge workload is + created. `kubectl unbounded site bootstrap-netboot` uses this mode. -- **Relay mode** (no `--dhcp-interface` or `--dhcp-auto-interface`): Listens - on a UDP port for unicast packets only. Use this when a DHCP relay agent - forwards requests from a remote subnet. +WireGuard is an L3 transport and does not carry DHCP broadcasts. Keep a DHCP +edge or relay on the client LAN. TFTP, HTTP, and Redfish can use routed paths. -Leader election is always enabled regardless of DHCP mode. Each site gets -its own leader-election lease (`metalman-`). - -#### Security Model - -A mostly-trusted network between the controller and the bare metal hosts is -assumed. Bootstrap tokens (Kubernetes ServiceAccount tokens) are issued to -nodes based on source IP - the controller looks up the Machine whose NIC -matches the requesting IP and issues a short-lived token for that node. - -Bootstrap tokens are delivered using the standard TPM 2.0 credential encryption workflow. -The client's endorsement key (EK) public key is stored in `status.tpm.ekPublicKey` when first seen. -So it's possible to prove that the bootstrap token was delivered only to trusted hosts. - -#### Sites - -The `--site` flag scopes a `metalman serve-pxe` instance to a subset of Machines. -The value is matched against the `unbounded-cloud.io/site` label on Machine -resources: - -```bash -# Manage only Machines labeled site=rack-a -metalman serve-pxe --site=rack-a --dhcp-interface=eth0 - -# Manage only unlabeled Machines (the default) -metalman serve-pxe --dhcp-interface=eth0 -``` - -Each site gets its own leader-election lease (`metalman-`), so -multiple sites can coexist on one cluster with independent HA. A `metalman serve-pxe` -instance with no `--site` manages Machines that do not have the site label -at all. - -### Images - -Metalman uses two OCI images when repaving a machine: - -Existing Machines may continue to use the deprecated top-level `spec.pxe` -shape; Metalman resolves both forms through the same compatibility accessor. - -- `spec.host.netboot.image` is the machine image. It contains `/disk/disk.img.gz`, a - gzip-compressed raw disk image written to the target disk. -- `spec.host.netboot.netbootImage` is the reusable PXE boot environment. It contains - bootloaders, kernel, initrd, templates, and metadata. Its cloud-init template - downloads and installs `unbounded-agent` from the configured release/source. - If omitted, Metalman uses the release-matched `--default-netboot-image`. - -Both images are built `FROM scratch` and use `/disk/` as the artifact root, -following the kubevirt containerDisk convention. Files with a `.tmpl` suffix in -the netboot image are Go templates rendered per-machine at serve time; other -files are served verbatim. A `metadata.yaml` file in the netboot image provides -image-level configuration such as `dhcpBootImageName` and `httpBootPath`. - -Images are built, tagged, and pushed using standard container tooling: - -```bash -docker build -t ghcr.io/azure/host-ubuntu2404:v1 -f images/host-ubuntu2404/Containerfile . -docker build -t ghcr.io/azure/netboot:v1 -f images/netboot/Containerfile . -docker push ghcr.io/azure/host-ubuntu2404:v1 -docker push ghcr.io/azure/netboot:v1 -``` - -### Machine - -A Machine is a cluster-scoped custom resource representing a single bare metal -host. At minimum it needs a NIC (MAC + static IP) and a machine image reference: +## Machine Example ```yaml apiVersion: unbounded-cloud.io/v1alpha3 kind: Machine metadata: name: node-01 + labels: + unbounded-cloud.io/site: rack-a spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - # Defaults to PXE. Set to HTTP to use Redfish UEFI HTTP boot. - bootProtocol: PXE - # Optional. Recommended when the host has multiple disks. - targetDisk: /dev/disk/by-id/example-os-disk - dhcpLeases: - - mac: "aa:bb:cc:dd:ee:01" - ipv4: "10.0.0.11" - subnetMask: "255.255.255.0" - gateway: "10.0.0.1" + host: + netboot: + image: ghcr.io/azure/host-ubuntu2404:v1 + netbootImage: ghcr.io/azure/netboot:v1 + endpointRef: rack-a-l2 + transport: TFTP + configurationSource: DHCP + networkMode: DHCP + dhcpLeases: + - mac: aa:bb:cc:dd:ee:01 + ipv4: 10.0.0.11 + subnetMask: 255.255.255.0 + gateway: 10.0.0.1 + dns: [10.0.0.1] + redfish: + url: https://10.0.10.11 + username: admin + passwordRef: + name: bmc-node-01-pass + namespace: default + key: password ``` -This is enough for the DHCP server to issue a lease and for TFTP/HTTP to serve -boot artifacts from the default netboot image. Set `spec.host.netboot.netbootImage` only -when a Machine needs a non-default PXE boot environment. The node must be -manually PXE-booted (or have PXE as its default boot option). - -When `spec.host.netboot.bootProtocol` is `HTTP`, `dhcpLeases` also supplies the static -UEFI HTTP boot client configuration. Metalman uses Redfish to disable DHCPv4 on -the host EthernetInterface matching the first lease MAC and writes that lease's -IPv4 address, subnet mask, gateway, and DNS servers before setting the UEFI HTTP -boot override. With Redfish access and an HTTP boot URL, repaving can run without -any DHCP server on the provisioning network. If a host has multiple NICs, put the -UEFI HTTP boot NIC first in `dhcpLeases`. +Supported boot combinations are: -The default netboot template passes the selected lease MAC to the installer -initrd, which uses it to select the provisioning NIC instead of assuming a fixed -interface name such as `eth0`. It also passes the lease DNS servers, configures -the installer network without DHCP, and writes matching MAC-based static netplan -configuration into the installed system before its first boot. It disables -cloud-init network rendering so fallback DHCP configuration cannot conflict with -that file. The default netboot image serves the same lease as NoCloud -`network-config`. If `spec.host.netboot.targetDisk` is -set, the installer writes the image to that disk; otherwise it falls back to -automatic disk selection. +| Transport | Configuration source | Network mode | +|-----------|----------------------|--------------| +| TFTP | DHCP | DHCP | +| HTTP | DHCP | DHCP | +| HTTP | Redfish | DHCP | +| HTTP | Redfish | Static | -Stock Ubuntu OVMF and sushy-emulator cannot emulate the complete DHCP-free -Redfish-to-firmware UEFI HTTP path. Repository CI therefore tests Metalman's -Redfish writes and then starts at a staged post-firmware EFI boundary, while -capturing the guest's traffic through installation and reboot to prove it emits -no DHCP packets. Applying Redfish settings and fetching the first EFI binary -remain firmware and BMC hardware-conformance responsibilities. +TFTP with Redfish configuration and static networking without Redfish are +rejected by API validation. -#### BMC +## Durable Provisioning -Adding a `redfish` block enables remote power management. Metalman uses it for -`MachineOperation` host actions without physical access: +Each HostReplace target gets an immutable `NetbootSession`. It snapshots the +Machine and operation identities, endpoint, boot settings, resolved OCI +digests, artifact allowlist, cluster settings, cloud-init data, and expiry. +Controller side effects wait until the session is ready. -```yaml -apiVersion: unbounded-cloud.io/v1alpha3 -kind: Machine -metadata: - name: node-01 -spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - dhcpLeases: - - mac: "aa:bb:cc:dd:ee:01" - ipv4: "10.0.0.11" - subnetMask: "255.255.255.0" - gateway: "10.0.0.1" - redfish: - url: https://bmc-node-01.example.com - username: admin - passwordRef: - name: bmc-node-01-pass - namespace: default - key: password -``` +Artifact and callback URLs contain an operation-scoped HMAC capability. HTTP, +TFTP, callbacks, and attestation resolve the exact session rather than trusting +source IP. Milestones are recorded on the exact session and operation target. +Artifacts are addressed by immutable digest and support HTTP ranges, allowing +an edge to resume a transfer through another server pod after disruption. -The BMC password is read from a Secret in the same namespace (key: `password`). -On first connection, the controller captures the BMC's TLS certificate -fingerprint and pins it in `status.redfish.certFingerprint` for subsequent -requests. +## Temporary First-Node Bootstrap -To repave a node with BMC access: +An administrator attached to the provisioning LAN can bootstrap the first Site +node without running Kubernetes controllers locally: ```bash -kubectl unbounded machine repave node-01 +kubectl unbounded site bootstrap-netboot rack-a \ + --machine node-01 \ + --interface eno1 \ + --address 10.0.0.2 ``` -This creates a `HostReplace` `MachineOperation`. Metalman handles the rest: it -configures the boot override for the selected `spec.host.netboot.bootProtocol`, executes -a Redfish force restart, waits for the installer `/pxe/disable` signal, tracks -first-boot cloud-init on the operation, and completes after the node is back up. +The command enables the in-cluster control and server planes, creates an +ephemeral ExternalL2 endpoint, port-forwards the local edge to a ready server +pod with reconnection, and stops when the designated Machine's Node is Ready. +The Machine endpoint reference is restored during cleanup. + +Use repeatable `--routed-cidr` flags when the administrator host must also act +as a temporary unbounded-net gateway for BMC or provisioning subnets. + +Run `metalman version` to print the binary version. diff --git a/cmd/metalman/main.go b/cmd/metalman/main.go index 3a75e90c0..d7b4cc3d3 100644 --- a/cmd/metalman/main.go +++ b/cmd/metalman/main.go @@ -22,7 +22,9 @@ func main() { Use: "metalman", Short: "Bare metal provisioning for Kubernetes", } - root.AddCommand(commands.ServePXECmd()) + root.AddCommand(commands.ControllerCmd()) + root.AddCommand(commands.ServerCmd()) + root.AddCommand(commands.EdgeCmd()) root.AddCommand(version.Command()) root.CompletionOptions.DisableDefaultCmd = true diff --git a/cmd/unbounded-net-node/main.go b/cmd/unbounded-net-node/main.go index 3e79a9f30..43858363a 100644 --- a/cmd/unbounded-net-node/main.go +++ b/cmd/unbounded-net-node/main.go @@ -5,798 +5,21 @@ package main import ( "context" - "flag" - "fmt" "os" "os/signal" "syscall" - "time" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/dynamic/dynamicinformer" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - - unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" - configpkg "github.com/Azure/unbounded/internal/net/config" - "github.com/Azure/unbounded/internal/net/metrics" - unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" - "github.com/Azure/unbounded/internal/version" -) - -// CNIConfig represents the CNI configuration file structure -type CNIConfig struct { - CNIVersion string `json:"cniVersion"` - Name string `json:"name"` - Plugins []PluginConf `json:"plugins"` -} - -// PluginConf represents a CNI plugin configuration -type PluginConf struct { - Type string `json:"type"` - Bridge string `json:"bridge,omitempty"` - IsGateway bool `json:"isGateway,omitempty"` - IsDefaultGW bool `json:"isDefaultGateway,omitempty"` - ForceAddress bool `json:"forceAddress,omitempty"` - IPMasq bool `json:"ipMasq,omitempty"` - HairpinMode bool `json:"hairpinMode,omitempty"` - MTU int `json:"mtu,omitempty"` - IPAM *IPAMConfig `json:"ipam,omitempty"` - Capabilities *Caps `json:"capabilities,omitempty"` -} - -// IPAMConfig represents the IPAM configuration -type IPAMConfig struct { - Type string `json:"type"` - Ranges [][]IPRange `json:"ranges,omitempty"` -} - -// IPRange represents an IP range for IPAM -type IPRange struct { - Subnet string `json:"subnet"` -} - -// Caps represents plugin capabilities -type Caps struct { - PortMappings bool `json:"portMappings,omitempty"` -} - -type config struct { - ConfigFile string - KubeconfigPath string - ApiserverURL string // Override Kubernetes API server URL (empty = use default) - NodeName string - CNIConfDir string - CNIConfFile string - BridgeName string - WireGuardDir string - WireGuardPort int - EnablePolicyRouting bool - MTU int - HealthPort int - InformerResyncPeriod time.Duration - StatusPushEnabled bool // Whether to push status to controller - StatusPushURL string // Controller URL for status push - StatusPushInterval time.Duration // Interval between status pushes to controller - StatusPushAPIServerInterval time.Duration // Interval between status pushes via aggregated API server - StatusPushDelta bool // Whether periodic HTTP pushes use deltas - StatusWSEnabled bool // Whether websocket push is enabled - StatusWSURL string // Controller websocket URL for status push - StatusWSAPIServerMode string // API server websocket mode: never, fallback, preferred - StatusWSAPIServerURL string // API server websocket URL for status push fallback - StatusWSAPIServerStartupDelay time.Duration // Delay before API server fallback is allowed after startup - StatusWSKeepaliveInterval time.Duration // Interval between websocket keepalive pings (0 disables keepalive) - StatusWSKeepaliveFailureCount int // Sequential websocket keepalive ping failures before reconnect - RemoveConfigurationOnShutdown bool // Remove all managed configuration (WireGuard, routes, masquerade, etc.) on shutdown - RemoveWireGuardOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown - CleanupNetlinkOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown - RemoveMasqueradeOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown - HealthCheckPort int // UDP port for health check probes (default 9997) - BaseMetric int // Base metric for programmed routes (default 1) - RouteTableID int // Route table ID for managed routes (default 252) - CriticalDeltaEvery time.Duration // Maximum critical delta publish frequency; changes are queued up to this interval for batching - StatsDeltaEvery time.Duration // Maximum statistics delta publish frequency; changes are queued up to this interval for batching - FullSyncEvery time.Duration // Forced full status sync interval; ensures controller has complete status periodically - GenevePort int // GENEVE UDP destination port (default 6081) - GeneveVNI int // GENEVE Virtual Network Identifier (default 1) - GeneveInterfaceName string // GENEVE shared tunnel interface name (default geneve0) - VXLANInterfaceName string // VXLAN shared tunnel interface name (default vxlan0) - IPIPInterfaceName string // IPIP shared tunnel interface name (default ipip0) - WireGuardInterfacePrefix string // Prefix for per-port WireGuard interfaces (default "wg"; per-peer name is ) - VXLANPort int // VXLAN UDP destination port (default 4789) - VXLANSrcPortLow int // VXLAN UDP source port range low (default 47891) - VXLANSrcPortHigh int // VXLAN UDP source port range high (default 47922) - PreferredPrivateEncap string // Preferred encap for private/internal networks (GENEVE, IPIP, VXLAN, WireGuard) - PreferredPublicEncap string // Preferred encap for public/external networks (WireGuard, IPIP, GENEVE, VXLAN) - HealthFlapMaxBackoff time.Duration // Maximum backoff duration for health check flap dampening - KubeProxyHealthInterval time.Duration // Interval between kube-proxy health checks (0 to disable) - NetlinkResyncPeriod time.Duration // Interval between full netlink cache resyncs - TunnelDataplaneMapSize int // Maximum LPM trie entries for eBPF tunnel map (default 16384) - TunnelIPFamily string // Tunnel underlay IP family: "IPv4" (default) or "IPv6" -} - -var siteGVR = schema.GroupVersionResource{ - Group: unboundedv1alpha3.GroupVersion.Group, - Version: unboundedv1alpha3.GroupVersion.Version, - Resource: "sites", -} - -var siteNodeSliceGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "sitenodeslices", -} - -var gatewayPoolGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "gatewaypools", -} - -var gatewayNodeGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "gatewaypoolnodes", -} - -var sitePeeringGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "sitepeerings", -} - -var siteGatewayPoolAssignmentGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "sitegatewaypoolassignments", -} - -var gatewayPoolPeeringGVR = schema.GroupVersionResource{ - Group: "net.unbounded-cloud.io", - Version: "v1alpha1", - Resource: "gatewaypoolpeerings", -} - -const ( - // WireGuard public key annotation on the node - WireGuardPubKeyAnnotation = "net.unbounded-cloud.io/wg-pubkey" - - // TunnelMTUAnnotation is the maximum tunnel MTU this node - // can support, based on its default-route interface MTU minus encapsulation - // overhead. The controller uses this to validate that the configured MTU - // does not exceed what any node in the cluster can handle. - TunnelMTUAnnotation = "net.unbounded-cloud.io/tunnel-mtu" - - // Gateway node taint key - prevents regular workloads from running on gateway nodes - // since they don't have regular pod CIDR routing - GatewayNodeTaintKey = "net.unbounded-cloud.io/gateway-node" - - // gatewayNodeHeartbeatInterval controls how frequently the node agent refreshes - // GatewayNode.status.lastUpdated. Route staleness is derived from this cadence. - gatewayNodeHeartbeatInterval = 10 * time.Second + "github.com/Azure/unbounded/internal/net/nodeagent" ) func main() { - // Initialize klog flags - klog.InitFlags(nil) - - // Add klog flags to pflag - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - - cfg := &config{ - ConfigFile: "/etc/unbounded-net/config.yaml", - CNIConfDir: "/etc/cni/net.d", - CNIConfFile: "10-unbounded.conflist", - BridgeName: "cbr0", - WireGuardDir: "/etc/wireguard", - WireGuardPort: 51820, - EnablePolicyRouting: false, - MTU: 1280, // default WireGuard MTU (IPv6 minimum) - HealthPort: 9998, - InformerResyncPeriod: 600 * time.Second, - StatusPushEnabled: true, // Enabled by default - StatusPushInterval: 10 * time.Second, // Default 10s push interval - StatusPushAPIServerInterval: 30 * time.Second, - StatusPushDelta: true, - StatusWSEnabled: true, - StatusWSAPIServerMode: statusWSAPIServerModeFallback, - StatusWSAPIServerStartupDelay: 60 * time.Second, - StatusWSKeepaliveInterval: 10 * time.Second, - StatusWSKeepaliveFailureCount: 2, - CriticalDeltaEvery: 1 * time.Second, - StatsDeltaEvery: 15 * time.Second, - FullSyncEvery: 2 * time.Minute, - GenevePort: 6081, - GeneveVNI: 1, - GeneveInterfaceName: "geneve0", - VXLANInterfaceName: "vxlan0", - IPIPInterfaceName: "ipip0", - WireGuardInterfacePrefix: "wg", - VXLANPort: 4789, - VXLANSrcPortLow: 47891, - VXLANSrcPortHigh: 47922, - PreferredPrivateEncap: "GENEVE", - PreferredPublicEncap: "WireGuard", - NetlinkResyncPeriod: 300 * time.Second, - TunnelDataplaneMapSize: 16384, - TunnelIPFamily: "IPv4", - } - - rootCmd := &cobra.Command{ - Use: "unbounded-net-node", - Short: "CNI configuration agent for unbounded-net", - Long: `unbounded-net-node runs on each node as a DaemonSet and configures CNI -networking by writing a CNI configuration file based on the node's podCIDRs. - -It watches the node object in Kubernetes and waits for podCIDRs to be assigned -by the unbounded-net-controller. Once assigned, it writes a CNI configuration -file that sets up pod networking using the bridge plugin with host-local IPAM. - -It also generates WireGuard keys for the node and stores them in /etc/wireguard, -then annotates the node with the public key.`, - Version: version.Version + " (commit: " + version.GitCommit + ")", - SilenceUsage: true, - PreRunE: func(cmd *cobra.Command, args []string) error { - return applyNodeRuntimeConfig(cmd, cfg) - }, - RunE: func(cmd *cobra.Command, args []string) error { - return run(cfg) - }, - } - - // Add flags - flags := rootCmd.Flags() - - // Change version flag from -v to -V to avoid conflict with klog's -v flag - rootCmd.Flags().BoolP("version", "V", false, "Print version information") - rootCmd.SetVersionTemplate(`{{printf "%s\n" .Version}}`) - - // General flags - flags.StringVar(&cfg.ConfigFile, "config-file", "/etc/unbounded-net/config.yaml", "Path to runtime YAML config file") - flags.StringVar(&cfg.KubeconfigPath, "kubeconfig", "", "Path to kubeconfig file (uses in-cluster config if not specified)") - flags.StringVar(&cfg.ApiserverURL, "apiserver-url", "", "Override Kubernetes API server URL (empty = use default from kubeconfig or in-cluster config)") - flags.StringVar(&cfg.NodeName, "node-name", os.Getenv("NODE_NAME"), "Name of this node (defaults to NODE_NAME env var)") - flags.IntVar(&cfg.HealthPort, "health-port", 9998, "Port for health check HTTP server (0 to disable)") - flags.DurationVar(&cfg.InformerResyncPeriod, "informer-resync-period", 600*time.Second, "Resync period for Kubernetes informers") - - // CNI configuration flags - flags.StringVar(&cfg.CNIConfDir, "cni-conf-dir", "/etc/cni/net.d", "Directory to write CNI configuration") - flags.StringVar(&cfg.CNIConfFile, "cni-conf-file", "10-unbounded.conflist", "Name of the CNI configuration file") - flags.StringVar(&cfg.BridgeName, "bridge-name", "cbr0", "Name of the bridge interface") - flags.IntVar(&cfg.MTU, "mtu", 1280, "MTU for WireGuard and bridge interfaces (default 1280, the IPv6 minimum)") - - // WireGuard configuration flags - flags.StringVar(&cfg.WireGuardDir, "wireguard-dir", "/etc/wireguard", "Directory to store WireGuard keys") - flags.IntVar(&cfg.WireGuardPort, "wireguard-port", 51820, "WireGuard listen port") - flags.BoolVar(&cfg.EnablePolicyRouting, "enable-policy-routing", false, "Enable policy-based routing on gateway interfaces (deprecated, UNBOUNDED-FORWARD chain rules replace PBR)") - - // Tunnel-interface configuration flags. All three shared tunnel device - // names must be non-empty, distinct, and must not collide with - // "unbounded0" (the agent's eBPF dummy device). Kernel interface names - // are limited to 15 bytes. - flags.IntVar(&cfg.GenevePort, "geneve-port", 6081, "GENEVE UDP destination port") - flags.IntVar(&cfg.GeneveVNI, "geneve-vni", 1, "GENEVE Virtual Network Identifier") - flags.StringVar(&cfg.GeneveInterfaceName, "geneve-interface", "geneve0", "Shared flow-based GENEVE interface name") - flags.StringVar(&cfg.VXLANInterfaceName, "vxlan-interface", "vxlan0", "Shared flow-based VXLAN interface name") - flags.StringVar(&cfg.IPIPInterfaceName, "ipip-interface", "ipip0", "Shared flow-based IPIP interface name") - flags.StringVar(&cfg.WireGuardInterfacePrefix, "wireguard-interface-prefix", "wg", "Prefix for per-port WireGuard interfaces; runtime name is ") - flags.IntVar(&cfg.VXLANPort, "vxlan-port", 4789, "VXLAN UDP destination port") - flags.IntVar(&cfg.VXLANSrcPortLow, "vxlan-src-port-low", 47891, "VXLAN UDP source port range low (narrow range reduces VM flow count in cloud platforms)") - flags.IntVar(&cfg.VXLANSrcPortHigh, "vxlan-src-port-high", 47922, "VXLAN UDP source port range high (narrow range reduces VM flow count in cloud platforms)") - flags.StringVar(&cfg.PreferredPrivateEncap, "preferred-private-encap", "GENEVE", "Preferred encapsulation for private networks (GENEVE, IPIP, VXLAN, WireGuard)") - flags.StringVar(&cfg.PreferredPublicEncap, "preferred-public-encap", "WireGuard", "Preferred encapsulation for public networks (WireGuard, IPIP, GENEVE, VXLAN)") - - // Status push flags - flags.BoolVar(&cfg.StatusPushEnabled, "status-push-enabled", true, "Enable pushing node status to controller") - flags.StringVar(&cfg.StatusPushURL, "status-push-url", "", "Controller URL for status push (default: use UNBOUNDED_NET_CONTROLLER_SERVICE_HOST/PORT)") - flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 60*time.Second, "Interval between status pushes to controller") - flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 60*time.Second, "Interval between status pushes via aggregated API server") - flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "Enable delta mode for periodic HTTP status push") - flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "Enable websocket status push to controller") - flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "Controller websocket URL for status push (default: ws://service/status/nodews)") - flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "API server websocket mode: never, fallback, preferred") - flags.StringVar(&cfg.StatusWSAPIServerURL, "status-ws-apiserver-url", "", "API server websocket URL for status push fallback (default: wss://$(KUBERNETES_SERVICE_HOST)/apis/status.net.unbounded-cloud.io/v1alpha1/status/nodews)") - flags.DurationVar(&cfg.StatusWSAPIServerStartupDelay, "status-ws-apiserver-startup-delay", 60*time.Second, "Delay before API server websocket/push fallback is allowed after startup (0 to disable delay)") - flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings (0 to disable)") - flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before reconnect") - flags.BoolVar(&cfg.RemoveConfigurationOnShutdown, "remove-configuration-on-shutdown", false, "Remove all managed configuration (WireGuard, routes, masquerade, tunnel interfaces) on shutdown") - flags.BoolVar(&cfg.RemoveWireGuardOnShutdown, "shutdown-remove-wireguard-configuration", false, "Remove WireGuard interfaces/configuration on shutdown (deprecated: use --remove-configuration-on-shutdown)") - flags.BoolVar(&cfg.CleanupNetlinkOnShutdown, "shutdown-cleanup-netlink", false, "Remove managed netlink routes/policy rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") - flags.BoolVar(&cfg.RemoveMasqueradeOnShutdown, "shutdown-remove-masquerade-rules", false, "Remove managed masquerade rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") - flags.IntVar(&cfg.HealthCheckPort, "healthcheck-port", 9997, "UDP port for health check probes") - flags.IntVar(&cfg.BaseMetric, "base-metric", 1, "Base metric for programmed routes") - flags.IntVar(&cfg.RouteTableID, "route-table-id", 252, "Route table ID for managed routes (default 252, set to 254 for main table)") - flags.DurationVar(&cfg.CriticalDeltaEvery, "status-critical-interval", 15*time.Second, "Maximum critical delta publish frequency; changed fields are queued up to this interval for batching") - flags.DurationVar(&cfg.StatsDeltaEvery, "status-stats-interval", 60*time.Second, "Maximum statistics delta publish frequency; changed fields are queued up to this interval for batching") - flags.DurationVar(&cfg.FullSyncEvery, "status-full-sync-interval", 2*time.Minute, "Forced full status sync interval; ensures controller has complete status periodically") - flags.DurationVar(&cfg.HealthFlapMaxBackoff, "health-flap-max-backoff", 120*time.Second, "Maximum backoff duration for health check flap dampening") - flags.DurationVar(&cfg.KubeProxyHealthInterval, "kube-proxy-health-interval", 30*time.Second, "Interval between kube-proxy health checks (0 to disable)") - flags.DurationVar(&cfg.NetlinkResyncPeriod, "netlink-resync-period", 300*time.Second, "Interval between full netlink cache resyncs") - flags.IntVar(&cfg.TunnelDataplaneMapSize, "tunnel-dataplane-map-size", 16384, "Maximum LPM trie entries for eBPF tunnel map") - flags.StringVar(&cfg.TunnelIPFamily, "tunnel-ip-family", "IPv4", "Tunnel underlay IP family: IPv4 (default) or IPv6") - - if err := rootCmd.Execute(); err != nil { - os.Exit(1) - } -} - -func applyNodeRuntimeConfig(cmd *cobra.Command, cfg *config) error { - runtimeCfg, err := configpkg.LoadRuntimeConfig(cfg.ConfigFile) - if err != nil { - return err - } - - flags := cmd.Flags() - nodeCfg := runtimeCfg.Node - - if !flags.Changed("informer-resync-period") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.InformerResyncPeriod, "node.informerResyncPeriod"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.InformerResyncPeriod = d - } - } - - if !flags.Changed("node-name") && nodeCfg.NodeName != "" { - cfg.NodeName = nodeCfg.NodeName - } - - if !flags.Changed("cni-conf-dir") && nodeCfg.CNIConfDir != "" { - cfg.CNIConfDir = nodeCfg.CNIConfDir - } - - if !flags.Changed("cni-conf-file") && nodeCfg.CNIConfFile != "" { - cfg.CNIConfFile = nodeCfg.CNIConfFile - } - - if !flags.Changed("bridge-name") && nodeCfg.BridgeName != "" { - cfg.BridgeName = nodeCfg.BridgeName - } - - if !flags.Changed("wireguard-dir") && nodeCfg.WireGuardDir != "" { - cfg.WireGuardDir = nodeCfg.WireGuardDir - } - - if !flags.Changed("wireguard-port") && nodeCfg.WireGuardPort != nil { - cfg.WireGuardPort = *nodeCfg.WireGuardPort - } - - if !flags.Changed("enable-policy-routing") && nodeCfg.EnablePolicyRouting != nil { //nolint:staticcheck // intentional use of deprecated field for backward compat - cfg.EnablePolicyRouting = *nodeCfg.EnablePolicyRouting //nolint:staticcheck // intentional use of deprecated field - } - - if !flags.Changed("mtu") && nodeCfg.MTU != nil { - cfg.MTU = *nodeCfg.MTU - } - - if !flags.Changed("health-port") && nodeCfg.HealthPort != nil { - cfg.HealthPort = *nodeCfg.HealthPort - } - - if !flags.Changed("status-push-enabled") && nodeCfg.StatusPushEnabled != nil { - cfg.StatusPushEnabled = *nodeCfg.StatusPushEnabled - } - - if !flags.Changed("status-push-url") && nodeCfg.StatusPushURL != "" { - cfg.StatusPushURL = nodeCfg.StatusPushURL - } - - if !flags.Changed("status-push-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushInterval, "node.statusPushInterval"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.StatusPushInterval = d - } - } - - if !flags.Changed("status-push-apiserver-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushAPIServerInterval, "node.statusPushApiserverInterval"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.StatusPushAPIServerInterval = d - } - } - - if !flags.Changed("status-push-delta") && nodeCfg.StatusPushDelta != nil { - cfg.StatusPushDelta = *nodeCfg.StatusPushDelta - } - - if !flags.Changed("status-ws-enabled") && nodeCfg.StatusWSEnabled != nil { - cfg.StatusWSEnabled = *nodeCfg.StatusWSEnabled - } - - if !flags.Changed("status-ws-url") && nodeCfg.StatusWSURL != "" { - cfg.StatusWSURL = nodeCfg.StatusWSURL - } - - if !flags.Changed("status-ws-apiserver-mode") && nodeCfg.StatusWSAPIServerMode != "" { - cfg.StatusWSAPIServerMode = nodeCfg.StatusWSAPIServerMode - } - - if !flags.Changed("status-ws-apiserver-url") && nodeCfg.StatusWSAPIServerURL != "" { - cfg.StatusWSAPIServerURL = nodeCfg.StatusWSAPIServerURL - } - - if !flags.Changed("status-ws-apiserver-startup-delay") && nodeCfg.StatusWSAPIServerStartupDelay != "" { - d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSAPIServerStartupDelay, "node.statusWebsocketApiserverStartupDelay") - if parseErr != nil { - return parseErr - } - - if d < 0 { - return fmt.Errorf("node.statusWebsocketApiserverStartupDelay must be >= 0") - } - - cfg.StatusWSAPIServerStartupDelay = d - } - - if !flags.Changed("status-ws-keepalive-interval") && nodeCfg.StatusWSKeepaliveInterval != "" { - d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSKeepaliveInterval, "node.statusWebsocketKeepaliveInterval") - if parseErr != nil { - return parseErr - } - - cfg.StatusWSKeepaliveInterval = d - } - - if !flags.Changed("status-ws-keepalive-failure-count") && nodeCfg.StatusWSKeepaliveFailCount != nil { - cfg.StatusWSKeepaliveFailureCount = *nodeCfg.StatusWSKeepaliveFailCount - } - // New consolidated shutdown cleanup flag. - if !flags.Changed("remove-configuration-on-shutdown") && nodeCfg.RemoveConfigurationOnShutdown != nil { - cfg.RemoveConfigurationOnShutdown = *nodeCfg.RemoveConfigurationOnShutdown - } - // Deprecated individual shutdown cleanup flags (kept for backward compatibility). - if !flags.Changed("shutdown-remove-wireguard-configuration") && nodeCfg.ShutdownRemoveWireGuardConfiguration != nil { - cfg.RemoveWireGuardOnShutdown = *nodeCfg.ShutdownRemoveWireGuardConfiguration - } - - if !flags.Changed("shutdown-cleanup-netlink") && nodeCfg.ShutdownRemoveIPRoutes != nil { - cfg.CleanupNetlinkOnShutdown = *nodeCfg.ShutdownRemoveIPRoutes - } - - if !flags.Changed("shutdown-remove-masquerade-rules") && nodeCfg.ShutdownRemoveMasqueradeRules != nil { - cfg.RemoveMasqueradeOnShutdown = *nodeCfg.ShutdownRemoveMasqueradeRules - } - // If any deprecated flag is true, activate the consolidated flag. - if cfg.RemoveWireGuardOnShutdown || cfg.CleanupNetlinkOnShutdown || cfg.RemoveMasqueradeOnShutdown { - cfg.RemoveConfigurationOnShutdown = true - } - - if _, err := parseStatusWSAPIServerMode(cfg.StatusWSAPIServerMode); err != nil { - return err - } - - if !flags.Changed("status-critical-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.CriticalDeltaEvery, "node.criticalDeltaEvery"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.CriticalDeltaEvery = d - } - } - - if !flags.Changed("status-stats-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatsDeltaEvery, "node.statsDeltaEvery"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.StatsDeltaEvery = d - } - } - - if !flags.Changed("status-full-sync-interval") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.FullSyncEvery, "node.fullSyncEvery"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.FullSyncEvery = d - } - } - - if cfg.StatusWSKeepaliveFailureCount < 1 { - return fmt.Errorf("node.statusWsKeepaliveFailureCount must be >= 1") - } - - // Apply preferred encapsulation from config file if not set via CLI. - if !flags.Changed("preferred-private-encap") && nodeCfg.PreferredPrivateNetworkEncapsulation != "" { - cfg.PreferredPrivateEncap = nodeCfg.PreferredPrivateNetworkEncapsulation - } - - if !flags.Changed("preferred-public-encap") && nodeCfg.PreferredPublicNetworkEncapsulation != "" { - cfg.PreferredPublicEncap = nodeCfg.PreferredPublicNetworkEncapsulation - } - - if !flags.Changed("health-flap-max-backoff") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.HealthFlapMaxBackoff, "node.healthFlapMaxBackoff"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.HealthFlapMaxBackoff = d - } - } - - if !flags.Changed("route-table-id") && nodeCfg.RouteTableID != nil { - cfg.RouteTableID = *nodeCfg.RouteTableID - } - - if !flags.Changed("kube-proxy-health-interval") && nodeCfg.KubeProxyHealthInterval != "" { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.KubeProxyHealthInterval, "node.kubeProxyHealthInterval"); parseErr != nil { - return parseErr - } else { - cfg.KubeProxyHealthInterval = d - } - } - - if !flags.Changed("netlink-resync-period") { - if d, parseErr := configpkg.ParseDurationField(nodeCfg.NetlinkResyncPeriod, "node.netlinkResyncPeriod"); parseErr != nil { - return parseErr - } else if d > 0 { - cfg.NetlinkResyncPeriod = d - } - } - - if !flags.Changed("tunnel-dataplane-map-size") && nodeCfg.TunnelDataplaneMapSize != nil { - cfg.TunnelDataplaneMapSize = *nodeCfg.TunnelDataplaneMapSize - } - - if !flags.Changed("tunnel-ip-family") && nodeCfg.TunnelIPFamily != "" { - cfg.TunnelIPFamily = nodeCfg.TunnelIPFamily - } - - if !flags.Changed("vxlan-src-port-low") && nodeCfg.VXLANSrcPortLow != nil { - cfg.VXLANSrcPortLow = *nodeCfg.VXLANSrcPortLow - } - - if !flags.Changed("vxlan-src-port-high") && nodeCfg.VXLANSrcPortHigh != nil { - cfg.VXLANSrcPortHigh = *nodeCfg.VXLANSrcPortHigh - } - - if !flags.Changed("geneve-interface") && nodeCfg.GeneveInterfaceName != "" { - cfg.GeneveInterfaceName = nodeCfg.GeneveInterfaceName - } - - if !flags.Changed("vxlan-interface") && nodeCfg.VXLANInterfaceName != "" { - cfg.VXLANInterfaceName = nodeCfg.VXLANInterfaceName - } - - if !flags.Changed("ipip-interface") && nodeCfg.IPIPInterfaceName != "" { - cfg.IPIPInterfaceName = nodeCfg.IPIPInterfaceName - } - - if !flags.Changed("wireguard-interface-prefix") && nodeCfg.WireGuardInterfacePrefix != "" { - cfg.WireGuardInterfacePrefix = nodeCfg.WireGuardInterfacePrefix - } - - // Validate and default tunnelIPFamily - switch cfg.TunnelIPFamily { - case "IPv4", "IPv6": - // valid - case "": - cfg.TunnelIPFamily = "IPv4" - default: - return fmt.Errorf("invalid tunnel-ip-family %q: must be 'IPv4' or 'IPv6'", cfg.TunnelIPFamily) - } - - if err := validateTunnelInterfaceNames(cfg.GeneveInterfaceName, cfg.VXLANInterfaceName, cfg.IPIPInterfaceName); err != nil { - return err - } - - if err := validateWireGuardInterfacePrefix(cfg.WireGuardInterfacePrefix); err != nil { - return err - } - - // Normalize MTU: treat 0 as 1280 (the IPv6 minimum, safe for all links). - if cfg.MTU == 0 { - cfg.MTU = 1280 - } - - // Apply common config. - if !flags.Changed("apiserver-url") && runtimeCfg.Common.ApiserverURL != "" { - cfg.ApiserverURL = runtimeCfg.Common.ApiserverURL - } - - return nil -} - -func run(cfg *config) error { - klog.Infof("unbounded-net-node version=%s commit=%s built=%s", version.Version, version.GitCommit, version.BuildTime) - - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() - // Handle shutdown signals - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + cmd := nodeagent.NewCommand() + cmd.SetContext(ctx) - go func() { - <-sigCh - klog.Info("Received shutdown signal") - cancel() - }() - - // Validate node name - if cfg.NodeName == "" { - klog.Fatal("Node name is required. Set NODE_NAME environment variable or use --node-name flag") - } - - klog.Infof("Running on node: %s", cfg.NodeName) - - if cfg.EnablePolicyRouting { - klog.Info("Policy-based routing on gateway interfaces is enabled") - } else { - klog.Info("Policy-based routing on gateway interfaces is disabled") - } - - // Build Kubernetes client - var ( - restConfig *rest.Config - err error - ) - - if cfg.KubeconfigPath != "" { - restConfig, err = clientcmd.BuildConfigFromFlags(cfg.ApiserverURL, cfg.KubeconfigPath) - } else { - restConfig, err = rest.InClusterConfig() - if err == nil && cfg.ApiserverURL != "" { - restConfig.Host = cfg.ApiserverURL - } - } - - if err != nil { - klog.Fatalf("Failed to build kubeconfig: %v", err) - } - - if cfg.ApiserverURL != "" { - klog.Infof("Using API server URL override: %s", cfg.ApiserverURL) - } - - // Wire client-go metrics into Prometheus before creating clients. - metrics.RegisterClientGoMetrics() - - clientset, err := kubernetes.NewForConfig(restConfig) - if err != nil { - klog.Fatalf("Failed to create Kubernetes client: %v", err) - } - - dynamicClient, err := dynamic.NewForConfig(restConfig) - if err != nil { - klog.Fatalf("Failed to create dynamic Kubernetes client: %v", err) - } - - // Generate WireGuard keys and annotate node - pubKey, err := ensureWireGuardKeys(cfg) - if err != nil { - klog.Fatalf("Failed to ensure WireGuard keys: %v", err) - } - - klog.Infof("WireGuard public key: %s", pubKey) - - if err := annotateNodeWithPubKey(ctx, clientset, cfg.NodeName, pubKey); err != nil { - klog.Fatalf("Failed to annotate node with WireGuard public key: %v", err) - } - - klog.Info("Node annotated with WireGuard public key") - - // Detect and annotate the node's maximum tunnel MTU so the controller - // can validate that the configured MTU is compatible across all nodes. - if detectedMTU := unboundednetnetlink.DetectDefaultRouteMTU(); detectedMTU > 0 { - wgMTU := detectedMTU - unboundednetnetlink.WireGuardMTUOverhead - if err := annotateNodeWithMTU(ctx, clientset, cfg.NodeName, wgMTU); err != nil { - klog.Warningf("Failed to annotate node with tunnel MTU: %v", err) - } else { - klog.Infof("Node annotated with tunnel MTU %d (detected default route MTU %d - %d overhead)", wgMTU, detectedMTU, unboundednetnetlink.WireGuardMTUOverhead) - } - } - - // Watch the config file for dynamic log level changes. - go configpkg.WatchConfigLogLevel(ctx, cfg.ConfigFile) - - // Warn if public network traffic will be sent unencrypted. - if cfg.PreferredPublicEncap != "" && cfg.PreferredPublicEncap != "WireGuard" { - klog.Warningf("WARNING: preferredPublicNetworkEncapsulation is set to %q -- traffic over public networks will be sent UNENCRYPTED", cfg.PreferredPublicEncap) - } - - // Create informers early - before any CRD-based lookups - // This allows us to use the informer cache for all CRD operations - informerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, cfg.InformerResyncPeriod) - sliceInformer := informerFactory.ForResource(siteNodeSliceGVR).Informer() - siteInformer := informerFactory.ForResource(siteGVR).Informer() - gatewayPoolInformer := informerFactory.ForResource(gatewayPoolGVR).Informer() - gatewayNodeInformer := informerFactory.ForResource(gatewayNodeGVR).Informer() - sitePeeringInformer := informerFactory.ForResource(sitePeeringGVR).Informer() - assignmentInformer := informerFactory.ForResource(siteGatewayPoolAssignmentGVR).Informer() - poolPeeringInformer := informerFactory.ForResource(gatewayPoolPeeringGVR).Informer() - - // Start the informers - informerFactory.Start(ctx.Done()) - - // Wait for caches to sync - klog.Info("Waiting for informer caches to sync") - - if !cache.WaitForCacheSync(ctx.Done(), sliceInformer.HasSynced, siteInformer.HasSynced, gatewayPoolInformer.HasSynced, gatewayNodeInformer.HasSynced, sitePeeringInformer.HasSynced, assignmentInformer.HasSynced, poolPeeringInformer.HasSynced) { - return fmt.Errorf("failed to sync informer caches") - } - - klog.Info("Informer caches synced") - - // Start the netlink cache (read-only network state snapshot). - netlinkCache := unboundednetnetlink.NewNetlinkCache(cfg.NetlinkResyncPeriod) - if err := netlinkCache.Start(ctx); err != nil { - klog.Fatalf("Failed to start netlink cache: %v", err) - } - - // Track if CNI is configured for health checks - cniConfigured := false - - // Create shared health state for health server - healthState := &nodeHealthState{ - cniConfigured: &cniConfigured, - informersSynced: []cache.InformerSynced{ - sliceInformer.HasSynced, - siteInformer.HasSynced, - gatewayPoolInformer.HasSynced, - gatewayNodeInformer.HasSynced, - sitePeeringInformer.HasSynced, - assignmentInformer.HasSynced, - poolPeeringInformer.HasSynced, - }, - } - - // Start health server if enabled (readiness should not wait on site membership) - if cfg.HealthPort > 0 { - go startHealthServer(cfg.HealthPort, healthState) - } - - // Check if this node is a gateway node by checking the informer cache - isGatewayNode := isGatewayNodeFromCRDs(gatewayPoolInformer, pubKey) - if isGatewayNode { - klog.Info("Node is a gateway node (found in GatewayPool status)") - } - - // Wait for this node to appear in a SiteNodeSlice or GatewayPool - // This ensures the site controller has processed this node before we continue - mySiteName, err := waitForSiteMembership(ctx, sliceInformer, gatewayPoolInformer, pubKey) - if err != nil { - if err == context.Canceled { - return nil - } - - return err - } - - // Check if this node's site has manageCniPlugin enabled using the informer cache - manageCniPlugin := getManageCniPluginFromCRDs(siteInformer, mySiteName) - - var nodePodCIDRs []string - if manageCniPlugin { - // Wait for podCIDRs and configure CNI - nodePodCIDRs, err = waitForPodCIDRsAndConfigure(ctx, clientset, cfg, &cniConfigured) - if err != nil { - if err == context.Canceled { - return nil - } - - return err - } - } else { - klog.Info("manageCniPlugin is false for this site - skipping CNI configuration") - // Still need to get the node's podCIDRs for WireGuard gateway IP calculation - node, err := clientset.CoreV1().Nodes().Get(ctx, cfg.NodeName, metav1.GetOptions{}) - if err != nil { - klog.Fatalf("Failed to get node: %v", err) - } - - nodePodCIDRs = node.Spec.PodCIDRs - // Mark CNI as "configured" since we're intentionally not managing it - cniConfigured = true + if err := cmd.Execute(); err != nil { + os.Exit(1) } - - // After CNI is configured (or skipped), watch Site CRD for WireGuard peers - // Pass the node's podCIDRs so routes can be configured with preferred source IPs - // Also pass the informers so they can be reused (already synced) - return watchSiteAndConfigureWireGuard(ctx, clientset, dynamicClient, cfg, pubKey, nodePodCIDRs, manageCniPlugin, healthState, netlinkCache, siteInformer, sliceInformer, gatewayPoolInformer, gatewayNodeInformer, sitePeeringInformer, assignmentInformer, poolPeeringInformer) } diff --git a/deploy/machina/06-metalman-rbac.yaml.tmpl b/deploy/machina/06-metalman-rbac.yaml.tmpl index 42dd9adf0..8b2df30da 100644 --- a/deploy/machina/06-metalman-rbac.yaml.tmpl +++ b/deploy/machina/06-metalman-rbac.yaml.tmpl @@ -5,6 +5,13 @@ metadata: name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} + --- apiVersion: v1 kind: ServiceAccount @@ -12,6 +19,13 @@ metadata: name: metalman-bootstrap namespace: {{ default "unbounded-system" .Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metalman-edge + namespace: {{ default "unbounded-system" .Namespace }} + --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -26,12 +40,38 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] - # Bootstrap token creation (scoped to metalman-bootstrap SA) + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + # Bootstrap token creation is scoped to the bootstrap ServiceAccount. - apiGroups: [""] resources: ["serviceaccounts/token"] resourceNames: ["metalman-bootstrap"] verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} +subjects: + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} +roleRef: + kind: Role + name: metalman-server + apiGroup: rbac.authorization.k8s.io + --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -42,6 +82,9 @@ subjects: - kind: ServiceAccount name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} roleRef: kind: Role name: metalman-controller @@ -70,6 +113,9 @@ subjects: - kind: ServiceAccount name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} roleRef: kind: Role name: metalman-controller @@ -98,6 +144,9 @@ subjects: - kind: ServiceAccount name: metalman-controller namespace: {{ default "unbounded-system" .Namespace }} + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} roleRef: kind: Role name: metalman-controller @@ -123,6 +172,16 @@ rules: - apiGroups: ["unbounded-cloud.io"] resources: ["machineoperations/status"] verbs: ["get", "update", "patch"] + # Netboot session controller + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootsessions"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootsessions/status"] + verbs: ["get", "update", "patch"] # Secrets (Redfish passwords) - apiGroups: [""] resources: ["secrets"] @@ -135,7 +194,6 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -150,6 +208,52 @@ roleRef: name: metalman-controller apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metalman-server +rules: + - apiGroups: ["unbounded-cloud.io"] + resources: ["machines"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machines/status"] + verbs: ["get", "update"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machineoperations"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machineoperations/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints", "netbootsessions"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootsessions/status"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["secrets", "configmaps", "nodes"] + verbs: ["get", "list", "watch"] + # Authenticate audience-bound edge ServiceAccount tokens. + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metalman-server +subjects: + - kind: ServiceAccount + name: metalman-server + namespace: {{ default "unbounded-system" .Namespace }} +roleRef: + kind: ClusterRole + name: metalman-server + apiGroup: rbac.authorization.k8s.io + --- # Allow bootstrap tokens issued for the metalman-bootstrap SA to create # certificate signing requests (kubelet TLS bootstrap). diff --git a/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml b/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml index a47f313a3..1d5c30b39 100644 --- a/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml @@ -331,6 +331,23 @@ spec: HostImage is the resolved provider-interpreted host image for HostReplace. An empty value instructs the provider to preserve the current image. type: string + netbootSessionRef: + description: |- + NetbootSessionRef identifies the immutable provisioning session assigned + to this HostReplace target. + properties: + name: + description: Name is the NetbootSession name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of + the session. + type: string + required: + - name + - uid + type: object providerRef: description: |- ProviderRef identifies the exact provider-owned resource referenced by diff --git a/deploy/machina/crd/unbounded-cloud.io_machines.yaml b/deploy/machina/crd/unbounded-cloud.io_machines.yaml index cad45fedf..70d3fc816 100644 --- a/deploy/machina/crd/unbounded-cloud.io_machines.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_machines.yaml @@ -331,16 +331,6 @@ spec: - amd64 - arm64 type: string - bootProtocol: - default: PXE - description: |- - BootProtocol selects how metalman should trigger network boot for - repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - HTTP boot with a URL derived from the netboot image metadata. - enum: - - PXE - - HTTP - type: string cloudInit: description: |- CloudInit contains optional cloud-init customization for PXE-booted @@ -368,6 +358,14 @@ spec: - namespace type: object type: object + configurationSource: + default: DHCP + description: ConfigurationSource selects how firmware receives + its boot target. + enum: + - DHCP + - Redfish + type: string dhcpLeases: description: |- DHCPLeases defines static IPv4 provisioning network settings. PXE boot @@ -401,6 +399,11 @@ spec: - subnetMask type: object type: array + endpointRef: + description: EndpointRef names the NetbootEndpoint that serves + this Machine. + minLength: 1 + type: string image: description: |- Image is an OCI image reference containing the machine disk image. @@ -433,6 +436,14 @@ spec: - name - namespace type: object + networkMode: + default: DHCP + description: NetworkMode selects how firmware configures the + provisioning network. + enum: + - DHCP + - Static + type: string pullSecretRef: description: |- PullSecretRef references a Docker registry credential Secret used to pull @@ -494,9 +505,27 @@ spec: Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... When omitted, the installer chooses a target disk automatically. type: string + transport: + default: TFTP + description: Transport selects the firmware boot artifact + transport. + enum: + - TFTP + - HTTP + type: string required: + - endpointRef - image type: object + x-kubernetes-validations: + - message: TFTP transport requires DHCP configuration + rule: '!(self.transport == ''TFTP'' && self.configurationSource + == ''Redfish'')' + - message: static networking requires Redfish configuration + rule: self.networkMode != 'Static' || self.configurationSource + == 'Redfish' + - message: Redfish configuration requires redfish connection details + rule: self.configurationSource != 'Redfish' || has(self.redfish) type: object x-kubernetes-validations: - message: at most one of netboot, azure, or external may be set @@ -585,16 +614,6 @@ spec: - amd64 - arm64 type: string - bootProtocol: - default: PXE - description: |- - BootProtocol selects how metalman should trigger network boot for - repaves. PXE uses DHCP/TFTP bootfile options. HTTP uses Redfish UEFI - HTTP boot with a URL derived from the netboot image metadata. - enum: - - PXE - - HTTP - type: string cloudInit: description: |- CloudInit contains optional cloud-init customization for PXE-booted @@ -622,6 +641,14 @@ spec: - namespace type: object type: object + configurationSource: + default: DHCP + description: ConfigurationSource selects how firmware receives + its boot target. + enum: + - DHCP + - Redfish + type: string dhcpLeases: description: |- DHCPLeases defines static IPv4 provisioning network settings. PXE boot @@ -655,6 +682,11 @@ spec: - subnetMask type: object type: array + endpointRef: + description: EndpointRef names the NetbootEndpoint that serves + this Machine. + minLength: 1 + type: string image: description: |- Image is an OCI image reference containing the machine disk image. @@ -687,6 +719,14 @@ spec: - name - namespace type: object + networkMode: + default: DHCP + description: NetworkMode selects how firmware configures the provisioning + network. + enum: + - DHCP + - Static + type: string pullSecretRef: description: |- PullSecretRef references a Docker registry credential Secret used to pull @@ -748,9 +788,26 @@ spec: Examples: /dev/nvme0n1, /dev/sda, /dev/disk/by-id/... When omitted, the installer chooses a target disk automatically. type: string + transport: + default: TFTP + description: Transport selects the firmware boot artifact transport. + enum: + - TFTP + - HTTP + type: string required: + - endpointRef - image type: object + x-kubernetes-validations: + - message: TFTP transport requires DHCP configuration + rule: '!(self.transport == ''TFTP'' && self.configurationSource + == ''Redfish'')' + - message: static networking requires Redfish configuration + rule: self.networkMode != 'Static' || self.configurationSource == + 'Redfish' + - message: Redfish configuration requires redfish connection details + rule: self.configurationSource != 'Redfish' || has(self.redfish) ssh: description: |- SSH contains the SSH connection and credential details for the diff --git a/deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml b/deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml new file mode 100644 index 000000000..70df872a7 --- /dev/null +++ b/deploy/machina/crd/unbounded-cloud.io_netbootendpoints.yaml @@ -0,0 +1,303 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: netbootendpoints.unbounded-cloud.io +spec: + group: unbounded-cloud.io + names: + kind: NetbootEndpoint + listKind: NetbootEndpointList + plural: netbootendpoints + shortNames: + - nbe + singular: netbootendpoint + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.siteRef + name: Site + type: string + - jsonPath: .spec.type + name: Type + type: string + - jsonPath: .spec.externalURL + name: URL + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + NetbootEndpoint declares a stable client-facing netboot endpoint and its edge + placement. The endpoint URL is snapshotted into each NetbootSession. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NetbootEndpointSpec defines a stable edge endpoint. + properties: + externalURL: + description: ExternalURL is the stable base URL advertised to firmware + and installers. + pattern: ^https?:// + type: string + http: + description: HTTP configures an operator-managed HTTP edge Service. + properties: + serviceType: + default: ClusterIP + description: ServiceType controls how the edge Service is exposed. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + managedL2: + description: ManagedL2 configures an operator-managed edge on a provisioning + LAN. + properties: + address: + description: Address is the stable address advertised by DHCP + and used by the edge. + minLength: 1 + type: string + interface: + description: Interface is the host interface used for DHCP and + TFTP. + minLength: 1 + type: string + nodeSelector: + description: NodeSelector selects nodes attached to the provisioning + network. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - address + - interface + - nodeSelector + type: object + siteRef: + description: SiteRef names the Site whose Machines may use this endpoint. + minLength: 1 + type: string + tls: + description: TLS defines the endpoint trust boundary and TLS termination + mode. + properties: + mode: + description: Mode identifies where TLS is terminated. + enum: + - Disabled + - Secret + - External + type: string + secretRef: + description: SecretRef references the serving certificate when + mode is Secret. + properties: + name: + description: Name of the secret. + minLength: 1 + type: string + namespace: + description: Namespace of the secret. + minLength: 1 + type: string + required: + - name + - namespace + type: object + trust: + description: |- + Trust identifies whether the endpoint is confined to a trusted LAN or is + reachable across an untrusted network. + enum: + - TrustedLAN + - Public + type: string + required: + - mode + - trust + type: object + x-kubernetes-validations: + - message: secretRef must be set only when TLS mode is Secret + rule: 'self.mode == ''Secret'' ? has(self.secretRef) : !has(self.secretRef)' + type: + description: Type identifies how the endpoint edge is operated. + enum: + - ManagedL2 + - ExternalL2 + - HTTP + type: string + required: + - externalURL + - siteRef + - tls + - type + type: object + x-kubernetes-validations: + - message: public endpoints require HTTPS + rule: self.tls.trust != 'Public' || (self.externalURL.startsWith('https://') + && self.tls.mode != 'Disabled') + - message: managedL2 configuration must be set only for ManagedL2 endpoints + rule: 'self.type == ''ManagedL2'' ? has(self.managedL2) : !has(self.managedL2)' + - message: http configuration must be set only for HTTP endpoints + rule: 'self.type == ''HTTP'' ? has(self.http) : !has(self.http)' + status: + description: NetbootEndpointStatus reports edge ownership and readiness. + properties: + claim: + description: Claim identifies the edge currently responsible for this + endpoint. + properties: + holderIdentity: + description: HolderIdentity uniquely identifies the claiming edge + process. + type: string + renewedAt: + description: RenewedAt records the latest successful claim heartbeat. + format: date-time + type: string + required: + - holderIdentity + - renewedAt + type: object + conditions: + description: Conditions report endpoint readiness and degradation. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the latest spec generation processed + by the edge. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml new file mode 100644 index 000000000..697e42dfb --- /dev/null +++ b/deploy/machina/crd/unbounded-cloud.io_netbootsessions.yaml @@ -0,0 +1,673 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: netbootsessions.unbounded-cloud.io +spec: + group: unbounded-cloud.io + names: + kind: NetbootSession + listKind: NetbootSessionList + plural: netbootsessions + shortNames: + - nbs + singular: netbootsession + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.machine.name + name: Machine + type: string + - jsonPath: .spec.endpoint.name + name: Endpoint + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.expiresAt + name: Expires + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + NetbootSession is an immutable provisioning contract for one + MachineOperation target. Runtime progress is recorded only in status. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NetbootSessionSpec snapshots all input needed to provision + one target. + properties: + artifacts: + description: Artifacts identifies immutable OCI sources and files + for this session. + properties: + files: + description: Files lists the named files an edge may request for + this session. + items: + description: NetbootSessionArtifact maps a public artifact name + to an immutable image path. + properties: + name: + description: Name is the stable route name used by edges + and rendered templates. + minLength: 1 + type: string + path: + description: Path is the absolute path within the unpacked + OCI image. + pattern: ^/ + type: string + size: + description: Size is the expected file size when known. + format: int64 + minimum: 0 + type: integer + source: + description: |- + Source selects the OCI image containing this file, or Session for + content snapshotted directly into the session. + enum: + - MachineImage + - NetbootImage + - Session + type: string + required: + - name + - path + - source + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + machineImage: + description: NetbootSessionImage identifies an OCI image resolved + to an immutable digest. + properties: + digest: + description: Digest is the immutable OCI manifest digest. + pattern: ^sha256:[a-f0-9]{64}$ + type: string + pullSecretRef: + description: |- + PullSecretRef references registry credentials without copying them into + the session. + properties: + name: + description: Name of the secret. + minLength: 1 + type: string + namespace: + description: Namespace of the secret. + minLength: 1 + type: string + required: + - name + - namespace + type: object + reference: + description: Reference is the source repository reference + used to resolve the image. + minLength: 1 + type: string + required: + - digest + - reference + type: object + netbootImage: + description: NetbootSessionImage identifies an OCI image resolved + to an immutable digest. + properties: + digest: + description: Digest is the immutable OCI manifest digest. + pattern: ^sha256:[a-f0-9]{64}$ + type: string + pullSecretRef: + description: |- + PullSecretRef references registry credentials without copying them into + the session. + properties: + name: + description: Name of the secret. + minLength: 1 + type: string + namespace: + description: Namespace of the secret. + minLength: 1 + type: string + required: + - name + - namespace + type: object + reference: + description: Reference is the source repository reference + used to resolve the image. + minLength: 1 + type: string + required: + - digest + - reference + type: object + required: + - files + - machineImage + - netbootImage + type: object + boot: + description: Boot snapshots firmware and provisioning network configuration. + properties: + architecture: + description: Architecture selects the boot artifact platform. + enum: + - amd64 + - arm64 + type: string + configurationSource: + description: NetbootConfigurationSource supplies the firmware + boot target. + enum: + - DHCP + - Redfish + type: string + dhcpLeases: + description: DHCPLeases snapshots the target's provisioning network + settings. + items: + description: DHCPLease defines static IPv4 provisioning network + settings. + properties: + dns: + description: DNS is a list of DNS server addresses. + items: + type: string + type: array + gateway: + description: Gateway is the default gateway. + type: string + ipv4: + description: IPv4 is the IP address to assign. + type: string + mac: + description: MAC is the MAC address of the network interface. + type: string + subnetMask: + description: SubnetMask is the subnet mask for the lease. + type: string + required: + - gateway + - ipv4 + - mac + - subnetMask + type: object + type: array + firmwareArtifact: + description: FirmwareArtifact is the named immutable artifact + advertised to firmware. + minLength: 1 + type: string + networkMode: + description: NetbootNetworkMode configures the firmware provisioning + interface. + enum: + - DHCP + - Static + type: string + targetDisk: + description: TargetDisk is the block device written by the installer. + type: string + transport: + description: NetbootTransport is the protocol firmware uses to + fetch boot artifacts. + enum: + - TFTP + - HTTP + type: string + required: + - architecture + - configurationSource + - firmwareArtifact + - networkMode + - transport + type: object + endpoint: + description: Endpoint snapshots the selected endpoint and advertised + URL. + properties: + externalURL: + description: ExternalURL is the immutable base URL advertised + for this session. + pattern: ^https?:// + type: string + name: + description: Name is the NetbootEndpoint name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of the endpoint. + type: string + required: + - externalURL + - name + - uid + type: object + expiresAt: + description: ExpiresAt is the last time new requests may use this + session. + format: date-time + type: string + machine: + description: Machine identifies the exact Machine revision being provisioned. + properties: + generation: + description: Generation records the observed desired-state generation. + format: int64 + minimum: 1 + type: integer + name: + description: Name is the cluster-scoped object name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of the object. + type: string + required: + - generation + - name + - uid + type: object + operation: + description: Operation identifies the owning MachineOperation. + properties: + generation: + description: Generation records the observed desired-state generation. + format: int64 + minimum: 1 + type: integer + name: + description: Name is the cluster-scoped object name. + minLength: 1 + type: string + uid: + description: UID detects deletion and recreation of the object. + type: string + required: + - generation + - name + - uid + type: object + provisioning: + description: |- + Provisioning snapshots inputs used to render installer and first-boot + configuration. + properties: + agent: + description: Agent contains the immutable agent installation configuration. + properties: + baseURL: + description: |- + BaseURL overrides the base URL used to construct the + unbounded-agent download URL. Defaults to the upstream GitHub + releases URL. The layout under BaseURL must match the GitHub + releases layout (/latest/download/ and + /download//). + type: string + downloads: + description: |- + Downloads overrides the download sources for the binaries the + agent installs into the nspawn rootfs (kubelet, containerd, runc, + CNI plugins, crictl). When unset the agent downloads each + artifact from its upstream default host. + properties: + cni: + description: |- + CNI overrides the download source for CNI plugins + (upstream default: https://github.com/containernetworking/plugins). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + containerd: + description: |- + Containerd overrides the download source for containerd + (upstream default: https://github.com/containerd/containerd). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + crictl: + description: |- + Crictl overrides the download source for crictl + (upstream default: https://github.com/kubernetes-sigs/cri-tools). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + kubernetes: + description: |- + Kubernetes overrides the download source for kubelet/kubectl/kube-proxy + (upstream default: https://dl.k8s.io). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + runc: + description: |- + Runc overrides the download source for runc + (upstream default: https://github.com/opencontainers/runc). + properties: + baseURL: + description: |- + BaseURL replaces the upstream host + path prefix used to + construct the download URL. Version and arch substitution are + preserved so mirrors need to publish assets under the same + layout as the upstream project. + type: string + url: + description: |- + URL is a fully qualified download URL template. Version/arch + substitution via fmt directives is preserved. When set it + overrides BaseURL entirely. + type: string + version: + description: |- + Version overrides the version of the artifact that would + otherwise be derived from the cluster Kubernetes version or the + agent's compiled-in defaults. + type: string + type: object + type: object + image: + description: |- + Image is the OCI image reference used for provisioning the + nspawn machine (e.g. "ghcr.io/org/repo:tag"). When empty the + agent falls back to its built-in default image. + type: string + url: + description: |- + URL is a fully qualified download URL for the unbounded-agent + tarball. When set it overrides Version and BaseURL entirely. + type: string + version: + description: |- + Version pins the unbounded-agent release tag that is downloaded + onto the host (e.g. "v0.0.10"). When empty the install script + tracks the latest published release. + type: string + type: object + cluster: + description: NetbootSessionCluster snapshots cluster connection + inputs used by the agent. + properties: + apiServerURL: + type: string + caCertBase64: + type: string + dns: + type: string + kubernetesVersion: + description: |- + KubernetesVersion is the cluster version used when the Machine does not + specify one. + type: string + required: + - apiServerURL + - caCertBase64 + - dns + - kubernetesVersion + type: object + kubernetes: + description: Kubernetes contains the target's immutable kubelet + configuration. + properties: + bootstrapTokenRef: + description: |- + BootstrapTokenRef references a bootstrap token Secret in + kube-system. The secret must be of type + bootstrap.kubernetes.io/token with the well-known keys + "token-id" and "token-secret". + properties: + name: + description: Name of the referenced resource. + minLength: 1 + type: string + required: + - name + type: object + nodeLabels: + additionalProperties: + type: string + description: NodeLabels are labels passed to kubelet's --node-labels + flag. + type: object + nodeRef: + description: NodeRef references the Node that corresponds + to this Machine. + properties: + name: + description: Name of the referenced resource. + minLength: 1 + type: string + required: + - name + type: object + registerWithTaints: + description: |- + RegisterWithTaints are taints passed to kubelet's --register-with-taints flag. + Each entry uses the standard Kubernetes taint format: key=value:Effect. + items: + type: string + type: array + version: + description: |- + Version is the Kubernetes version to install (e.g., "v1.34.0"). + When omitted the controller falls back to the cluster's + Kubernetes version. + type: string + type: object + providerLabels: + additionalProperties: + type: string + description: ProviderLabels are merged into the rendered kubelet + labels. + type: object + userData: + description: UserData is the resolved cloud-init user-data content. + type: string + required: + - cluster + - userData + type: object + required: + - artifacts + - boot + - endpoint + - expiresAt + - machine + - operation + - provisioning + type: object + x-kubernetes-validations: + - message: netboot session spec is immutable + rule: self == oldSelf + status: + description: NetbootSessionStatus reports preparation and target-scoped + milestones. + properties: + capabilityID: + description: |- + CapabilityID identifies the signing key and capability generation without + persisting a bearer capability. + type: string + conditions: + description: |- + Conditions contain preparation, endpoint readiness, and target-scoped + provisioning milestones. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + phase: + description: Phase is the current durable lifecycle phase. + enum: + - Pending + - Preparing + - Ready + - Active + - Complete + - Failed + - Expired + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/machina/crd/unbounded-cloud.io_sites.yaml b/deploy/machina/crd/unbounded-cloud.io_sites.yaml index d584feddb..a78e80ed8 100644 --- a/deploy/machina/crd/unbounded-cloud.io_sites.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_sites.yaml @@ -108,20 +108,9 @@ spec: description: Metalman configures the Metalman PXE controller for this site. properties: - dhcpAutoInterface: - description: DHCPAutoInterface lets Metalman choose the DHCP - interface automatically. - type: boolean enabled: description: Enabled controls whether the component is reconciled. type: boolean - replicas: - description: |- - Replicas is the desired number of Metalman replicas. Defaults to 1 when - omitted. - format: int32 - minimum: 0 - type: integer type: object storage: description: Storage configures the unbounded-storage supervisor diff --git a/deploy/net/node/03-daemonset.yaml.tmpl b/deploy/net/node/03-daemonset.yaml.tmpl index 71adf66b7..0bb5e49af 100644 --- a/deploy/net/node/03-daemonset.yaml.tmpl +++ b/deploy/net/node/03-daemonset.yaml.tmpl @@ -35,6 +35,13 @@ spec: # Tolerate all taints - CNI must run on every node tolerations: - operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: net.unbounded-cloud.io/external-node + operator: DoesNotExist # Prioritize scheduling - CNI is critical for pod networking priorityClassName: system-node-critical initContainers: diff --git a/deploy/net/node/daemonset_test.go b/deploy/net/node/daemonset_test.go new file mode 100644 index 000000000..78f3e9818 --- /dev/null +++ b/deploy/net/node/daemonset_test.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package node + +import ( + "os" + "strings" + "testing" +) + +func TestDaemonSetExcludesExternalSyntheticNodes(t *testing.T) { + contents, err := os.ReadFile("03-daemonset.yaml.tmpl") + if err != nil { + t.Fatalf("read DaemonSet template: %v", err) + } + + template := string(contents) + for _, expected := range []string{ + "nodeAffinity:", + "requiredDuringSchedulingIgnoredDuringExecution:", + "net.unbounded-cloud.io/external-node", + "operator: DoesNotExist", + } { + if !strings.Contains(template, expected) { + t.Fatalf("DaemonSet template missing %q", expected) + } + } +} diff --git a/deploy/unbounded-operator/02-rbac.yaml.tmpl b/deploy/unbounded-operator/02-rbac.yaml.tmpl index 66a3f9716..0157e7f1d 100644 --- a/deploy/unbounded-operator/02-rbac.yaml.tmpl +++ b/deploy/unbounded-operator/02-rbac.yaml.tmpl @@ -16,6 +16,12 @@ rules: - apiGroups: ["unbounded-cloud.io"] resources: ["sites/status"] verbs: ["get", "patch", "update"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["netbootendpoints/status"] + verbs: ["get", "patch", "update"] # Legacy net-group Sites are read during migration and translated into the # machina-group Sites above; their finalizers are cleared (update/patch) so the # CRD can be deleted once drained. @@ -45,6 +51,9 @@ rules: - apiGroups: ["apps"] resources: ["deployments", "daemonsets"] verbs: ["get", "list", "watch", "create", "patch", "update", "delete", "deletecollection"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete", "deletecollection"] - apiGroups: ["apps"] resources: ["statefulsets", "replicasets"] verbs: ["get", "list", "watch"] diff --git a/designs/metalman-netboot-architecture.md b/designs/metalman-netboot-architecture.md new file mode 100644 index 000000000..47440730c --- /dev/null +++ b/designs/metalman-netboot-architecture.md @@ -0,0 +1,328 @@ +# Metalman Netboot Architecture + +## Status + +Implemented for the `v1alpha3` API. This is a breaking alpha redesign; the new +boot axes and endpoint reference are the supported contract. + +## Goals + +- Keep the normal Metalman control and serving planes off host networking. +- Keep active provisioning valid across controller failover, server pod loss, + and rolling updates. +- Support traditional DHCP/TFTP PXE on a local L2 and UEFI HTTP boot through + routed, NAT, NodePort, load-balancer, and internet paths. +- Configure HTTP boot through either DHCP or Redfish, with DHCP or static + firmware networking. +- Let an administrator bootstrap the first Site node without running Kubernetes + controllers on the administrator machine. +- Use the existing unbounded-net dataplane for optional routed access from an + external bootstrap host without pretending WireGuard provides L2. +- Remove source-address identity and correlate every artifact, callback, and + attestation request with one operation target. + +## Non-goals + +- Extending DHCP broadcasts through WireGuard. +- Hiding loss of a ManagedL2 address with ordinary pod replication. The L2 + address must be stable through node placement, a VIP, or an external edge. +- Preserving the old alpha `bootProtocol`, Site replica, or DHCP auto-interface + API fields. +- Providing a multi-machine temporary bootstrap handoff. The command owns one + designated first Machine and exits when that Node is Ready. +- Requiring shared writable storage for OCI caches. + +## Runtime Roles + +```mermaid +flowchart LR + K["Kubernetes API"] + C["metalman controller
leader elected"] + SVC["metalman-server Service"] + S1["server replica A"] + S2["server replica B"] + E["edge
ManagedL2, HTTP, or ExternalL2"] + F["Firmware / installer / agent"] + B["Redfish BMC"] + R["OCI registry"] + + C <--> K + S1 <--> K + S2 <--> K + C --> R + S1 --> R + S2 --> R + C --> B + SVC --> S1 + SVC --> S2 + E --> SVC + F <--> E +``` + +### Controller + +The singleton per-Site controller owns leader-elected reconciliation, Redfish +certificate pinning and power/boot operations, OCI resolution, and immutable +session creation. It has no DHCP, TFTP, artifact, callback, or attestation +listener and does not use host networking. + +### Server + +Two server replicas sit behind a ClusterIP Service. They materialize disposable +digest-addressed OCI caches and serve authenticated session artifacts, +callbacks, DHCP decisions, and TPM attestation. A PodDisruptionBudget, +zero-unavailable rollout, health probes, and topology spread preserve service +availability. A lost local cache is reconstructed from the immutable digest. + +### Edge + +The edge has no controller RBAC. It proxies HTTP and, when attached to a +provisioning LAN, owns DHCP and TFTP. Its authenticated internal DHCP decision +calls use a projected `metalman-edge` ServiceAccount token. Firmware-facing +paths carry the session capability and do not rely on client IP. + +## Endpoint Model + +`NetbootEndpoint` is cluster-scoped and declares a stable external URL, Site, +trust boundary, TLS policy, placement, and readiness. + +| Type | Ownership | Networking | Typical use | +|------|-----------|------------|-------------| +| `ManagedL2` | Operator Deployment | Host network | Rack-local DHCP, TFTP, and private HTTP | +| `HTTP` | Operator Deployment and Service | Pod network | UEFI HTTP through ClusterIP, NodePort, load balancer, or ingress | +| `ExternalL2` | External process | Host network outside cluster | First-node bootstrap or dedicated appliance | + +Only `ManagedL2` uses host networking in the normal operator deployment. Public +endpoints require an `https://` URL and TLS mode `Secret` or `External`. Secret +mode mirrors the selected certificate into the operator namespace and rolls +the edge when its checksum changes. + +An endpoint is usable only when status observes the current generation and has +`Ready=True`. External edges claim and renew status themselves. + +## Machine Boot Axes + +`Machine.spec.host.netboot` separates concepts previously conflated as one boot +protocol: + +- `transport`: `TFTP` or `HTTP`. +- `configurationSource`: `DHCP` or `Redfish` supplies the firmware boot target. +- `networkMode`: `DHCP` or `Static` configures firmware networking. +- `endpointRef`: required stable NetbootEndpoint identity. + +Supported combinations are: + +| Transport | Configuration source | Network mode | Behavior | +|-----------|----------------------|--------------|----------| +| TFTP | DHCP | DHCP | DHCP supplies lease and tokenized TFTP path. | +| HTTP | DHCP | DHCP | DHCP supplies lease and signed HTTP boot URL. | +| HTTP | Redfish | DHCP | Redfish writes the signed URL; firmware obtains networking through DHCP. | +| HTTP | Redfish | Static | Redfish writes both the signed URL and static NIC configuration. | + +TFTP+Redfish, static networking without Redfish, and Redfish configuration +without BMC details are rejected by validation. + +## Immutable Sessions + +One `NetbootSession` is created per HostReplace operation target. Its spec is +immutable and records: + +- Exact Machine, MachineOperation, and endpoint names, UIDs, and generations. +- Endpoint external URL, boot axes, architecture, leases, and target disk. +- Machine and netboot OCI references resolved to SHA-256 digests. +- The complete artifact allowlist and firmware artifact. +- Cluster API, CA, DNS, Kubernetes version, agent settings, provider labels, + and resolved cloud-init user-data. +- Expiration. + +The operation target stores the session name and UID before any Redfish side +effect. Reconciliation returns after persisting this reference, then proceeds +only when the same session is Ready or Active. A replaced endpoint, Machine, or +session UID fails closed. + +```mermaid +stateDiagram-v2 + [*] --> Pending + Pending --> Preparing: session persisted + Preparing --> Ready: digests and endpoint ready + Ready --> Active: first target milestone + Ready --> RequestsRejected: expiresAt elapsed + Active --> RequestsRejected: expiresAt elapsed +``` + +Milestones are conditions on both the exact session and exact +`MachineOperation.status.targets[]` entry. Operation-wide conditions are not +used for target progress, preventing multi-target cross-talk and stale callback +completion. + +## Capabilities and Trust + +The controller and servers share a per-Site HMAC-SHA256 key. A capability binds +the session name, UID, expiration, and key ID. The bearer token is never stored +in Kubernetes status. Canonical routes are rooted at: + +```text +/v1/netboot/sessions/// +``` + +Allowlisted artifacts, installer callbacks, cloud-init reporting, logs, and TPM +attestation all use this identity. Server validation checks the current session +UID and expiration before resolving any file or Machine. Internal edge decision +routes additionally use TokenReview with audience `metalman-edge`. + +Trusted-LAN endpoints may use HTTP. Public endpoints require HTTPS because URL +capabilities are bearer credentials. Redfish and TPM identities retain their +existing trust-on-first-use pinning. + +## Artifact Availability + +Session artifacts resolve by OCI digest and architecture, not by mutable tag. +Static files use HTTP range semantics. The edge preserves client Range requests; +if a backend response truncates, it reconnects and requests exactly the missing +range from any healthy server replica. TFTP uses the same immutable backend and +resuming reader before reporting `BootLoaderDownloaded`. + +No RWX volume is required. Pod-local cache loss may increase latency but cannot +change the selected bits. Server readiness and retry behavior must distinguish a +cold cache from an invalid session. + +```mermaid +sequenceDiagram + participant F as Firmware/installer + participant E as Edge + participant A as Server A + participant B as Server B + + F->>E: GET capability/artifacts/disk.img.gz + E->>A: GET artifact + A-->>E: bytes 0..N (connection lost) + E->>B: GET artifact, Range: bytes=N+1- + B-->>E: 206 remaining bytes + E-->>F: continuous response +``` + +## Provisioning Flows + +### Traditional PXE + +DHCP identity is the NIC MAC. The server selects one unexpired Ready/Active +session for the endpoint and MAC, returning conflict on ambiguity. DHCP returns +a tokenized TFTP path. TFTP and all subsequent HTTP requests use the capability, +not source IP. A remote DHCP server requires a relay on the client LAN. + +### UEFI HTTP + +DHCP may supply the signed URL directly, or the controller may configure it +through Redfish. Redfish static NIC writes occur only for `networkMode: Static`. +HTTP boot can traverse routed and NAT paths because identity is in the URL. + +### First-node Bootstrap + +```mermaid +sequenceDiagram + participant CLI as kubectl unbounded + participant K as Kubernetes API + participant PF as Reconnecting port-forward + participant E as Local edge + participant H as First host + + CLI->>K: enable Metalman; wait controller/server + CLI->>K: create ExternalL2 endpoint; update designated Machine + CLI->>PF: forward loopback port to a Ready server pod + CLI->>E: start edge with projected edge token + CLI->>K: claim endpoint Ready + H->>E: DHCP/TFTP/HTTP provisioning + H->>K: kubelet creates designated Node + CLI->>K: observe designated Node Ready + CLI->>E: stop + CLI->>K: restore Machine endpoint; delete ephemeral resources +``` + +`kubectl unbounded site bootstrap-netboot` never runs Metalman controllers +locally. It reconnects its local-to-pod SPDY forward across server replacement +and stops automatically only for the Node selected by the requested Machine. + +With `--routed-cidr`, the command also starts the reusable unbounded-net node +agent as a temporary external gateway. It creates a synthetic unschedulable, +tainted Node, an External WireGuard GatewayPool, and a Site assignment. The +agent runs without kubelet, CNI, or status-transport assumptions and cleans host +networking on shutdown. WireGuard provides only L3 reachability for BMC and +downstream CIDRs; DHCP remains on the local L2. + +## Failure Semantics + +| Failure | Expected behavior | +|---------|-------------------| +| Controller leader loss | Durable operation/session references allow the new leader to resume before issuing another side effect. | +| Server pod deletion | Edge reconnects; range-capable transfers resume against another replica. | +| Mutable OCI tag changes | Existing sessions continue using their pinned digest. | +| Machine edit during provision | Existing session rendering and boot configuration remain unchanged. | +| Stale callback | Capability/session UID mismatch rejects it; sibling targets are untouched. | +| Endpoint replacement | UID mismatch blocks the session rather than using a new endpoint under the same name. | +| ManagedL2 pod/node loss | New sessions wait for endpoint readiness; active firmware connections may fail unless the L2 address itself is redundant. | +| Bootstrap CLI or edge exits | Ephemeral endpoint, Machine endpoint override, gateway resources, token files, port-forward, and child processes are cleaned. | + +## Deployment and RBAC + +- Controller and server have separate ServiceAccounts and least-role-specific + Kubernetes permissions. +- The edge ServiceAccount has no controller permissions; its internal requests + are authenticated by TokenReview. +- Controller and server mount the capability key. Only TLS-terminating edges + mount serving certificates. +- Controller/server use ordinary pod networking. ManagedL2 alone may use the + host network and required node placement. +- Synthetic external gateway Nodes are marked unschedulable, tainted, and + labeled so the normal unbounded-net DaemonSet excludes them. + +## Alternatives Rejected + +### Replicate the old monolith + +Nonleaders served from independent caches while only the leader reconciled +images, and every replica advertised a different node-local URL. Replication did +not provide a stable endpoint or coherent state. + +### Keep Metalman on every Site host network + +This preserves direct DHCP but creates a first-node scheduling dependency, +couples controllers to privileged ports, and makes ordinary rollouts disrupt +the advertised endpoint. + +### Shared RWX artifact cache + +Shared storage can reduce cold-cache latency but does not solve mutable request +identity, stale callbacks, or active connection loss. Digest snapshots and range +resume provide correctness without requiring a storage class. + +### Identify clients by source IP + +NAT, proxies, address reuse, and multiple interfaces make source addresses +ambiguous and forgeable. Session capabilities provide explicit operation-target +identity across every routed topology. + +### Run controllers in the kubectl process + +Local controller state would compete with the in-cluster deployment and require +broad persistent credentials. The bootstrap CLI instead runs only an edge and +uses a reconnecting tunnel to the in-cluster server. + +### Carry DHCP over WireGuard + +The existing dataplane is routed L3, not an Ethernet extension. A local edge or +DHCP relay is simpler and preserves the routing model. + +## Verification + +- API schema tests cover endpoint/session scope, immutability, and valid axes. +- MachineOperation tests prove session persistence before Redfish side effects, + immutable inputs, and target-scoped milestones. +- Server and edge tests cover capability validation, digest selection, Range + requests, truncated-backend resume, DHCP decisions, and tokenized TFTP. +- Operator tests cover split roles, no host networking on controller/server, + Services, PDBs, TLS rotation, endpoint workload types, and RBAC separation. +- CLI tests cover preflight-before-mutation, rollout waits, token rotation, + port-forward reconnection, exact-node handoff, early child exit, rollback, and + optional external gateway cleanup. +- Smoke suites exercise traditional PXE and Redfish HTTP boot through split + roles; the traditional suite deletes a server pod during provisioning. diff --git a/docs/content/concepts/bare-metal.md b/docs/content/concepts/bare-metal.md index 22d4b1e5d..5d54a0b2e 100644 --- a/docs/content/concepts/bare-metal.md +++ b/docs/content/concepts/bare-metal.md @@ -18,30 +18,30 @@ If your machines already have Linux installed and are reachable via SSH, use the ## How PXE Boot Works -PXE (Preboot Execution Environment) is a firmware feature that lets a machine -boot from the network instead of a local disk. metalman acts as the PXE -infrastructure: +PXE and UEFI HTTP boot let a machine start from the network instead of a local +disk. Metalman separates the Kubernetes control plane, replicated artifact +servers, and network-facing edges: ![PXE boot flow: Bare-Metal Machine boots via DHCP, TFTP, and HTTP from metalman, then joins the Kubernetes API with a bootstrap token](../../img/bare-metal-pxe-boot.svg) The boot flow in detail: -1. **DHCP Discovery** -- The machine's PXE firmware broadcasts a DHCP request. - metalman responds with an IP address and the location of the bootloader. +1. **Session creation** -- A HostReplace operation creates an immutable + NetbootSession containing the endpoint, resolved OCI digests, boot settings, + rendered inputs, and expiry. -2. **TFTP Boot** -- The firmware downloads the bootloader via TFTP. +2. **Firmware configuration** -- DHCP or Redfish supplies a capability-scoped + TFTP path or HTTP URL and, when selected, network configuration. -3. **HTTP Artifacts** -- The bootloader fetches the kernel, initramfs, and - configuration files from metalman's HTTP server. These are sourced from - the Machine's `spec.pxe.netbootImage`, or from Metalman's default netboot - image when that field is omitted. +3. **Boot artifacts** -- The edge obtains the bootloader, kernel, initramfs, and + configuration from replicated servers. Files come from the Machine's + `spec.host.netboot.netbootImage`, or Metalman's default netboot image. 4. **Kernel Boot** -- The machine boots into the downloaded kernel and initramfs. -5. **Token Retrieval** -- The init process contacts metalman's health endpoint - to retrieve a bootstrap token. If TPM 2.0 is available, the token is - encrypted to the machine's TPM. +5. **Token retrieval** -- The agent uses the authenticated session attestation + route. If TPM 2.0 is available, the token is encrypted to that machine's TPM. 6. **Cluster Join** -- kubelet uses the bootstrap token to join the Kubernetes cluster, just like an SSH-provisioned node. @@ -51,29 +51,31 @@ The boot flow in detail: ## Key Concepts -### DHCP Modes +### Endpoints -metalman supports two DHCP modes depending on your network topology: +`NetbootEndpoint` declares the stable client-facing address and edge placement: -- **Interface mode** (`--dhcp-interface eth0`) -- metalman listens for - broadcast DHCP on a specific NIC. Use this when metalman runs on the same - L2 segment as the bare-metal machines. +- **ManagedL2** -- The operator places a host-network edge on a selected node + attached to the provisioning LAN. +- **HTTP** -- The operator creates replicated HTTP edge pods and a Service. + Public endpoints require HTTPS. +- **ExternalL2** -- An edge outside the cluster owns DHCP/TFTP/private HTTP. + The first-node bootstrap command uses this type temporarily. -- **Relay mode** (default, no `--dhcp-interface`) -- metalman listens for - unicast DHCP forwarded by a DHCP relay agent. Use this when metalman and - the machines are on different subnets. +DHCP broadcasts require an edge on the target L2 or a local relay. WireGuard is +L3 only and does not extend the broadcast domain. ### OCI Images Metalman uses two OCI images during PXE provisioning: -- The machine image, referenced by `spec.pxe.image`, contains `/disk/disk.img.gz`. -- The netboot image, referenced by `spec.pxe.netbootImage` or by Metalman's +- The machine image, referenced by `spec.host.netboot.image`, contains `/disk/disk.img.gz`. +- The netboot image, referenced by `spec.host.netboot.netbootImage` or by Metalman's default, contains the reusable PXE boot environment. Netboot images are built `FROM scratch` and contain all files needed for PXE booting a machine under `/disk/`. Files with a `.tmpl` suffix are Go templates -rendered per-machine at serve time; other files are served verbatim. A +rendered from immutable session data; other files are served verbatim. A `metadata.yaml` file provides image-level configuration such as `dhcpBootImageName`. @@ -83,39 +85,42 @@ rendered per-machine at serve time; other files are served verbatim. A | **Templates** | Files with `.tmpl` suffix - rendered from Go templates with per-machine context (e.g., kernel command line) | | **Configuration** | `metadata.yaml` - image-level settings such as the DHCP boot filename | -Machine and netboot images are pulled and cached locally by the OCI reconciler. +Machine and netboot image tags are resolved to digests before the host is +powered on. Server caches are disposable; the durable session records the exact +digest and allowlisted files. ### Machine CRD (PXE Fields) For PXE-provisioned machines, the `Machine` resource includes: -- **`spec.pxe.image`** -- OCI machine image reference containing `/disk/disk.img.gz` +- **`spec.host.netboot.image`** -- OCI machine image reference containing `/disk/disk.img.gz` (e.g. `"ghcr.io/azure/host-ubuntu2404:v1"`). -- **`spec.pxe.architecture`** -- Optional target CPU architecture for PXE boot +- **`spec.host.netboot.architecture`** -- Optional target CPU architecture for boot artifacts and machine images. Defaults to `amd64`; allowed values are `amd64` and `arm64`. -- **`spec.pxe.netbootImage`** -- Optional OCI netboot image reference containing +- **`spec.host.netboot.netbootImage`** -- Optional OCI netboot image reference containing PXE boot artifacts. When omitted, Metalman uses its configured default `netboot` image. -- **`spec.pxe.dhcpLeases`** -- NIC specifications: MAC address and IP +- **`spec.host.netboot.endpointRef`** -- Required NetbootEndpoint name. +- **`spec.host.netboot.transport`**, **`configurationSource`**, and + **`networkMode`** -- Independent firmware transport, boot configuration + source, and network configuration mode. +- **`spec.host.netboot.dhcpLeases`** -- NIC specifications: MAC address and IP assignment for each interface. During install, the default netboot template passes the matching lease MAC to the initrd so it can select the provisioning NIC without relying on names such as `eth0`. -- **`spec.pxe.targetDisk`** -- Optional block device path for the disk that +- **`spec.host.netboot.targetDisk`** -- Optional block device path for the disk that receives the machine image. Set this on hosts with multiple disks; when omitted, the installer selects a disk automatically. -- **`spec.pxe.redfish`** -- Optional BMC connection details (endpoint, username, +- **`spec.host.netboot.redfish`** -- Optional BMC connection details (endpoint, username, password secret) for remote power management. -- **`spec.pxe.cloudInit`** -- Optional cloud-init customization. References a +- **`spec.host.netboot.cloudInit`** -- Optional cloud-init customization. References a ConfigMap containing user-data that is merged with the vendor-data managed by Unbounded. -### Site Isolation - -In environments with multiple metalman instances (e.g., different racks or -sites), the `--site` flag scopes each instance to machines labeled with -`unbounded-cloud.io/site=`. This prevents one metalman from interfering -with another's machines. +Supported combinations are TFTP+DHCP+DHCP, HTTP+DHCP+DHCP, +HTTP+Redfish+DHCP, and HTTP+Redfish+Static. Invalid combinations are rejected by +the API. ### TPM 2.0 Attestation @@ -131,8 +136,9 @@ metalman uses TPM 2.0 for secure bootstrap token delivery: 3. **AES-256-GCM** -- The actual token payload is encrypted with AES-256-GCM, with the key wrapped by the TPM credential. -This ensures that bootstrap tokens cannot be intercepted by other machines on -the network. +The attestation request is also bound to an expiring session capability. Public +HTTP boot endpoints require HTTPS so artifact and callback capabilities are not +exposed in transit. ### MachineOperation-Based Operations diff --git a/docs/content/guides/pxe.md b/docs/content/guides/pxe.md index fff8fac5f..fd7ab3309 100644 --- a/docs/content/guides/pxe.md +++ b/docs/content/guides/pxe.md @@ -1,271 +1,194 @@ --- -title: "Bare Metal (PXE)" +title: "Bare Metal Netboot" weight: 3 -description: "Netboot bare metal machines into your cluster." +description: "Provision bare-metal machines with resilient PXE or UEFI HTTP boot." --- ## Overview -Metalman is a controller that PXE-boots bare-metal servers and joins them to your Kubernetes cluster. It bundles DHCP, TFTP, and HTTP servers into a single binary, integrates with Redfish BMCs for remote power management, and uses TPM 2.0 attestation for secure bootstrap token delivery. +Metalman provisions bare-metal nodes from OCI images and joins them to a +Kubernetes cluster. Its controller, serving, and network-edge responsibilities +are separate so the normal control plane does not need host networking and a +server pod restart does not invalidate an in-flight provision. -API group: `unbounded-cloud.io/v1alpha3`. CRD: **Machine** (`mach`), cluster-scoped. +## Architecture -## Prerequisites - -- A Kubernetes cluster with access to `kubectl`. -- Bare-metal servers with UEFI PXE firmware and a BMC exposing a Redfish API. -- Layer-2 network connectivity (or a DHCP relay agent) between metalman and the PXE NICs. -- Network access from metalman to each BMC (HTTPS/443) and to the Kubernetes API (TCP/6443). -- Network access from target machines to metalman (UDP/67, UDP/69, TCP/8880) and to the Kubernetes API (TCP/6443). -- TPM 2.0 modules on target machines (required for secure attestation). - -## Deploy Metalman - -Metalman is a per-site component managed by the unbounded operator. First install -the operator (this also installs the CRDs): - -```bash -kubectl unbounded install -``` - -Then enable metalman on a Site by passing `--enable-metalman` to `site init`, or by -setting the metalman entry in `Site.spec.components`: - -```bash -kubectl unbounded site init --name my-edge-site --enable-metalman -``` +The operator deploys these per-Site resources after Metalman is enabled: -`kubectl unbounded site init` also runs `install` by default, so a fresh site only -needs the single command above. The operator then reconciles the `unbounded-system` -namespace, ServiceAccounts (`metalman-controller`, `metalman-bootstrap`), RBAC roles, -and a metalman Deployment for the Site. - -Key `serve-pxe` flags (baked into the operator-managed per-site Deployment; `--site` -scoping is inherent to the per-site component): - -| Flag | Default | Description | -|------|---------|-------------| -| `--dhcp-interface` | *(none - relay mode)* | NIC for broadcast DHCP | -| `--site` | *(none)* | Scope to machines with a specific site label | -| `--http-port` | 8880 | HTTP server port | -| `--cache-dir` | `~/.unbounded/metalman/cache` | Local cache for downloaded images | -| `--health-port` | 8081 | Health/readiness probe port | -| `--serve-url` | | External URL of this metalman instance | -| `--default-netboot-image` | release-matched `netboot` image | PXE boot environment used when `spec.pxe.netbootImage` is omitted | - -When `--dhcp-interface` is set, metalman binds to the interface for broadcast DHCP, and the DHCP server requires leader election. Without it, metalman accepts relayed (unicast) DHCP packets and the DHCP server responds regardless of leader status. Leader election always runs at the manager level for the reconcilers. - -## Images - -Metalman uses a machine image and a netboot image for each PXE repave. - -- `spec.pxe.image` is the machine image. It contains `/disk/disk.img.gz`, a - gzip-compressed raw disk image written to the target disk. -- `spec.pxe.architecture` selects the target architecture (`amd64` or `arm64`) - used when pulling machine and netboot image platform manifests. It defaults - to `amd64`. -- `spec.pxe.netbootImage` is the reusable PXE boot environment. It contains - bootloaders, kernel, initrd, templates, metadata, and `unbounded-agent`. If - omitted, Metalman uses the release-matched `--default-netboot-image`. -- `spec.pxe.bootProtocol` selects the network boot trigger. `PXE` is the - default and uses DHCP/TFTP bootfile options. `HTTP` uses Redfish UEFI HTTP - boot and requires a Redfish block. - -Both images are standard OCI container images built `FROM scratch` with -artifacts under `/disk/`. Files with a `.tmpl` suffix in the netboot image are -Go templates rendered per-machine at serve time; other files are served -verbatim. A `metadata.yaml` file in the netboot image provides image-level -configuration such as `dhcpBootImageName` and `httpBootPath`. - -Images are built, tagged, and pushed using standard container tooling: +- One leader-elected `metalman-controller-` Deployment for operations, + Redfish, OCI resolution, and immutable session creation. +- A two-replica `metalman-server-` Deployment and ClusterIP Service for + artifact serving, callbacks, and TPM attestation. +- A capability-signing Secret shared by the controller and servers. +- Edge workloads selected by `NetbootEndpoint` resources. -```bash -docker build -t ghcr.io/azure/host-ubuntu2404:v1 -f images/host-ubuntu2404/Containerfile . -docker build -t ghcr.io/azure/netboot:v1 -f images/netboot/Containerfile . -docker push ghcr.io/azure/host-ubuntu2404:v1 -docker push ghcr.io/azure/netboot:v1 -``` +Only a `ManagedL2` edge uses host networking. `HTTP` edges are replicated +ordinary pods. `ExternalL2` endpoints run outside the cluster, including the +temporary administrator-machine bootstrap flow. -Template context includes `.Machine`, `.ApiserverURL`, `.ServeURL`, `.KubernetesVersion`, and `.ClusterDNS`. +## Prerequisites -See the [CRD Reference]({{< relref "/reference/machina-crd" >}}) for the full Machine spec. +- An installed Unbounded operator and a Site with Metalman enabled. +- A target Machine with a UEFI-capable NIC and TPM 2.0. +- BMC connectivity from the controller when Redfish is configured. +- Target access to the endpoint and Kubernetes API. +- For TFTP: UDP 69 plus negotiated TFTP transfer ports. +- For DHCP: an edge on the target L2 or a DHCP relay. WireGuard does not carry + Ethernet broadcasts. +- For internet-facing HTTP boot: HTTPS with a trusted certificate. -## Machine CRD +## Define an Endpoint -A Machine represents a single bare-metal host. The `spec.pxe` section ties together the machine image, optional netboot image override, network config, and BMC credentials: +This endpoint places DHCP, TFTP, and HTTP on a selected provisioning node: ```yaml apiVersion: unbounded-cloud.io/v1alpha3 -kind: Machine +kind: NetbootEndpoint metadata: - name: server-01 - labels: - unbounded-cloud.io/site: rack-a + name: rack-a-l2 spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - architecture: amd64 - # Optional. Defaults to PXE. Set to HTTP for Redfish UEFI HTTP boot. - bootProtocol: PXE - # Optional. Omit to use Metalman's default netboot image. - netbootImage: ghcr.io/azure/netboot:v1 - # Optional. Recommended on hosts with multiple disks. - targetDisk: /dev/disk/by-id/example-os-disk - dhcpLeases: - - ipv4: "10.10.0.50" - mac: "aa:bb:cc:dd:ee:ff" - subnetMask: "255.255.255.0" - gateway: "10.10.0.1" - dns: ["8.8.8.8"] - redfish: - url: "https://bmc-01.example.com" - username: admin - passwordRef: - name: bmc-passwords - namespace: unbounded-system - key: bmc-01 + siteRef: rack-a + type: ManagedL2 + externalURL: http://10.20.0.2:8880 + trust: TrustedLAN + tls: + mode: Disabled + managedL2: + interface: eno1 + address: 10.20.0.2 + nodeSelector: + matchLabels: + provisioning.unbounded-cloud.io/rack: a ``` -Store BMC passwords in a Secret referenced by `passwordRef`. See the [CRD Reference]({{< relref "/reference/machina-crd" >}}) for all fields. +Use an `HTTP` endpoint for a Service-backed HTTP edge. Set `trust: Public`, an +`https://` external URL, and TLS mode `Secret` or `External` for public access. -## Cloud-Init Customization - -Cloud-init on PXE-booted machines uses two data sources that are merged at boot: - -- **Vendor-data** (managed by Unbounded) -- Contains the agent configuration, bootstrap scripts, and system defaults required for the node to join the cluster. This is not user-editable. -- **User-data** (managed by the cluster operator) -- Optional customization such as SSH keys, additional packages, or host-level configuration. - -When no user-data is configured, metalman serves a minimal `#cloud-config` document. To supply custom user-data, create a ConfigMap and reference it from the Machine spec: - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: my-cloud-init - namespace: unbounded-system -data: - user-data: | - #cloud-config - ssh_authorized_keys: - - ssh-rsa AAAA... - packages: - - vim - - htop -``` - -Then reference the ConfigMap in the Machine: +## Define a Machine ```yaml apiVersion: unbounded-cloud.io/v1alpha3 kind: Machine metadata: name: server-01 + labels: + unbounded-cloud.io/site: rack-a spec: - pxe: - image: ghcr.io/azure/host-ubuntu2404:v1 - dhcpLeases: - - ipv4: "10.10.0.50" - mac: "aa:bb:cc:dd:ee:ff" - subnetMask: "255.255.255.0" - gateway: "10.10.0.1" - dns: ["8.8.8.8"] - cloudInit: - userDataConfigMapRef: - name: my-cloud-init - namespace: unbounded-system + host: + netboot: + image: ghcr.io/azure/host-ubuntu2404:v1 + netbootImage: ghcr.io/azure/netboot:v1 + architecture: amd64 + endpointRef: rack-a-l2 + transport: TFTP + configurationSource: DHCP + networkMode: DHCP + targetDisk: /dev/disk/by-id/example-os-disk + dhcpLeases: + - mac: aa:bb:cc:dd:ee:ff + ipv4: 10.20.0.50 + subnetMask: 255.255.255.0 + gateway: 10.20.0.1 + dns: [10.20.0.1] + redfish: + url: https://bmc-01.example.com + username: admin + passwordRef: + name: bmc-passwords + namespace: unbounded-system + key: bmc-01 ``` -The `key` field defaults to `user-data` but can be overridden to select a different key from the ConfigMap. Both `data` and `binaryData` entries are supported. - -If the referenced ConfigMap does not exist, metalman falls back to the default minimal cloud-config. If the ConfigMap exists but the referenced key is not found, metalman returns an error and the machine will not receive user-data. - -## Boot Flow +The supported axes are independent: -1. **Repave requested.** A `HostReplace` `MachineOperation` targets the Machine. Metalman sets the boot override and force-restarts the server. For `bootProtocol: PXE`, it selects PXE boot. For `bootProtocol: HTTP`, it sets a Redfish UEFI HTTP boot URL from the netboot image metadata. -2. **Network boot.** In PXE mode, DHCP assigns the static IP by MAC, advertises the TFTP bootfile, and TFTP serves `shimx64.efi`. In HTTP mode, Metalman configures the first lease as a static Redfish EthernetInterface and the firmware downloads the Redfish-supplied URL from Metalman's HTTP server; DHCP is not required for this path. -3. **GRUB decision.** A rendered `grub.cfg` checks for an active `HostReplace` `MachineOperation` targeting the Machine. If a repave is requested, GRUB boots the PXE installer; otherwise it chainloads the local OS. When a Machine has multiple DHCP leases, metalman renders the lease matching the request source IP and passes that lease's MAC as `unbounded.boot_mac`. -4. **Installer (initrd overlay).** An init script in the initrd: - - Loads storage and network drivers, selects the provisioning NIC by MAC, and configures the static IP and DNS from kernel cmdline. - - Writes matching MAC-based static netplan configuration into the installed system before reboot and disables cloud-init network rendering so fallback DHCP configuration cannot conflict with it. The default netboot image also serves the selected lease as NoCloud `network-config`. - - Downloads the gzip-compressed raw disk image from the machine image over HTTP (retries up to 120 times). - - Writes the image to `spec.pxe.targetDisk` when set, otherwise to an automatically selected block device. - - Mounts the root filesystem and injects cloud-init config and the agent configuration. - - Calls `/pxe/disable` on metalman to signal completion, then reboots. -5. **First boot.** cloud-init downloads the `unbounded-agent` binary from metalman and runs `unbounded-agent start`. -6. **Node join.** The agent installs containerd and kubelet, performs TPM attestation to obtain a bootstrap token (see below), configures kubelet, and starts it. The node TLS-bootstraps and reaches `Ready`. +| Transport | Boot target source | Firmware network | +|-----------|--------------------|------------------| +| TFTP | DHCP | DHCP | +| HTTP | DHCP | DHCP | +| HTTP | Redfish | DHCP | +| HTTP | Redfish | Static | -## TPM Attestation +For HTTP with Redfish and static networking, the first DHCP lease supplies the +Redfish EthernetInterface address, gateway, DNS, and NIC identity. HTTP with +DHCP receives its capability boot URL through DHCP instead. -Metalman uses TPM 2.0 to securely deliver a bootstrap token without embedding secrets in the image. +## Provision -1. On first boot, the `unbounded-agent` creates a TPM Endorsement Key (EK) and Storage Root Key (SRK), then POSTs them to metalman's `/attest` endpoint. -{{< callout type="important" >}} -Metalman uses trust-on-first-use (TOFU) for TPM attestation: once a machine's EK public key is stored in `status.tpm.ekPublicKey`, any attestation from a different EK is rejected (HTTP 403). If a TPM is legitimately replaced, you must clear `status.tpm.ekPublicKey` from the Machine CR before the machine can re-enroll. -{{< /callout >}} +Create a HostReplace operation with the CLI: -3. Metalman wraps an AES-256 key via `tpm2.CreateCredential` (bound to the EK and SRK), then encrypts a 1-hour ServiceAccount token with AES-256-GCM and returns both to the client. -4. The agent uses the TPM `ActivateCredential` operation to recover the AES key, decrypts the token, and writes a bootstrap kubeconfig for kubelet to use during TLS bootstrapping. +```bash +kubectl unbounded machine replace server-01 --force +``` -The `metalman-bootstrap` ServiceAccount has RBAC for `system:node-bootstrapper` and `certificatesigningrequests:nodeclient` auto-approval. +The controller creates an immutable `NetbootSession` for that operation target. +The session pins OCI digests and all rendered inputs before the BMC is powered +on. Firmware, installer, cloud-init, and attestation use capability-scoped URLs. +Callbacks update only the matching operation target, so stale boots and +multi-target operations cannot satisfy each other's milestones. -## Site Isolation +Static artifacts support HTTP ranges. If an edge loses its backend server pod, +it reconnects and resumes the missing byte range against another replica. -Use the `--site` flag to scope a metalman instance to machines labeled `unbounded-cloud.io/site=`. Each site gets its own leader-election lease. +## Bootstrap the First Site Node -Run separate metalman instances for different racks or network segments: +When no Site node exists for a managed L2 edge, run an edge temporarily on an +administrator machine attached to that LAN: ```bash -# Instance for rack-a -metalman serve-pxe --site=rack-a --dhcp-interface=eth1 - -# Instance for rack-b -metalman serve-pxe --site=rack-b --dhcp-interface=eth2 +kubectl unbounded site bootstrap-netboot rack-a \ + --machine server-01 \ + --interface eno1 \ + --address 10.20.0.2 ``` -## Operations +The command runs only the edge data plane locally. Controllers remain in the +cluster. It creates an ephemeral endpoint, uses a reconnecting port-forward to +the server Service, and automatically stops after the designated Node becomes +Ready. This first-node policy intentionally supports one designated Machine. -Use `kubectl unbounded` to create and watch `MachineOperation` objects for day-2 actions: +For routed BMC or downstream LAN access, add repeatable CIDRs: ```bash -kubectl unbounded machine reboot server-01 -kubectl unbounded machine repave server-01 +kubectl unbounded site bootstrap-netboot rack-a \ + --machine server-01 \ + --interface eno1 \ + --address 10.20.0.2 \ + --routed-cidr 10.30.0.0/24 \ + --gateway-external-address 203.0.113.20 ``` -`machine reboot` creates a `HostReboot` operation. `machine repave` creates a -`HostReplace` operation. Metalman records progress on the operation target and -conditions, including `BootImageWritten` and `CloudInitDone`. - -## Troubleshooting +This starts the existing unbounded-net dataplane as an ephemeral external +gateway. It still provides L3 routing only; DHCP remains local to the LAN. -{{< callout type="warning" >}} -Running metalman's DHCP server on a network segment that already has an active DHCP server will cause conflicts. Ensure metalman is the only DHCP server on the PXE segment, or use relay mode to isolate DHCP traffic. -{{< /callout >}} +## Cloud-Init and Attestation -**Machine stuck in repaving.** Check the active `HostReplace` `MachineOperation` target stage and conditions. Verify the target machine can reach metalman on TCP/8880 and that the BMC is reachable from metalman. Metalman retries the Redfish boot request while the operation remains active. +Optional cloud-init user-data comes from +`spec.host.netboot.cloudInit.userDataConfigMapRef`. Metalman snapshots it into +the session, while vendor-data installs and configures `unbounded-agent`. -**DHCP not responding.** Confirm `--dhcp-interface` points to the correct NIC (broadcast mode) or that your relay agent forwards to metalman's DHCP port. Check that no other DHCP server is competing on the same segment. +The agent sends TPM EK/SRK material to the session's authenticated attestation +URL. Metalman uses TPM CreateCredential and AES-GCM to deliver a short-lived +bootstrap token. The TPM EK and Redfish certificate use trust on first use and +are pinned in Machine status. -**BMC connection failures.** Metalman uses TLS TOFU for Redfish - the first connection captures the BMC certificate fingerprint in `status.redfish.certFingerprint`. If a BMC certificate rotates, clear the fingerprint from the Machine status. Verify HTTPS/443 connectivity from metalman to the BMC. +## Troubleshooting -**TPM attestation rejected (403).** The EK public key has changed since the initial TOFU. If the TPM was legitimately replaced, clear `status.tpm.ekPublicKey` from the Machine CR to allow re-enrollment. +**Session remains Preparing.** Check endpoint `Ready`, controller/server +rollouts, registry reachability, and whether both OCI images resolve. -**Node not joining.** Verify the target machine can reach the Kubernetes API on TCP/6443. Check that the `metalman-bootstrap` ServiceAccount and RBAC are in place. Inspect kubelet logs on the target for certificate signing request errors. +**DHCP does not respond.** Confirm the ManagedL2 or external edge owns the +correct interface. For remote subnets, use a DHCP relay; do not expect a +WireGuard tunnel to forward broadcasts. -## Limitations +**Firmware gets 401 or 404.** Verify it is using the current session capability +URL. Capabilities are session-bound and expire with the session. -{{< callout type="note" >}} -Only Ubuntu 24.04 images are currently supported. The repave boot request timeout is fixed at 30 minutes. -{{< /callout >}} +**Transfer stops after pod deletion.** Check that another server replica is +Ready and the edge can reconnect. Immutable artifact requests are range +resumable, but firmware itself must retry if it loses its direct edge connection. -## See Also +**Attestation is rejected.** If the TPM was intentionally replaced, clear the +pinned EK only after validating the hardware change. -- **[Bare Metal Concepts]({{< relref "concepts/bare-metal" >}})** -- Deeper - explanation of PXE boot, DHCP modes, TPM attestation, and pool isolation. -- **[Project Overview]({{< relref "concepts/overview" >}})** -- How metalman - fits into the broader system. -- **[CRD Reference]({{< relref "reference/machina-crd" >}})** -- Complete - Machine API specification. -- **[Architecture]({{< relref "reference/architecture" >}})** -- Internal - design of the PXE provisioning pipeline. -- **[SSH Guide]({{< relref "guides/ssh" >}})** -- Alternative provisioning - path for machines with an existing OS. +See the [CRD reference]({{< relref "/reference/machina-crd" >}}) and +[CLI reference]({{< relref "/reference/cli" >}}) for all fields and flags. diff --git a/docs/content/reference/agent/configuration.md b/docs/content/reference/agent/configuration.md index da6cb0b49..66f435eb4 100644 --- a/docs/content/reference/agent/configuration.md +++ b/docs/content/reference/agent/configuration.md @@ -51,4 +51,4 @@ for command usage, exit behavior, and the current check list. | `Kubelet.Labels` | Key-value labels applied to the Node on registration. | | `Kubelet.RegisterWithTaints` | Taints applied to the Node on registration (`key=value:effect`). | | `OCIImage` | *(optional)* OCI image reference for the rootfs. Uses the built-in default image when empty. | -| `Attest.URL` | *(optional)* Base URL of a metalman serve-pxe instance for TPM attestation. | +| `Attest.URL` | *(optional)* Capability-scoped Metalman session URL for TPM attestation. | diff --git a/docs/content/reference/architecture.md b/docs/content/reference/architecture.md index ecfd244de..7c84fccf2 100644 --- a/docs/content/reference/architecture.md +++ b/docs/content/reference/architecture.md @@ -44,24 +44,27 @@ service host to a public FQDN. but the shipped ConfigMap (rendered from `deploy/machina/03-config.yaml.tmpl`) sets it to 50. `provisioningTimeout` defaults to 5 minutes. -### metalman -- Bare Metal PXE Controller +### metalman -- Bare Metal Netboot -Binary `cmd/metalman`, deployed as `metalman-controller` in `unbounded-system`. +Binary `cmd/metalman`, deployed in separate per-Site roles in +`unbounded-system`: -Runs three reconcilers and four network servers: +| Role | Responsibility | +|------|----------------| +| `metalman controller` | Leader-elected MachineOperation and Redfish reconciliation, OCI resolution, and immutable NetbootSession creation. | +| `metalman server` | Replicated, Service-backed artifact serving, callbacks, TPM attestation, and authenticated edge decisions. | +| `metalman edge` | DHCP/TFTP on a provisioning LAN and HTTP proxying. It has no controller credentials. | -| Reconciler / Server | Role | -|-------------------------|-------------------------------------------------------------| -| OCIReconciler | Pulls and caches OCI netboot images from container registries. | -| Redfish Reconciler | TOFU TLS cert pinning for BMC Redfish endpoints. | -| MachineOperation Reconciler | BMC power control, boot override, reboot, and repave operations via Redfish REST. | -| DHCP server (UDP/67) | Static IP assignment by MAC address. | -| TFTP server (UDP/69) | Bootloader delivery. | -| HTTP server (TCP/8880) | Kernel, initrd, Go-templated configs, `/attest` (TPM), `/pxe/disable`. | -| Health server (TCP/8081)| Liveness and readiness probes. | +The operator deploys one controller, two server replicas, a server Service and +PodDisruptionBudget, and a shared capability-signing Secret. Controller and +server pods use ordinary pod networking. `NetbootEndpoint` resources determine +edge placement; only a `ManagedL2` edge uses host networking. -Site-based scoping: the `--site` flag and the `unbounded-cloud.io/site` label -restrict each instance to a subset of Machines. Leader election is per-site. +Each HostReplace target receives an immutable `NetbootSession` that pins the +endpoint, OCI digests, artifact allowlist, rendered inputs, and expiry. HMAC +capability URLs identify the exact session for HTTP, TFTP, callbacks, and +attestation. Static HTTP transfers support ranges so an edge can reconnect to a +different server replica and resume an interrupted download. ### kubectl-unbounded -- CLI Plugin @@ -71,6 +74,7 @@ Binary `cmd/kubectl-unbounded`. Provides subcommands: |--------------------|---------| | `install` | Bootstraps CRDs and `unbounded-operator`; component workloads are reconciled from `Site.spec.components`. | | `site init` | Initializes a new site by bootstrapping Unbounded when needed, creating site resources, and creating the bootstrap token. | +| `site bootstrap-netboot` | Runs a temporary local netboot edge until one designated first Site Node is Ready. | | `machine register` | Registers a machine to a site, creating a `Machine` CR with auto-discovery of SSH secrets and bootstrap tokens. | ### inventory -- Hardware Collector @@ -92,7 +96,7 @@ Represents a host and drives its lifecycle. | Spec field | Description | |-----------------------|-------------| | `spec.ssh` | SSH connectivity (host, port, user, privateKeyRef) and optional bastion config. | -| `spec.pxe` | PXE config: machine image reference, optional netboot image override, dhcpLeases, redfish settings. | +| `spec.host.netboot` | Netboot image, endpoint, independent transport/configuration/network axes, DHCP leases, and Redfish settings. | | `spec.kubernetes` | Kubernetes version, bootstrapTokenRef, nodeRef, nodeLabels. | Status includes phase, message, conditions, SSH fingerprint, Redfish cert @@ -103,15 +107,17 @@ fingerprint, and TPM info. The API defines condition type constants including ### Netboot OCI Images -Metalman uses two OCI images for PXE repaves. `Machine.spec.pxe.image` references -the machine image containing `/disk/disk.img.gz`. `Machine.spec.pxe.netbootImage` -optionally references the reusable PXE boot environment; when omitted, Metalman -uses its configured default `netboot` image. `Machine.spec.pxe.architecture` +Metalman uses two OCI images for netboot repaves. +`Machine.spec.host.netboot.image` references the machine image containing +`/disk/disk.img.gz`. `Machine.spec.host.netboot.netbootImage` optionally +references the reusable boot environment; when omitted, Metalman uses its +configured default netboot image. `Machine.spec.host.netboot.architecture` selects the OCI platform manifest for both images and defaults to `amd64`. -Netboot images contain all files needed for PXE booting under `/disk/`. Files -with a `.tmpl` suffix are Go templates rendered per-machine at serve time. A -`metadata.yaml` provides image-level configuration such as `dhcpBootImageName`. +Netboot images contain all files needed for network booting under `/disk/`. +Files with a `.tmpl` suffix are Go templates rendered from the immutable session +snapshot. A `metadata.yaml` provides image-level configuration such as the TFTP +or HTTP firmware artifact path. ### Resource relationships @@ -153,18 +159,19 @@ For a walkthrough, see the [SSH Provisioning Guide]({{< ref "guides/ssh" >}}). ### PXE Path (metalman) -1. `Machine` CR created with `spec.pxe`. -2. A `HostReplace` `MachineOperation` requests a repave; metalman sets the PXE or HTTP boot override and force-restarts the host through Redfish. -3. Host PXE-boots: DHCP (IP + boot filename) -> TFTP (bootloader) -> HTTP - (kernel, initrd, configs). -4. Init script: writes disk image, injects configs, calls `/pxe/disable`, - reboots. -5. Cloud-init: installs containerd, kubelet, tpm2-tools. -6. TPM attestation: TOFU Endorsement Key pinning, - `MakeCredential`/`ActivateCredential` exchange, AES-256-GCM encrypted - bootstrap token delivered via `/attest`. -7. kubelet TLS-bootstraps into the cluster. -8. Subsequent reboots: GRUB chainloads the local OS (no PXE). +1. A `Machine` selects a `NetbootEndpoint` through `spec.host.netboot`. +2. A `HostReplace` operation creates an immutable session and waits for its + endpoint and digest-addressed artifacts to become ready. +3. Metalman configures TFTP or UEFI HTTP boot through DHCP or Redfish, according + to the Machine's independent boot axes, and restarts the host. +4. Firmware and the installer fetch capability-scoped artifacts through an edge. +5. The installer writes the disk image and posts the exact target's + `BootImageWritten` callback before rebooting. +6. Cloud-init installs the agent and reports target-scoped progress. +7. TPM attestation uses TOFU Endorsement Key pinning, + `MakeCredential`/`ActivateCredential` exchange, and an AES-256-GCM encrypted + bootstrap token delivered through the authenticated session route. +8. kubelet TLS-bootstraps into the cluster; subsequent boots chainload the local OS. For a walkthrough, see the [PXE Provisioning Guide]({{< ref "guides/pxe" >}}). @@ -177,8 +184,9 @@ For a walkthrough, see the [PXE Provisioning Guide]({{< ref "guides/pxe" >}}). | Bootstrap tokens | Standard kubeadm tokens (`token-id` + `token-secret`). SSH path passes as env var; PXE path encrypts via TPM. | | TPM attestation | TOFU EK pinning. AES-256-GCM encrypted service-account tokens with 1-hour expiry. | | Redfish TLS | TOFU cert fingerprint pinning stored in `status.redfish.certFingerprint`. | +| Netboot requests | Expiring HMAC capabilities bound to one immutable session; public endpoints require HTTPS. | | RBAC | Separate ServiceAccounts, Roles, and ClusterRoles per controller. | -| Secret access | Via Kubernetes API only; never mounted as volumes. | +| Secret access | Scoped API reads; capability and TLS keys mount only into roles that need them. | ## Deployment @@ -195,8 +203,9 @@ applied by hand. | `deploy/machina/crd/` | `Machine` CRD definition. | | `deploy/machina/` | Namespace, RBAC (machina + metalman), ConfigMap, Deployment, Service. | -Resource defaults for both controllers: 100m CPU / 128Mi memory requests, -500m CPU / 256Mi memory limits. Both tolerate `CriticalAddonsOnly`. +Metalman controller and server workloads request 100m CPU and 128Mi memory and +limit themselves to 2 CPU and 2Gi memory. Server replicas use topology spread, +readiness/liveness probes, zero-unavailable rollouts, and a PodDisruptionBudget. Container images are multi-stage builds on Azure Linux 3.0, built with `podman`. CRDs are generated with `controller-gen` v0.20.1. diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index 3ba70474e..98fee7ccd 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -149,6 +149,50 @@ kubectl unbounded site init \ --- +### `kubectl unbounded site bootstrap-netboot` + +Run a temporary netboot edge on the administrator machine until one designated +Site Node becomes Ready. The command leaves controllers in the cluster, enables +the Site Metalman component, waits for the controller and server rollouts, +creates an ephemeral ExternalL2 endpoint, and connects the local edge to a +server pod through a reconnecting port-forward. + +```bash +kubectl unbounded site bootstrap-netboot rack-a \ + --machine server-01 \ + --interface eno1 \ + --address 10.20.0.2 +``` + +Required flags: + +| Flag | Description | +|------|-------------| +| `--machine` | Machine whose corresponding Node readiness completes bootstrap | +| `--interface` | Administrator-machine interface attached to the provisioning L2 | +| `--address` | IPv4 address on that interface, advertised to netboot clients | + +Optional flags: + +| Flag | Default | Description | +|------|---------|-------------| +| `--endpoint-name` | `bootstrap-` | Ephemeral NetbootEndpoint name | +| `--http-port` | `8880` | Local HTTP artifact port | +| `--namespace` | `unbounded-system` | Namespace containing Metalman workloads | +| `--metalman-binary` | `metalman` on PATH | Edge binary to execute | +| `--timeout` | `30m` | Maximum wait for the designated Node | +| `--routed-cidr` | none | CIDR routed through an ephemeral external gateway; repeatable | +| `--gateway-external-address` | `--address` | WireGuard endpoint address reachable by remote peers | +| `--kubeconfig` | standard lookup | Path to kubeconfig | + +The command restores the Machine's previous endpoint and deletes its ephemeral +resources during cleanup. It intentionally stops after the one designated Node +is Ready. Using `--routed-cidr` starts the unbounded-net dataplane and requires +root networking privileges. The gateway provides L3 routing, not DHCP broadcast +extension. + +--- + ### `kubectl unbounded machine` Manage Unbounded machines. diff --git a/docs/content/reference/machina-crd.md b/docs/content/reference/machina-crd.md index 9f3f90527..6ef95631b 100644 --- a/docs/content/reference/machina-crd.md +++ b/docs/content/reference/machina-crd.md @@ -56,7 +56,10 @@ top-level `spec.pxe` remains a deprecated fallback for existing Machines. | `host.netboot.image` | string | Yes | - | OCI machine image reference containing `/disk/disk.img.gz` (e.g. `"ghcr.io/azure/host-ubuntu2404:v1"`). | | `host.netboot.architecture` | string | No | `amd64` | Target CPU architecture for PXE boot artifacts and machine images. Allowed values: `amd64`, `arm64`. | | `host.netboot.netbootImage` | string | No | Metalman default | OCI netboot image reference containing PXE boot artifacts. | -| `host.netboot.bootProtocol` | string | No | `PXE` | Network boot trigger protocol for repaves. `PXE` uses DHCP/TFTP bootfile options. `HTTP` uses Redfish UEFI HTTP boot with a URL derived from the netboot image metadata. Allowed values: `PXE`, `HTTP`. | +| `host.netboot.transport` | string | No | `TFTP` | Firmware artifact transport. Allowed values: `TFTP`, `HTTP`. | +| `host.netboot.configurationSource` | string | No | `DHCP` | Source of the firmware boot target. Allowed values: `DHCP`, `Redfish`. | +| `host.netboot.networkMode` | string | No | `DHCP` | Firmware provisioning network mode. Allowed values: `DHCP`, `Static`. Static requires Redfish. | +| `host.netboot.endpointRef` | string | Yes | - | Name of the `NetbootEndpoint` serving this Machine. | | `host.netboot.dhcpLeases` | []DHCPLease | No | - | Provisioning network settings. They are served as static DHCP leases during PXE boot and used for Redfish firmware, installer, NoCloud, and installed-system static configuration during HTTP boot. | | `host.netboot.dhcpLeases[].ipv4` | string | Yes | - | Static IPv4 address to assign. | | `host.netboot.dhcpLeases[].mac` | string | Yes | - | NIC MAC address (matched case-insensitively). | @@ -170,6 +173,55 @@ spec: name: remote-oci-auth ``` +## NetbootEndpoint + +Cluster-scoped `NetbootEndpoint` resources declare stable client-facing +netboot addresses. A Machine references one by name through +`spec.host.netboot.endpointRef`. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `spec.siteRef` | string | Yes | Site whose Machines may use the endpoint. | +| `spec.type` | string | Yes | `ManagedL2`, `ExternalL2`, or `HTTP`. | +| `spec.externalURL` | string | Yes | Stable HTTP(S) base URL snapshotted into each session. | +| `spec.tls.trust` | string | Yes | `TrustedLAN` or `Public`. Public endpoints require HTTPS. | +| `spec.tls.mode` | string | Yes | `Disabled`, `Secret`, or `External`. | +| `spec.tls.secretRef` | Secret reference | For Secret mode | Namespaced TLS Secret copied to the managed edge. | +| `spec.managedL2.nodeSelector` | LabelSelector | For ManagedL2 | Selects nodes attached to the provisioning network. | +| `spec.managedL2.interface` | string | For ManagedL2 | Host interface used for DHCP and TFTP. | +| `spec.managedL2.address` | string | For ManagedL2 | Stable edge address on the provisioning network. | +| `spec.http.serviceType` | string | For HTTP | `ClusterIP`, `NodePort`, or `LoadBalancer`; defaults to `ClusterIP`. | + +`ManagedL2` creates a host-network edge. `HTTP` creates replicated HTTP edge +pods and a Service. `ExternalL2` creates no in-cluster workload; an external +process claims it through status. + +Status records the processed generation, current external claimant and renewal +time, and conditions. A session becomes Ready only after endpoint +`status.observedGeneration` matches its generation and `Ready=True`. + +## NetbootSession + +Cluster-scoped `NetbootSession` is the immutable provisioning contract for one +MachineOperation target. Metalman creates it automatically; users should not +edit or reuse sessions. + +| Field | Description | +|-------|-------------| +| `spec.machine` | Exact Machine name, UID, and generation. | +| `spec.operation` | Exact MachineOperation name, UID, and generation. | +| `spec.endpoint` | Endpoint name, UID, and external URL snapshot. | +| `spec.boot` | Transport, configuration source, network mode, firmware artifact, architecture, leases, and target disk. | +| `spec.provisioning` | Cluster, Kubernetes, agent, provider-label, and resolved cloud-init inputs used for rendering. | +| `spec.artifacts` | Machine/netboot OCI references pinned to SHA-256 digests and the allowed public file names. | +| `spec.expiresAt` | Last time authenticated session requests are accepted. | + +Session spec is immutable through CRD validation. Status contains the +preparation phase, signing-key identifier, endpoint readiness, and exact-target +milestones such as `BootLoaderDownloaded`, `BootImageWritten`, +`CloudInitDone`, and `Attested`. The bearer capability itself is never stored +in Kubernetes. + ## MachineOperation | Property | Value | @@ -193,8 +245,8 @@ spec: | `status.message` | string | No | Human-readable status message. | | `status.startedAt` | time | No | Operation start timestamp. | | `status.completedAt` | time | No | Terminal phase timestamp. | -| `status.targets` | []TargetStatus | No | Per-Machine target status snapshot used by host operation controllers. | -| `status.conditions` | []Condition | No | Operation conditions. `Completed` tracks terminal state. `BootLoaderDownloaded=True` is latched by metalman when a target first downloads the initial PXE boot loader, usually over TFTP. `BootImageWritten` starts as `Unknown` for metalman `HostReplace`, transitions to `False` when the PXE installer requests `disk.img.gz`, and transitions to `True` when the existing `/pxe/disable` completion signal is received. `CloudInitDone` starts as `Unknown`, transitions to `False` when first-boot cloud-init starts, and transitions to `True` on final cloud-init success or `False` with reason `Failed` and a summarized error when cloud-init reports a failure. | +| `status.targets` | []TargetStatus | No | Per-Machine target status, immutable input, session reference, attempts, and target-scoped conditions. Metalman records `BootLoaderDownloaded`, `BootImageWritten`, and `CloudInitDone` on the exact target. | +| `status.conditions` | []Condition | No | Operation-wide conditions such as terminal `Completed`. Provisioning milestones are target-scoped. | `AgentUpgrade` is handled by the in-host agent and requires `spec.parameters.downloadURL`. The URL must point to an `unbounded-agent` release tarball; the agent stages it as the inactive blue/green daemon binary, records the previous binary as last known good, and restarts `unbounded-agent-daemon.service`. If systemd cannot keep the upgraded daemon running, `unbounded-agent-daemon-recovery.service` switches the daemon back to the last known good binary. @@ -456,7 +508,7 @@ Both images are standard OCI container images built `FROM scratch` with artifact under `/disk/`. This follows the kubevirt containerDisk convention. Files with a `.tmpl` suffix in the netboot image are Go templates rendered -per-machine at serve time; other files are served verbatim. A `metadata.yaml` +from the immutable NetbootSession snapshot; other files are served verbatim. A `metadata.yaml` file in the netboot image provides image-level configuration such as `dhcpBootImageName` and `httpBootPath`. @@ -470,10 +522,11 @@ Templates receive the following data object: | Field | Type | Description | |-------|------|-------------| -| `.Machine` | *Machine | The Machine CR that initiated the request. | -| `.BootLease` | *DHCPLease | The DHCP lease matching the request source IP, or the first lease when no match is available. Netboot templates use this to pass the provisioning NIC MAC, static IP, gateway, and DNS to the installer and NoCloud network configuration. | +| `.Machine` | *Machine | A synthetic Machine built from the immutable session snapshot. | +| `.BootLease` | *DHCPLease | The snapshotted provisioning lease selected for rendering. | | `.ApiserverURL` | string | External Kubernetes API server URL. | -| `.ServeURL` | string | External metalman HTTP URL. | +| `.ArtifactBaseURL` | string | Capability-scoped session artifact URL. | +| `.ServeURL` | string | Capability-scoped session base URL used by callbacks. | | `.KubernetesVersion` | string | Resolved Kubernetes version for the machine. | | `.ClusterDNS` | string | Cluster DNS service IP. | @@ -505,11 +558,10 @@ httpBootPath: shimx64.efi ``` The `dhcpBootImageName` field specifies the boot filename included in DHCP -responses (option 67) for `spec.host.netboot.bootProtocol: PXE`. +responses for `transport: TFTP`. -The `httpBootPath` field specifies the file path, relative to metalman's HTTP -artifact server, used for `spec.host.netboot.bootProtocol: HTTP`. If `httpBootPath` is -omitted, metalman falls back to `dhcpBootImageName` for the UEFI HTTP boot URL. +The `httpBootPath` field specifies the firmware artifact used for +`transport: HTTP`. Metalman signs a session capability URL for that artifact. --- diff --git a/e2e/operator/reaper_e2e_test.go b/e2e/operator/reaper_e2e_test.go index 975e95986..b9102f60c 100644 --- a/e2e/operator/reaper_e2e_test.go +++ b/e2e/operator/reaper_e2e_test.go @@ -462,8 +462,8 @@ func assertTranslatedSites(ctx context.Context, t *testing.T, cli client.Client, t.Fatalf("expected metalman enabled on edge site") } - if !nestedBool(edge, "spec", "components", "metalman", "dhcpAutoInterface") { - t.Fatalf("expected Metalman DHCP auto-interface mode preserved") + if _, found, err := unstructured.NestedBool(edge.Object, "spec", "components", "metalman", "dhcpAutoInterface"); err != nil || found { + t.Fatalf("removed Metalman DHCP auto-interface mode retained: found=%t err=%v", found, err) } repair := getMachinaSite(ctx, t, cli, fixture.siteName) diff --git a/hack/metalman-redfish-fixture.py b/hack/metalman-redfish-fixture.py index 35dc6f670..94c661e94 100644 --- a/hack/metalman-redfish-fixture.py +++ b/hack/metalman-redfish-fixture.py @@ -45,7 +45,6 @@ def __init__(self, args: argparse.Namespace) -> None: self.efi_source = Path(args.efi_source) if args.efi_source else None self.efi_active = Path(args.efi_active) if args.efi_active else None self.bridge = args.bridge - self.cache_dir = Path(args.cache_dir) if args.cache_dir else None self.manage_boot_order = args.manage_boot_order self.username = args.username self.password = args.password @@ -113,7 +112,7 @@ def power_state(self) -> str: return "On" if result.returncode == 0 and "running" in result.stdout else "Off" def set_efi_boundary(self, enabled: bool) -> None: - if not all((self.efi_source, self.efi_active, self.bridge, self.cache_dir)): + if not all((self.efi_source, self.efi_active, self.bridge)): raise ValueError("UefiHttp requested without HTTP boundary arguments") assert self.efi_source is not None @@ -127,14 +126,8 @@ def set_efi_boundary(self, enabled: bool) -> None: def stage(client_ip: str) -> None: with subprocess.Popen(["mktemp", "-d"], stdout=subprocess.PIPE, text=True) as proc: artifact_dir = Path(proc.communicate()[0].strip()) - entrypoint = boot_url.rsplit("/", 1)[-1] - candidates = list(self.cache_dir.glob(f"oci/*/amd64/disk/{entrypoint}")) - if len(candidates) != 1: - raise ValueError( - f"expected one cached HTTP entrypoint {entrypoint}, found {len(candidates)}" - ) - shutil.copyfile(candidates[0], artifact_dir / "http-entrypoint.efi") for path, url in ( + ("http-entrypoint.efi", boot_url), ("grubx64.efi", f"{base_url}/grubx64.efi"), ("vmlinuz", f"{base_url}/vmlinuz"), ("initrd", f"{base_url}/initrd"), @@ -357,15 +350,14 @@ def main() -> None: parser.add_argument("--efi-source") parser.add_argument("--efi-active") parser.add_argument("--bridge") - parser.add_argument("--cache-dir") parser.add_argument("--username", default="") parser.add_argument("--password", default="") parser.add_argument("--manage-boot-order", action="store_true") args = parser.parse_args() - boundary_values = (args.efi_source, args.efi_active, args.bridge, args.cache_dir) + boundary_values = (args.efi_source, args.efi_active, args.bridge) if any(boundary_values) and not all(boundary_values): - parser.error("--efi-source, --efi-active, --bridge, and --cache-dir must be used together") + parser.error("--efi-source, --efi-active, and --bridge must be used together") state = State(args) state.record.parent.mkdir(parents=True, exist_ok=True) diff --git a/hack/smoke-metalman-http.py b/hack/smoke-metalman-http.py index 01cd2ca3c..e17df3f9f 100644 --- a/hack/smoke-metalman-http.py +++ b/hack/smoke-metalman-http.py @@ -6,9 +6,9 @@ Stock Noble OVMF and sushy cannot emulate firmware-native DHCP-free UEFI HTTP. The VM therefore starts at the post-firmware EFI boundary: after Metalman has written standard Redfish static IPv4 and UefiHttp settings, the recording BMC -fetches the real Metalman boot artifacts with the VM's source IP and exposes -them on an EFI disk. From shim/GRUB onward the real kernel, installer initrd, -machine image, cloud-init, and branch-built agent path runs unchanged. +fetches the session capability URLs and exposes them on an EFI disk. From +shim/GRUB onward the real kernel, installer initrd, machine image, cloud-init, +and branch-built agent path runs unchanged. """ from __future__ import annotations @@ -49,13 +49,13 @@ AGENT_PORT = 8883 REDFISH_PORT = 8444 REGISTRY_PORT = 5556 -DHCP_PORT = 6768 REGISTRY = "unbounded-http-smoke-registry" HOST_IMAGE = "localhost:5556/unbounded/host-ubuntu2404:http-smoke" NETBOOT_IMAGE = "localhost:5556/unbounded/netboot:http-smoke" AGENT_IMAGE = "localhost:5556/unbounded/agent-ubuntu2404:http-smoke" +HOST_IMAGE_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/host-ubuntu2404:http-smoke" +NETBOOT_IMAGE_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/netboot:http-smoke" AGENT_IMAGE_VM = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/agent-ubuntu2404:http-smoke" -SERVE_URL = f"http://{SERVER_IP}:{HTTP_PORT}" RECORD = TMP / "redfish.jsonl" PCAP = TMP / "traffic.pcap" VIRSH = ["virsh", "--connect", "qemu:///system"] @@ -209,7 +209,11 @@ def create_vm() -> tuple[Path, Path]: def setup_kubernetes_and_images() -> None: log("Building binaries and applying Metalman RBAC/CRDs") run(["make", "machina-manifests"], cwd=ROOT) - for output, package in (("metalman", "./cmd/metalman"), ("unbounded-agent", "./cmd/agent")): + for output, package in ( + ("metalman", "./cmd/metalman"), + ("unbounded-agent", "./cmd/agent"), + ("kubectl-unbounded", "./cmd/kubectl-unbounded"), + ): run(["go", "build", "-o", str(ROOT / "bin" / output), package], cwd=ROOT) run(["kubectl", "apply", "--server-side", "--force-conflicts", "-f", str(ROOT / "deploy/machina/rendered/01-namespace.yaml")]) run(["kubectl", "apply", "--server-side", "--force-conflicts", "-f", str(ROOT / "deploy/machina/crd")]) @@ -263,51 +267,53 @@ def setup_kubernetes_and_images() -> None: def start_fixture(blank_efi: Path, active_efi: Path) -> None: run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", - "-subj", "/CN=metalman-http-fixture", "-addext", "subjectAltName=IP:127.0.0.1", + "-subj", "/CN=metalman-http-fixture", "-addext", f"subjectAltName=IP:{SERVER_IP}", "-keyout", str(TMP / "redfish.key"), "-out", str(TMP / "redfish.crt")]) spawn([sys.executable, str(ROOT / "hack/metalman-redfish-fixture.py"), - "--domain", VM, "--mac", MAC, "--port", str(REDFISH_PORT), - "--cert", str(TMP / "redfish.crt"), "--key", str(TMP / "redfish.key"), - "--record", str(RECORD), "--efi-source", str(blank_efi), - "--efi-active", str(active_efi), "--bridge", BRIDGE, - "--cache-dir", str(TMP / "cache"), "--username", "smoke", "--password", "smoke"], - "redfish.log") + "--domain", VM, "--mac", MAC, "--bind", SERVER_IP, "--port", str(REDFISH_PORT), + "--cert", str(TMP / "redfish.crt"), "--key", str(TMP / "redfish.key"), + "--record", str(RECORD), "--efi-source", str(blank_efi), + "--efi-active", str(active_efi), "--bridge", BRIDGE, + "--username", "smoke", "--password", "smoke"], + "redfish.log") time.sleep(1) def start_metalman_and_replace() -> None: - kubeconfig = TMP / "metalman.kubeconfig" - smoke.write_service_account_kubeconfig(smoke.METALMAN_NAMESPACE, "metalman-controller", kubeconfig) # The guest cannot resolve Docker's kind-control-plane hostname. Use the # control-plane address attached directly to this test's L2 network. api_url = f"https://{KIND_IP}:6443" - spawn(["sudo", "env", f"PATH={os.environ['PATH']}", f"KUBECONFIG={kubeconfig}", - f"METALMAN_APISERVER_URL={api_url}", str(ROOT / "bin/metalman"), "serve-pxe", - f"--site={SITE}", f"--bind-address={SERVER_IP}", f"--serve-url={SERVE_URL}", - f"--http-port={HTTP_PORT}", - f"--dhcp-port={DHCP_PORT}", - f"--cache-dir={TMP / 'cache'}", f"--default-netboot-image={NETBOOT_IMAGE}"], - "metalman.log") + smoke.deploy_split_metalman( + SITE, api_url, smoke.METALMAN_IMAGE, NETBOOT_IMAGE_CLUSTER, + ) machine = { "apiVersion": "unbounded-cloud.io/v1alpha3", "kind": "Machine", "metadata": {"name": NODE, "labels": {"unbounded-cloud.io/site": SITE}}, "spec": { - "pxe": { - "image": HOST_IMAGE, "netbootImage": NETBOOT_IMAGE, "bootProtocol": "HTTP", + "host": {"netboot": { + "image": HOST_IMAGE_CLUSTER, + "netbootImage": NETBOOT_IMAGE_CLUSTER, + "transport": "HTTP", + "configurationSource": "Redfish", + "networkMode": "Static", + "endpointRef": "bootstrap-pending", "targetDisk": "/dev/vda", "dhcpLeases": [{"mac": MAC, "ipv4": NODE_IP, "subnetMask": "255.255.255.0", "gateway": SERVER_IP, "dns": ["8.8.8.8"]}], - "redfish": {"url": f"https://127.0.0.1:{REDFISH_PORT}", "username": "smoke", + "redfish": {"url": f"https://{SERVER_IP}:{REDFISH_PORT}", "username": "smoke", "deviceID": VM, "passwordRef": {"name": "http-bmc-pass", "namespace": "default", "key": "password"}}, "cloudInit": {"userDataConfigMapRef": {"name": "http-smoke-user-data", "namespace": "default", "key": "user-data"}}, - }, + }}, "agent": {"image": AGENT_IMAGE_VM, "url": f"http://{SERVER_IP}:{AGENT_PORT}/unbounded-agent-linux-amd64.tar.gz"}, }, } run(["kubectl", "apply", "-f", "-"], input=json.dumps(machine), text=True) + bootstrap = smoke.start_bootstrap_netboot( + SITE, NODE, BRIDGE, SERVER_IP, HTTP_PORT, TMP / "bootstrap-netboot.log", + ) for _ in range(120): fingerprint = subprocess.run( ["kubectl", "get", "machine", NODE, "-o", "jsonpath={.status.redfish.certFingerprint}"], @@ -319,28 +325,11 @@ def start_metalman_and_replace() -> None: else: raise RuntimeError("Redfish certificate fingerprint was not recorded") - log("Waiting for host and netboot OCI images to be cached") - cache_dir = TMP / "cache" / "oci" - metalman_log = TMP / "metalman.log" - for _ in range(300): - disk_dirs = list(cache_dir.glob("*/amd64/disk")) - host_ready = any((path / "disk.img.gz").is_file() for path in disk_dirs) - netboot_ready = any((path / "bootx64.efi").is_file() for path in disk_dirs) - log_text = metalman_log.read_text(encoding="utf-8") if metalman_log.exists() else "" - host_published = any("OCI image cached" in line and f"image={HOST_IMAGE}" in line - for line in log_text.splitlines()) - netboot_published = any("OCI image cached" in line and f"image={NETBOOT_IMAGE}" in line - for line in log_text.splitlines()) - if host_ready and netboot_ready and host_published and netboot_published: - break - time.sleep(1) - else: - raise RuntimeError("host and netboot OCI images were not cached within 5 minutes") - operation = smoke.create_machine_operation( "http-smoke-host-replace", "HostReplace", machine_ref=NODE ) smoke.wait_machine_operation_complete(operation, timeout=1800) + smoke.wait_process_success(bootstrap, timeout=120) def fixture_writes() -> list[dict[str, Any]]: @@ -359,12 +348,15 @@ def assert_contract() -> None: boot_patches = [entry["body"].get("Boot", {}) for entry in writes if entry["method"] == "PATCH" and entry["path"] == f"/redfish/v1/Systems/{VM}"] if not any(boot.get("BootSourceOverrideTarget") == "UefiHttp" - and boot.get("BootSourceOverrideEnabled") == "Continuous" + and boot.get("BootSourceOverrideEnabled") == "Once" and boot.get("BootSourceOverrideMode") == "UEFI" - and boot.get("HttpBootUri") == f"{SERVE_URL}/bootx64.efi" for boot in boot_patches): + and "/v1/netboot/sessions/" in boot.get("HttpBootUri", "") + and boot.get("HttpBootUri", "").endswith("/artifacts/bootx64.efi") + for boot in boot_patches): raise AssertionError(f"standard UefiHttp PATCH not recorded: {boot_patches}") firmware_fetches = [entry for entry in writes if entry["method"] == "FIRMWARE_FETCH"] - if not any(entry["path"] == f"{SERVE_URL}/bootx64.efi" + if not any("/v1/netboot/sessions/" in entry["path"] + and entry["path"].endswith("/artifacts/bootx64.efi") and entry["body"] == {"source": NODE_IP} for entry in firmware_fetches): raise AssertionError(f"state-derived post-power-on firmware fetch not recorded: {firmware_fetches}") diff --git a/hack/smoke-metalman.py b/hack/smoke-metalman.py index f05eb3442..7166cbf00 100755 --- a/hack/smoke-metalman.py +++ b/hack/smoke-metalman.py @@ -8,6 +8,7 @@ import base64 import json import os +import secrets import signal import shutil import socket @@ -32,7 +33,7 @@ NODE_LABEL_KEY = "unbounded-cloud.io/smoke-test" NODE_LABEL_VALUE = "metalman" METALMAN_NAMESPACE = "unbounded-system" -METALMAN_CONTROLLER_SA = "metalman-controller" +METALMAN_IMAGE = "unbounded/metalman:smoke" VM_NAME = "unbounded-metal-smoke" NET_NAME = "unbounded-metal-smoke" SUBNET = "192.168.200" @@ -45,9 +46,7 @@ REDFISH_PORT = 8443 HTTP_PORT = 8880 AGENT_DOWNLOAD_PORT = 8881 -CACHE_DIR = TMPDIR / "cache" ARTIFACT_DIR = TMPDIR / "artifacts" -SERVE_URL = f"http://{SERVER_IP}:{HTTP_PORT}" AGENT_TARBALL = ARTIFACT_DIR / "unbounded-agent-linux-amd64.tar.gz" AGENT_DOWNLOAD_URL = f"http://{SERVER_IP}:{AGENT_DOWNLOAD_PORT}/{AGENT_TARBALL.name}" REGISTRY_PORT = 5555 @@ -55,6 +54,8 @@ IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/host-ubuntu2404:smoke" NETBOOT_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/netboot:smoke" AGENT_IMAGE_NAME = f"localhost:{REGISTRY_PORT}/unbounded/agent-ubuntu2404:smoke" +IMAGE_NAME_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/host-ubuntu2404:smoke" +NETBOOT_IMAGE_NAME_CLUSTER = f"{SERVER_IP}:{REGISTRY_PORT}/unbounded/netboot:smoke" # The agent runs inside a VM on an isolated libvirt network. "localhost" inside # the VM resolves to the VM's own loopback, not the host. Use the host's # bridge IP so the VM can reach the registry over the virtual network. @@ -355,7 +356,7 @@ def clean_libvirt() -> None: run_quiet(["sudo", "ip", "link", "delete", "veth-kind-smoke"]) # Kill any leftover Redfish fixture from a previous run. run_quiet(["sudo", "pkill", "-f", "metalman-redfish-fixture.py"]) - # Kill any leftover metalman serve-pxe from a previous run. + # Kill any leftover local Metalman edge from a previous run. # Use the binary path to avoid matching this script (smoke-metalman.py). run_quiet(["sudo", "pkill", "-f", "bin/metalman"]) # Kill any leftover artifact download server from a previous run. @@ -478,6 +479,181 @@ def apiserver_url() -> str: return url +def deploy_split_metalman(site: str, api_url: str, metalman_image: str, + default_netboot_image: str) -> None: + """Deploy the controller/server roles used by the operator into kind.""" + inspect = subprocess.run( + ["docker", "image", "inspect", metalman_image], + stdout=DEVNULL, stderr=DEVNULL, + ) + if inspect.returncode != 0: + run([ + "docker", "build", "-t", metalman_image, + "-f", str(REPO_ROOT / "images/metalman/Containerfile"), str(REPO_ROOT), + ]) + run(["kind", "load", "docker-image", metalman_image, "--name", "kind"]) + + site_resource = { + "apiVersion": API_VERSION, + "kind": "Site", + "metadata": {"name": site}, + "spec": { + "nodeCidrs": ["192.168.200.0/24"], + "podCidrAssignments": [{"cidrBlocks": ["10.250.0.0/16"]}], + "components": {"metalman": {"enabled": True}}, + }, + } + kubectl(["apply", "-f", "-"], input=json.dumps(site_resource).encode(), stdout=DEVNULL) + + capability = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": f"metalman-capability-{site}", "namespace": METALMAN_NAMESPACE}, + "stringData": {"capability.key": secrets.token_hex(32)}, + } + kubectl(["apply", "-f", "-"], input=json.dumps(capability).encode(), stdout=DEVNULL) + + labels = { + "app": "unbounded-metalman", + f"{API_GROUP}/site": site, + } + for role, replicas in (("controller", 1), ("server", 2)): + role_labels = { + **labels, + "app.kubernetes.io/name": f"metalman-{role}", + "app.kubernetes.io/component": role, + } + ports = [{"name": "health", "containerPort": 8081}] + if role == "server": + ports.append({"name": "http", "containerPort": 8880}) + args = [ + role, + f"--site={site}", + "--cache-dir=/var/cache/metalman", + f"--default-netboot-image={default_netboot_image}", + ] + if role == "controller": + args.extend([ + "--leader-elect-lease-duration=60s", + "--leader-elect-renew-deadline=40s", + "--leader-elect-retry-period=5s", + ]) + deployment = { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": f"metalman-{role}-{site}", "namespace": METALMAN_NAMESPACE}, + "spec": { + "replicas": replicas, + "selector": {"matchLabels": role_labels}, + "template": { + "metadata": {"labels": role_labels}, + "spec": { + "serviceAccountName": f"metalman-{role}", + "containers": [{ + "name": "metalman", + "image": metalman_image, + "imagePullPolicy": "IfNotPresent", + "args": args, + "env": [ + {"name": "POD_NAMESPACE", "value": METALMAN_NAMESPACE}, + {"name": "METALMAN_APISERVER_URL", "value": api_url}, + ], + "ports": ports, + "volumeMounts": [ + {"name": "cache", "mountPath": "/var/cache/metalman"}, + {"name": "capability", "mountPath": "/var/run/secrets/metalman", "readOnly": True}, + ], + }], + "volumes": [ + {"name": "cache", "emptyDir": {}}, + {"name": "capability", "secret": {"secretName": f"metalman-capability-{site}"}}, + ], + }, + }, + }, + } + kubectl(["apply", "-f", "-"], input=json.dumps(deployment).encode(), stdout=DEVNULL) + + service = { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": f"metalman-server-{site}", "namespace": METALMAN_NAMESPACE}, + "spec": { + "selector": { + **labels, + "app.kubernetes.io/name": "metalman-server", + "app.kubernetes.io/component": "server", + }, + "ports": [{"name": "http", "port": 8880, "targetPort": "http"}], + }, + } + kubectl(["apply", "-f", "-"], input=json.dumps(service).encode(), stdout=DEVNULL) + for role in ("controller", "server"): + kubectl([ + "-n", METALMAN_NAMESPACE, "rollout", "status", + f"deployment/metalman-{role}-{site}", "--timeout=5m", + ]) + + +def start_bootstrap_netboot(site: str, machine: str, interface: str, address: str, + http_port: int, log_path: Path) -> subprocess.Popen[Any]: + process = spawn([ + str(KUBECTL_UNBOUNDED), "site", "bootstrap-netboot", site, + f"--machine={machine}", f"--interface={interface}", f"--address={address}", + f"--http-port={http_port}", f"--metalman-binary={BINARY}", "--timeout=30m", + ], log_path) + endpoint = f"bootstrap-{machine}" + for _ in range(180): + result = subprocess.run( + [KUBECTL, "get", "netbootendpoint", endpoint, "-o", "json"], + capture_output=True, text=True, + ) + if result.returncode == 0: + resource = json.loads(result.stdout) + ready = any( + condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in resource.get("status", {}).get("conditions", []) + ) + if ready: + return process + if process.poll() is not None: + die(f"bootstrap-netboot exited before endpoint {endpoint} became Ready") + time.sleep(1) + die(f"Timed out waiting for bootstrap endpoint {endpoint}") + + +def delete_metalman_server_during_provisioning(site: str) -> None: + """Delete one serving replica after a session exists to exercise edge reconnect.""" + for _ in range(300): + result = subprocess.run( + [KUBECTL, "get", "netbootsessions", "-o", "json"], + capture_output=True, text=True, + ) + if result.returncode == 0 and json.loads(result.stdout).get("items"): + break + time.sleep(1) + else: + die("Timed out waiting for a NetbootSession before server disruption") + pods = json.loads(run( + [KUBECTL, "-n", METALMAN_NAMESPACE, "get", "pod", + "-l", f"app.kubernetes.io/name=metalman-server,{API_GROUP}/site={site}", + "-o", "json"], + capture_output=True, text=True, + ).stdout).get("items", []) + ready_pods = sorted( + pod["metadata"]["name"] for pod in pods + if not pod["metadata"].get("deletionTimestamp") and any( + condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in pod.get("status", {}).get("conditions", []) + ) + ) + if not ready_pods: + die("No Metalman server pod available for disruption") + pod = ready_pods[0] + log(f"Deleting Metalman server pod {pod} during provisioning") + kubectl(["-n", METALMAN_NAMESPACE, "delete", "pod", pod, "--wait=false"]) + + def configure_kind_control_plane_node_ip(container: str, node_ip: str) -> None: """Make the kind control-plane Node advertise its VM-reachable IP.""" log(f"Configuring {container} kubelet node IP as {node_ip}") @@ -829,7 +1005,12 @@ def assert_cloud_init_done(timeout: int = 900) -> None: op_status = op.get("status", {}) phase = op_status.get("phase", "") message = op_status.get("message", "") - for c in op_status.get("conditions", []): + target = next( + (item for item in op_status.get("targets", []) + if item.get("machineRef") == NODE_NAME), + {}, + ) + for c in target.get("conditions", []): if c.get("type") == "CloudInitDone": status = c.get("status", "") reason = c.get("reason", "") @@ -1082,13 +1263,13 @@ def main() -> None: "-out", str(TMPDIR / "redfish.crt"), "-days", "1", "-nodes", "-subj", "/CN=metalman-redfish-fixture", - "-addext", "subjectAltName=IP:127.0.0.1", + "-addext", f"subjectAltName=IP:{SERVER_IP}", ], check=True) - redfish_url = f"https://127.0.0.1:{REDFISH_PORT}" + redfish_url = f"https://{SERVER_IP}:{REDFISH_PORT}" proc = spawn([ sys.executable, str(REPO_ROOT / "hack" / "metalman-redfish-fixture.py"), "--domain", VM_NAME, "--mac", MAC_ADDRESS, - "--bind", "127.0.0.1", "--port", str(REDFISH_PORT), + "--bind", SERVER_IP, "--port", str(REDFISH_PORT), "--cert", str(TMPDIR / "redfish.crt"), "--key", str(TMPDIR / "redfish.key"), "--record", str(TMPDIR / "redfish.jsonl"), @@ -1210,6 +1391,8 @@ def main() -> None: server_url = apiserver_url() log(f" API server URL: {server_url}") + deploy_split_metalman(SITE, server_url, METALMAN_IMAGE, NETBOOT_IMAGE_NAME_CLUSTER) + protonode = { "apiVersion": API_VERSION, "kind": "Machine", @@ -1218,21 +1401,28 @@ def main() -> None: "labels": {f"{API_GROUP}/site": SITE}, }, "spec": { - "pxe": { - "image": IMAGE_NAME, - "redfish": { - "url": redfish_url, - "username": "", - "deviceID": VM_NAME, - "passwordRef": {"name": "bmc-pass", "key": "password", "namespace": NODE_NS}, + "host": { + "netboot": { + "image": IMAGE_NAME_CLUSTER, + "netbootImage": NETBOOT_IMAGE_NAME_CLUSTER, + "transport": "TFTP", + "configurationSource": "DHCP", + "networkMode": "DHCP", + "endpointRef": "bootstrap-pending", + "redfish": { + "url": redfish_url, + "username": "", + "deviceID": VM_NAME, + "passwordRef": {"name": "bmc-pass", "key": "password", "namespace": NODE_NS}, + }, + "dhcpLeases": [{ + "mac": MAC_ADDRESS, + "ipv4": NODE_IP, + "subnetMask": "255.255.255.0", + "gateway": GATEWAY, + "dns": [DNS_SERVER], + }], }, - "dhcpLeases": [{ - "mac": MAC_ADDRESS, - "ipv4": NODE_IP, - "subnetMask": "255.255.255.0", - "gateway": GATEWAY, - "dns": [DNS_SERVER], - }], }, "agent": { "image": AGENT_IMAGE_NAME_VM, @@ -1247,26 +1437,11 @@ def main() -> None: stdout=DEVNULL) log(" Resources created") - log("Starting metalman serve-pxe") - metalman_kubeconfig = TMPDIR / "metalman-controller.kubeconfig" - write_service_account_kubeconfig(METALMAN_NAMESPACE, METALMAN_CONTROLLER_SA, metalman_kubeconfig) - metalman_env = [f"METALMAN_APISERVER_URL={server_url}"] - metalman_env.append(f"KUBECONFIG={metalman_kubeconfig}") - - proc = spawn([ - "sudo", "env", *metalman_env, - str(BINARY), "serve-pxe", f"--site={SITE}", f"--bind-address={SERVER_IP}", - f"--cache-dir={CACHE_DIR}", - f"--serve-url={SERVE_URL}", "--dhcp-interface=virbr-smoke", - f"--default-netboot-image={NETBOOT_IMAGE_NAME}", - "--leader-elect-lease-duration=60s", - "--leader-elect-renew-deadline=40s", - "--leader-elect-retry-period=5s", - ], TMPDIR / "serve.log") - log(f" serve PID={proc.pid}") - - time.sleep(2) - check_procs() + log("Starting local Metalman edge through kubectl-unbounded") + bootstrap_proc = start_bootstrap_netboot( + SITE, NODE_NAME, "virbr-smoke", SERVER_IP, HTTP_PORT, + TMPDIR / "bootstrap-netboot.log", + ) log("Triggering HostReplace through kubectl-unbounded") operation_log = TMPDIR / "kubectl-host-replace.log" @@ -1274,6 +1449,7 @@ def main() -> None: ["replace", NODE_NAME, "--force", "--ttl=3600"], operation_log.name, ) + delete_metalman_server_during_provisioning(SITE) # Log free space so we can correlate disk exhaustion with VM failures. df = subprocess.run(["df", "-h", str(TMPDIR)], capture_output=True, text=True) @@ -1289,6 +1465,7 @@ def main() -> None: wait_k8s_node(NODE_NAME, timeout=900) assert_node_ready(NODE_NAME, timeout=720) assert_node_label(NODE_NAME, NODE_LABEL_KEY, NODE_LABEL_VALUE) + wait_process_success(bootstrap_proc, timeout=120) run_operation_smoke_suite() diff --git a/hack/smoke_metalman_contract_test.go b/hack/smoke_metalman_contract_test.go new file mode 100644 index 000000000..38c17c134 --- /dev/null +++ b/hack/smoke_metalman_contract_test.go @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package hack + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestMetalmanSmokeSuitesUseSplitRuntime(t *testing.T) { + t.Parallel() + + for _, name := range []string{"smoke-metalman.py", "smoke-metalman-http.py"} { + name := name + t.Run(name, func(t *testing.T) { + t.Parallel() + + contents, err := os.ReadFile(filepath.Join(".", name)) + if err != nil { + t.Fatal(err) + } + + text := string(contents) + for _, stale := range []string{"serve-pxe", `"bootProtocol"`} { + if strings.Contains(text, stale) { + t.Errorf("%s still contains legacy %q wiring", name, stale) + } + } + + for _, required := range []string{"deploy_split_metalman", "bootstrap-netboot"} { + if !strings.Contains(text, required) { + t.Errorf("%s does not exercise %q", name, required) + } + } + }) + } +} + +func TestTraditionalMetalmanSmokeDisruptsServerDuringProvisioning(t *testing.T) { + t.Parallel() + + contents, err := os.ReadFile(filepath.Join(".", "smoke-metalman.py")) + if err != nil { + t.Fatal(err) + } + + if !strings.Contains(string(contents), "delete_metalman_server_during_provisioning") { + t.Fatal("traditional smoke does not disrupt a server pod during provisioning") + } +} + +func TestHTTPFixtureFetchesCapabilityEntrypoint(t *testing.T) { + t.Parallel() + + contents, err := os.ReadFile(filepath.Join(".", "metalman-redfish-fixture.py")) + if err != nil { + t.Fatal(err) + } + + text := string(contents) + if strings.Contains(text, "cache_dir.glob") { + t.Fatal("HTTP fixture still reads a process-local Metalman cache") + } + + if !strings.Contains(text, `("http-entrypoint.efi", boot_url)`) { + t.Fatal("HTTP fixture does not fetch the capability-scoped firmware URL") + } +} diff --git a/images/netboot/assets/grub.cfg.tmpl b/images/netboot/assets/grub.cfg.tmpl index 1af1b9994..d05c33e29 100644 --- a/images/netboot/assets/grub.cfg.tmpl +++ b/images/netboot/assets/grub.cfg.tmpl @@ -27,10 +27,11 @@ set default=0 set timeout=0 menuentry "Unbounded Metal Install" { - linux /vmlinuz \ - unbounded.image_url={{ .ServeURL }}/disk.img.gz \ + linux {{ .ArtifactBaseURL }}/vmlinuz \ + unbounded.image_url={{ .ArtifactBaseURL }}/disk.img.gz \ unbounded.serve_url={{ .ServeURL }} \ - unbounded.ds_url={{ .ServeURL }}/cloud-init/ \ + unbounded.boot_image_written_url={{ .BootImageWrittenURL }} \ + unbounded.ds_url={{ .ArtifactBaseURL }}/cloud-init/ \ unbounded.node_name={{ .Machine.Name }} \ unbounded.node_namespace={{ .Machine.Namespace }} \ unbounded.apiserver_url={{ .ApiserverURL }} \ @@ -46,5 +47,5 @@ menuentry "Unbounded Metal Install" { {{- end }} console=tty0 console=ttyS0,115200n8 \ --- - initrd /initrd /init.cpio + initrd {{ .ArtifactBaseURL }}/initrd {{ .ArtifactBaseURL }}/init.cpio } diff --git a/images/netboot/assets/init b/images/netboot/assets/init index 53829033e..6aafee75a 100644 --- a/images/netboot/assets/init +++ b/images/netboot/assets/init @@ -121,6 +121,7 @@ done IMAGE_URL=$(get_param unbounded.image_url) || fatal "unbounded.image_url not set" SERVE_URL=$(get_param unbounded.serve_url || true) +BOOT_IMAGE_WRITTEN_URL=$(get_param unbounded.boot_image_written_url || true) TARGET_DISK=$(get_param unbounded.disk || true) BOOT_MAC=$(get_param unbounded.boot_mac || true) BOOTIF=$(get_param BOOTIF || true) @@ -416,10 +417,10 @@ if [ -d /sys/firmware/efi ]; then done fi -if [ -n "$SERVE_URL" ]; then - log "disabling PXE boot" - retry 5 2 "disable PXE" wget -q -O /dev/null "$SERVE_URL/pxe/disable" \ - || log "WARNING: failed to disable PXE boot" +if [ -n "$BOOT_IMAGE_WRITTEN_URL" ]; then + log "reporting completed disk installation" + retry 5 2 "report completed disk installation" wget -q -O /dev/null --post-data='' "$BOOT_IMAGE_WRITTEN_URL" \ + || log "WARNING: failed to report completed disk installation" fi log "installation complete, rebooting" diff --git a/images/netboot/assets/vendor-data.tmpl b/images/netboot/assets/vendor-data.tmpl index 94a35ad01..9fb032b3e 100644 --- a/images/netboot/assets/vendor-data.tmpl +++ b/images/netboot/assets/vendor-data.tmpl @@ -16,7 +16,7 @@ ssh_pwauth: false reporting: unbounded: type: webhook - endpoint: {{ .ServeURL }}/cloudinit/log + endpoint: {{ .CloudInitURL }} write_files: - path: /etc/unbounded/agent/config.json permissions: '0600' @@ -67,7 +67,7 @@ runcmd: else echo "could not read ${install_log}: file does not exist" fi - } | curl -fsS -m 10 -X POST -H "Content-Type: text/plain" --data-binary @- "{{ .ServeURL }}/unbounded-agent/install-log" >/dev/null 2>&1 || true + } | curl -fsS -m 10 -X POST -H "Content-Type: text/plain" --data-binary @- "{{ .InstallLogURL }}" >/dev/null 2>&1 || true exit "$status" } trap report_unbounded_failure ERR diff --git a/internal/metalman/attestation/attestation.go b/internal/metalman/attestation/attestation.go index 98f5206ce..27e50708c 100644 --- a/internal/metalman/attestation/attestation.go +++ b/internal/metalman/attestation/attestation.go @@ -324,6 +324,21 @@ func (h *Handler) Attest(w http.ResponseWriter, r *http.Request) { } } +// AttestMachine performs attestation for an already authenticated Machine +// identity instead of deriving identity from the request network path. +func (h *Handler) AttestMachine(w http.ResponseWriter, r *http.Request, machine *v1alpha3.Machine) { + if machine == nil { + http.Error(w, "node not found", http.StatusNotFound) + return + } + + exact := *h + exact.LookupNodeByIP = func(context.Context, string) (*v1alpha3.Machine, error) { + return machine, nil + } + exact.Attest(w, r) +} + // publicKeysEqual compares two crypto.PublicKey values structurally. func publicKeysEqual(a, b crypto.PublicKey) bool { type equaler interface { diff --git a/internal/metalman/attestation/attestation_test.go b/internal/metalman/attestation/attestation_test.go index b1f8eae01..85d19f18a 100644 --- a/internal/metalman/attestation/attestation_test.go +++ b/internal/metalman/attestation/attestation_test.go @@ -342,6 +342,32 @@ func TestAttestTOFUStoresKey(t *testing.T) { } } +func TestAttestMachineDoesNotUseRequestSourceIP(t *testing.T) { + _, ekPub := testEKKeyPair(t) + srkPub, _ := testSRKPub(t) + node := &v1alpha3.Machine{ObjectMeta: metav1.ObjectMeta{Name: "session-machine"}} + handler := testHandler(t, node) + handler.LookupNodeByIP = func(context.Context, string) (*v1alpha3.Machine, error) { + t.Fatal("session attestation must not resolve a Machine by source IP") + return nil, nil + } + + body, err := json.Marshal(AttestRequest{EKPub: ekPub, SRKPub: srkPub}) + if err != nil { + t.Fatal(err) + } + + request := httptest.NewRequest(http.MethodPost, "/session/attest", bytes.NewReader(body)) + request.RemoteAddr = "198.51.100.20:12345" + response := httptest.NewRecorder() + + handler.AttestMachine(response, request, node) + + if response.Code != http.StatusOK { + t.Fatalf("attest exact Machine: expected 200, got %d: %s", response.Code, response.Body.String()) + } +} + func TestAttestTOFURejectsNewKey(t *testing.T) { _, ekPub := testEKKeyPair(t) srkPub, _ := testSRKPub(t) diff --git a/internal/metalman/commands/controller.go b/internal/metalman/commands/controller.go new file mode 100644 index 000000000..1f3085f02 --- /dev/null +++ b/internal/metalman/commands/controller.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import "github.com/spf13/cobra" + +// ControllerCmd runs Metalman's leader-elected Kubernetes control loops. +func ControllerCmd() *cobra.Command { + return newMetalmanRoleCmd(metalmanRoleController, "Run the Metalman control loops") +} diff --git a/internal/metalman/commands/edge.go b/internal/metalman/commands/edge.go new file mode 100644 index 000000000..8695b9b98 --- /dev/null +++ b/internal/metalman/commands/edge.go @@ -0,0 +1,405 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/spf13/cobra" + + "github.com/Azure/unbounded/internal/metalman/dhcp" + "github.com/Azure/unbounded/internal/metalman/netboot" +) + +const defaultHTTPReadHeaderTimeout = 10 * time.Second + +const maxArtifactBackendAttempts = 3 + +// EdgeCmd runs Metalman's provisioning-network protocol edge. +func EdgeCmd() *cobra.Command { + var ( + backendURL string + bindAddress string + httpPort int + tlsCertFile string + tlsKeyFile string + endpoint string + edgeTokenFile string + dhcpEnabled bool + dhcpInterface string + dhcpServerIP string + dhcpPort int + tftpEnabled bool + tftpBindAddr string + tftpPort int + ) + + cmd := &cobra.Command{ + Use: string(metalmanRoleEdge), + Short: "Run the Metalman provisioning protocol edge", + RunE: func(cmd *cobra.Command, _ []string) error { + backend, err := parseEdgeBackendURL(backendURL) + if err != nil { + return err + } + + if err := validateEdgeTLSFiles(tlsCertFile, tlsKeyFile); err != nil { + return err + } + + ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + addr := fmt.Sprintf("%s:%d", bindAddress, httpPort) + + server := &http.Server{ + Addr: addr, + Handler: newEdgeProxy(backend), + ReadHeaderTimeout: defaultHTTPReadHeaderTimeout, + } + if dhcpEnabled { + decisionProvider, err := dhcp.NewHTTPDecisionProviderFromTokenFile(backend.String(), endpoint, edgeTokenFile, nil) + if err != nil { + return fmt.Errorf("creating DHCP backend: %w", err) + } + + serverIP, err := edgeDHCPServerIP(dhcpServerIP, dhcpInterface) + if err != nil { + return err + } + + dhcpServer := &dhcp.Server{Interface: dhcpInterface, Port: dhcpPort, DecisionProvider: decisionProvider, ServerIP: serverIP} + + go func() { + if err := dhcpServer.Start(ctx); err != nil && ctx.Err() == nil { + slog.ErrorContext(ctx, "Metalman edge DHCP server failed", "err", err) + stop() + } + }() + } + + if tftpEnabled { + artifactBackend, err := netboot.NewHTTPArtifactBackend(backend.String(), nil) + if err != nil { + return fmt.Errorf("creating TFTP backend: %w", err) + } + + tftpServer := &netboot.TFTPServer{BindAddr: tftpBindAddr, Port: tftpPort, Backend: artifactBackend} + + go func() { + if err := tftpServer.Start(ctx); err != nil && ctx.Err() == nil { + slog.ErrorContext(ctx, "Metalman edge TFTP server failed", "err", err) + stop() + } + }() + } + + go func() { + <-ctx.Done() + + if err := server.Shutdown(context.Background()); err != nil { + slog.Warn("shutting down Metalman edge HTTP server failed", "err", err) + } + }() + + PrintConfig("role", string(metalmanRoleEdge)) + PrintConfig("backend-url", backend.String()) + PrintService("HTTP", addr) + PrintReady() + slog.InfoContext(ctx, "starting Metalman edge", "addr", addr, "backend", backend.String()) + + if err := serveEdgeHTTP(server, tlsCertFile, tlsKeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("serving edge HTTP: %w", err) + } + + return nil + }, + } + + cmd.Flags().StringVar(&backendURL, "backend-url", "", "Metalman server base URL") + cmd.Flags().StringVar(&bindAddress, "bind-address", "0.0.0.0", "IP address to bind the edge HTTP listener") + cmd.Flags().IntVar(&httpPort, "http-port", 8880, "Port for the edge HTTP listener") + cmd.Flags().StringVar(&tlsCertFile, "tls-cert-file", "", "TLS certificate file; requires --tls-key-file") + cmd.Flags().StringVar(&tlsKeyFile, "tls-key-file", "", "TLS private key file; requires --tls-cert-file") + cmd.Flags().StringVar(&endpoint, "endpoint", "", "NetbootEndpoint served by this edge") + cmd.Flags().StringVar(&edgeTokenFile, "edge-token-file", "/var/run/secrets/metalman/token", "Audience-bound ServiceAccount token file") + cmd.Flags().BoolVar(&dhcpEnabled, "dhcp-enabled", false, "Enable the DHCP protocol edge") + cmd.Flags().StringVar(&dhcpInterface, "dhcp-interface", "", "Provisioning interface for direct DHCP; empty enables relay-only mode") + cmd.Flags().StringVar(&dhcpServerIP, "dhcp-server-ip", "", "DHCP server IPv4 address; defaults to the interface or outbound address") + cmd.Flags().IntVar(&dhcpPort, "dhcp-port", 67, "DHCP listener port") + cmd.Flags().BoolVar(&tftpEnabled, "tftp-enabled", false, "Enable the TFTP protocol edge") + cmd.Flags().StringVar(&tftpBindAddr, "tftp-bind-address", "0.0.0.0", "IP address to bind the TFTP listener") + cmd.Flags().IntVar(&tftpPort, "tftp-port", 69, "TFTP listener port") + + if err := cmd.MarkFlagRequired("backend-url"); err != nil { + panic(fmt.Sprintf("mark backend-url flag required: %v", err)) + } + + if err := cmd.MarkFlagRequired("endpoint"); err != nil { + panic(fmt.Sprintf("mark endpoint flag required: %v", err)) + } + + return cmd +} + +func validateEdgeTLSFiles(certFile, keyFile string) error { + if (certFile == "") != (keyFile == "") { + return errors.New("--tls-cert-file and --tls-key-file must be set together") + } + + return nil +} + +func serveEdgeHTTP(server *http.Server, certFile, keyFile string) error { + if certFile != "" { + return server.ListenAndServeTLS(certFile, keyFile) + } + + return server.ListenAndServe() +} + +func edgeDHCPServerIP(configured, iface string) (net.IP, error) { + if configured != "" { + ip := net.ParseIP(configured).To4() + if ip == nil { + return nil, errors.New("--dhcp-server-ip must be an IPv4 address") + } + + return ip, nil + } + + if iface != "" { + return InterfaceIPv4(iface) + } + + ip, err := OutboundIP() + if err != nil || ip.To4() == nil { + return nil, errors.New("detecting DHCP server IPv4 address; set --dhcp-server-ip") + } + + return ip.To4(), nil +} + +func parseEdgeBackendURL(value string) (*url.URL, error) { + backend, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("parsing --backend-url: %w", err) + } + + if backend.Scheme != "http" && backend.Scheme != "https" { + return nil, errors.New("--backend-url must use http or https") + } + + if backend.Host == "" { + return nil, errors.New("--backend-url must include a host") + } + + return backend, nil +} + +func newEdgeProxy(backend *url.URL) http.Handler { + proxy := httputil.NewSingleHostReverseProxy(backend) + proxy.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) { + slog.Warn("Metalman edge backend request failed", "err", err) + http.Error(w, "Metalman backend unavailable", http.StatusBadGateway) + } + + return &edgeProxy{ + backend: backend, + proxy: proxy, + transport: http.DefaultTransport, + } +} + +type edgeProxy struct { + backend *url.URL + proxy *httputil.ReverseProxy + transport http.RoundTripper +} + +func (e *edgeProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/v1/netboot/sessions/") && strings.Contains(r.URL.Path, "/artifacts/") { + e.serveArtifact(w, r) + return + } + + e.proxy.ServeHTTP(w, r) +} + +func (e *edgeProxy) serveArtifact(w http.ResponseWriter, r *http.Request) { + request := r.Clone(r.Context()) + e.rewriteRequest(request) + + response, err := e.transport.RoundTrip(request) + if err != nil { + slog.Warn("Metalman edge artifact request failed", "err", err) + http.Error(w, "Metalman backend unavailable", http.StatusBadGateway) + + return + } + + copyResponseHeaders(w.Header(), response.Header) + w.WriteHeader(response.StatusCode) + + remaining := response.ContentLength + if remaining <= 0 || (response.StatusCode != http.StatusOK && response.StatusCode != http.StatusPartialContent) { + if _, err := io.Copy(w, response.Body); err != nil { + slog.Warn("Metalman edge response copy failed", "path", r.URL.Path, "err", err) + } + + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + + return + } + + start, end, ok := responseByteRange(response) + if !ok { + if _, err := io.Copy(w, response.Body); err != nil { + slog.Warn("Metalman edge response copy failed", "path", r.URL.Path, "err", err) + } + + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + + return + } + + written, copyErr := io.Copy(w, response.Body) + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + + start += written + + remaining -= written + if copyErr == nil && remaining == 0 { + return + } + + for attempt := 2; attempt <= maxArtifactBackendAttempts && remaining > 0 && start <= end; attempt++ { + request = r.Clone(r.Context()) + e.rewriteRequest(request) + request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + + response, err = e.transport.RoundTrip(request) + if err != nil { + slog.Warn("Metalman edge artifact resume failed", "path", r.URL.Path, "err", err) + continue + } + + if response.StatusCode != http.StatusPartialContent || response.ContentLength != remaining { + response.Body.Close() //nolint:errcheck // Invalid resume response. + slog.Warn("Metalman edge artifact resume returned an invalid range", "path", r.URL.Path, "status", response.StatusCode) + + return + } + + resumeStart, resumeEnd, valid := responseByteRange(response) + if !valid || resumeStart != start || resumeEnd != end { + response.Body.Close() //nolint:errcheck // Invalid resume response. + slog.Warn("Metalman edge artifact resume returned mismatched bytes", "path", r.URL.Path) + + return + } + + written, copyErr = io.Copy(w, response.Body) + response.Body.Close() //nolint:errcheck // The response body is no longer needed. + + start += written + + remaining -= written + if copyErr == nil && remaining == 0 { + return + } + } + + slog.Warn("Metalman edge artifact transfer failed", "path", r.URL.Path, "err", copyErr) +} + +func (e *edgeProxy) rewriteRequest(request *http.Request) { + request.URL.Scheme = e.backend.Scheme + request.URL.Host = e.backend.Host + request.URL.Path, request.URL.RawPath = joinURLPath(e.backend, request.URL) + request.Host = e.backend.Host + + if e.backend.RawQuery == "" || request.URL.RawQuery == "" { + request.URL.RawQuery = e.backend.RawQuery + request.URL.RawQuery + } else { + request.URL.RawQuery = e.backend.RawQuery + "&" + request.URL.RawQuery + } +} + +func joinURLPath(base, request *url.URL) (string, string) { + if base.RawPath == "" && request.RawPath == "" { + return singleJoiningSlash(base.Path, request.Path), "" + } + + basePath := base.EscapedPath() + requestPath := request.EscapedPath() + + return singleJoiningSlash(base.Path, request.Path), singleJoiningSlash(basePath, requestPath) +} + +func singleJoiningSlash(left, right string) string { + leftSlash := strings.HasSuffix(left, "/") + rightSlash := strings.HasPrefix(right, "/") + + switch { + case leftSlash && rightSlash: + return left + right[1:] + case !leftSlash && !rightSlash: + return left + "/" + right + default: + return left + right + } +} + +func responseByteRange(response *http.Response) (int64, int64, bool) { + if response.StatusCode == http.StatusOK { + return 0, response.ContentLength - 1, true + } + + value := response.Header.Get("Content-Range") + if !strings.HasPrefix(value, "bytes ") { + return 0, 0, false + } + + rangeAndSize := strings.SplitN(strings.TrimPrefix(value, "bytes "), "/", 2) + if len(rangeAndSize) != 2 { + return 0, 0, false + } + + bounds := strings.SplitN(rangeAndSize[0], "-", 2) + if len(bounds) != 2 { + return 0, 0, false + } + + start, err := strconv.ParseInt(bounds[0], 10, 64) + if err != nil { + return 0, 0, false + } + + end, err := strconv.ParseInt(bounds[1], 10, 64) + if err != nil || start < 0 || end < start || end-start+1 != response.ContentLength { + return 0, 0, false + } + + return start, end, true +} + +func copyResponseHeaders(dst, src http.Header) { + for key, values := range src { + dst[key] = append([]string(nil), values...) + } +} diff --git a/internal/metalman/commands/roles.go b/internal/metalman/commands/roles.go new file mode 100644 index 000000000..aa9eccc9a --- /dev/null +++ b/internal/metalman/commands/roles.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +type metalmanRole string + +const ( + metalmanRoleController metalmanRole = "controller" + metalmanRoleServer metalmanRole = "server" + metalmanRoleEdge metalmanRole = "edge" + metalmanRoleLegacy metalmanRole = "serve-pxe" +) + +type roleComponents struct { + leaderElection bool + ociReconciler bool + redfish bool + machineOps bool + dhcp bool + tftp bool + http bool + attestation bool + statusUpdates bool + sessionHTTP bool + sessionManager bool +} + +func componentsForRole(role metalmanRole) roleComponents { + switch role { + case metalmanRoleController: + return roleComponents{ + leaderElection: true, + ociReconciler: true, + redfish: true, + machineOps: true, + sessionManager: true, + } + case metalmanRoleServer: + return roleComponents{ + ociReconciler: true, + http: true, + attestation: true, + statusUpdates: true, + sessionHTTP: true, + } + case metalmanRoleEdge: + return roleComponents{ + http: true, + } + case metalmanRoleLegacy: + return roleComponents{ + leaderElection: true, + ociReconciler: true, + redfish: true, + machineOps: true, + dhcp: true, + tftp: true, + http: true, + attestation: true, + statusUpdates: true, + } + default: + return roleComponents{} + } +} diff --git a/internal/metalman/commands/roles_test.go b/internal/metalman/commands/roles_test.go new file mode 100644 index 000000000..ca7712aa3 --- /dev/null +++ b/internal/metalman/commands/roles_test.go @@ -0,0 +1,300 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + + "github.com/spf13/cobra" +) + +func TestMetalmanRoleComponentsAreIsolated(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + role metalmanRole + want roleComponents + }{ + { + name: "controller", + role: metalmanRoleController, + want: roleComponents{ + leaderElection: true, + ociReconciler: true, + redfish: true, + machineOps: true, + sessionManager: true, + }, + }, + { + name: "server", + role: metalmanRoleServer, + want: roleComponents{ + ociReconciler: true, + http: true, + attestation: true, + statusUpdates: true, + sessionHTTP: true, + }, + }, + { + name: "edge", + role: metalmanRoleEdge, + want: roleComponents{ + http: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := componentsForRole(tt.role); got != tt.want { + t.Fatalf("componentsForRole(%q) = %#v, want %#v", tt.role, got, tt.want) + } + }) + } +} + +func TestMetalmanRoleCommands(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + cmd func() *cobra.Command + }{ + {name: "controller", cmd: ControllerCmd}, + {name: "server", cmd: ServerCmd}, + {name: "edge", cmd: EdgeCmd}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := tt.cmd().Name(); got != tt.name { + t.Fatalf("command name = %q, want %q", got, tt.name) + } + }) + } +} + +func TestEdgeCommandRequiresOnlyBackendConnection(t *testing.T) { + t.Parallel() + + cmd := EdgeCmd() + for _, name := range []string{"backend-url", "bind-address", "http-port", "tls-cert-file", "tls-key-file", "endpoint", "edge-token-file", "dhcp-enabled", "dhcp-interface", "dhcp-server-ip", "dhcp-port", "tftp-enabled", "tftp-bind-address", "tftp-port"} { + if cmd.Flags().Lookup(name) == nil { + t.Errorf("edge command has no --%s flag", name) + } + } + + for _, name := range []string{"site", "cache-dir", "leader-elect-lease-duration"} { + if cmd.Flags().Lookup(name) != nil { + t.Errorf("edge command unexpectedly has controller flag --%s", name) + } + } +} + +func TestEdgeTLSRequiresCertificateAndKeyTogether(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + certFile string + keyFile string + wantErr bool + }{ + {name: "plaintext"}, + {name: "TLS", certFile: "tls.crt", keyFile: "tls.key"}, + {name: "certificate only", certFile: "tls.crt", wantErr: true}, + {name: "key only", keyFile: "tls.key", wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := validateEdgeTLSFiles(tt.certFile, tt.keyFile) + if (err != nil) != tt.wantErr { + t.Fatalf("validateEdgeTLSFiles(%q, %q) error = %v, wantErr %v", tt.certFile, tt.keyFile, err, tt.wantErr) + } + }) + } +} + +func TestEdgeProxyPreservesSessionPathAndRange(t *testing.T) { + t.Parallel() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/netboot/capability/artifact/disk.img.gz" { + t.Errorf("backend path = %q", r.URL.Path) + } + + if got := r.Header.Get("Range"); got != "bytes=4096-8191" { + t.Errorf("backend Range = %q", got) + } + + w.Header().Set("Content-Range", "bytes 4096-8191/16384") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("range")) + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/capability/artifact/disk.img.gz", nil) + request.Header.Set("Range", "bytes=4096-8191") + + response := httptest.NewRecorder() + + newEdgeProxy(backendURL).ServeHTTP(response, request) + + if response.Code != http.StatusPartialContent { + t.Fatalf("status = %d, want %d", response.Code, http.StatusPartialContent) + } + + if got := response.Header().Get("Content-Range"); got != "bytes 4096-8191/16384" { + t.Errorf("Content-Range = %q", got) + } + + body, err := io.ReadAll(response.Result().Body) + if err != nil { + t.Fatal(err) + } + + if got := string(body); got != "range" { + t.Errorf("body = %q", got) + } +} + +func TestEdgeProxyResumesTruncatedArtifactFromBackendRange(t *testing.T) { + t.Parallel() + + const artifact = "immutable-artifact" + + var requests atomic.Int32 + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + if got := r.Header.Get("Range"); got != "" { + t.Errorf("initial Range = %q, want empty", got) + } + + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("resume Range = %q, want %q", got, "bytes=9-17") + } + + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + t.Errorf("unexpected backend request %d", requests.Load()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + edge := httptest.NewServer(newEdgeProxy(backendURL)) + defer edge.Close() + + response, err := http.Get(edge.URL + "/v1/netboot/sessions/session/capability/artifacts/disk.img.gz") //nolint:noctx // Test request. + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() //nolint:errcheck // Test cleanup. + + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + + if got := string(body); got != artifact { + t.Errorf("body = %q, want %q", got, artifact) + } + + if got := requests.Load(); got != 2 { + t.Errorf("backend requests = %d, want 2", got) + } +} + +func TestEdgeProxyRetriesFailedArtifactResumeRequest(t *testing.T) { + t.Parallel() + + const artifact = "immutable-artifact" + + var requests atomic.Int32 + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijacking failed resume request: %v", err) + return + } + + _ = conn.Close() + case 3: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("resume Range = %q, want %q", got, "bytes=9-17") + } + + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + t.Errorf("unexpected backend request %d", requests.Load()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + edge := httptest.NewServer(newEdgeProxy(backendURL)) + defer edge.Close() + + response, err := http.Get(edge.URL + "/v1/netboot/sessions/session/capability/artifacts/disk.img.gz") //nolint:noctx // Test request. + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() //nolint:errcheck // Test cleanup. + + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + + if got := string(body); got != artifact { + t.Errorf("body = %q, want %q", got, artifact) + } + + if got := requests.Load(); got != 3 { + t.Errorf("backend requests = %d, want 3", got) + } +} diff --git a/internal/metalman/commands/serve_pxe.go b/internal/metalman/commands/serve_pxe.go index 75142a6e7..f88bfd987 100644 --- a/internal/metalman/commands/serve_pxe.go +++ b/internal/metalman/commands/serve_pxe.go @@ -38,8 +38,13 @@ import ( // omits spec.host.netboot.netbootImage. It is set at build time via -ldflags. var DefaultNetbootImage = "netboot:latest" -// ServePXECmd returns a cobra.Command that runs PXE servers and the BMC control loop. +// ServePXECmd returns the legacy monolithic command. Production wiring uses +// ControllerCmd, ServerCmd, and EdgeCmd instead. func ServePXECmd() *cobra.Command { + return newMetalmanRoleCmd(metalmanRoleLegacy, "Run PXE servers and BMC control loop") +} + +func newMetalmanRoleCmd(role metalmanRole, short string) *cobra.Command { var ( site string cacheDir string @@ -58,11 +63,16 @@ func ServePXECmd() *cobra.Command { operationPollInterval time.Duration defaultNetbootImage string defaultNetbootPullSecret string + capabilityKeyFile string + capabilityKeyID string + edgeServiceAccount string ) + components := componentsForRole(role) + cmd := &cobra.Command{ - Use: "serve-pxe", - Short: "Run PXE servers and BMC control loop", + Use: string(role), + Short: short, RunE: func(cmd *cobra.Command, _ []string) error { ctx := ctrl.SetupSignalHandler() cfg := ctrl.GetConfigOrDie() @@ -85,7 +95,7 @@ func ServePXECmd() *cobra.Command { mgr, err := ctrl.NewManager(cfg, manager.Options{ Scheme: scheme, - LeaderElection: true, + LeaderElection: components.leaderElection, LeaderElectionID: leID, LeaderElectionNamespace: leaderElectionNamespace, LeaseDuration: &leaseDuration, @@ -203,19 +213,35 @@ func ServePXECmd() *cobra.Command { ociCache := netboot.NewOCICache(cacheDir) - if err := (&netboot.OCIReconciler{ - Client: mgr.GetClient(), - Cache: ociCache, - DefaultNetbootRef: defaultNetbootImage, - DefaultNetbootPullSecretRef: defaultNetbootPullSecretRef, - }).SetupWithManager(mgr); err != nil { - return fmt.Errorf("setting up OCI reconciler: %w", err) + if components.ociReconciler { + if err := (&netboot.OCIReconciler{ + Client: mgr.GetClient(), + Cache: ociCache, + DefaultNetbootRef: defaultNetbootImage, + DefaultNetbootPullSecretRef: defaultNetbootPullSecretRef, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setting up OCI reconciler: %w", err) + } } redfishPool := redfish.NewPool() defer redfishPool.Close() - statusQueue := &metalmachineops.StatusQueue{Client: mgr.GetClient()} + sessionManager := &metalmachineops.KubernetesSessionManager{ + Client: mgr.GetClient(), + Cache: ociCache, + DefaultNetbootRef: defaultNetbootImage, + DefaultNetbootPullSecret: defaultNetbootPullSecretRef, + Cluster: clusterInfoWatcher, + KubernetesVersion: kubeVersion, + ClusterDNS: clusterDNS, + ProviderLabels: providerLabels, + } + + var statusQueue *metalmachineops.StatusQueue + if components.statusUpdates { + statusQueue = &metalmachineops.StatusQueue{Client: mgr.GetClient()} + } resolver := netboot.FileResolver{ Cache: ociCache, @@ -228,21 +254,45 @@ func ServePXECmd() *cobra.Command { ProviderLabels: providerLabels, } - if err := (&redfish.Reconciler{Client: mgr.GetClient(), Pool: redfishPool, FileResolver: &resolver}).SetupWithManager(mgr); err != nil { - return fmt.Errorf("setting up Redfish reconciler: %w", err) + if components.redfish { + if err := (&redfish.Reconciler{Client: mgr.GetClient(), Pool: redfishPool, FileResolver: &resolver}).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setting up Redfish reconciler: %w", err) + } } - if err := (&metalmachineops.Reconciler{ - Client: mgr.GetClient(), - APIReader: mgr.GetAPIReader(), - Site: site, - PowerClients: &metalmachineops.RedfishPowerClientFactory{Reader: mgr.GetClient(), Pool: redfishPool}, - HTTPBootURL: resolver.HTTPBootURL, - MaxConcurrentMachines: operationMaxConcurrentMachines, - MaxAttempts: operationMaxAttempts, - PollInterval: operationPollInterval, - }).SetupWithManager(mgr); err != nil { - return fmt.Errorf("setting up MachineOperation reconciler: %w", err) + if components.machineOps { + var sessionHTTPBootURL func(*v1alpha3.NetbootSession) (string, error) + + if components.sessionManager { + capabilityKey, err := os.ReadFile(capabilityKeyFile) + if err != nil { + return fmt.Errorf("reading capability key: %w", err) + } + + capabilities, err := netboot.NewCapabilitySigner(capabilityKey, capabilityKeyID, nil) + if err != nil { + return fmt.Errorf("creating capability signer: %w", err) + } + + sessionHTTPBootURL = func(session *v1alpha3.NetbootSession) (string, error) { + return netboot.SessionArtifactURL(capabilities, session, session.Spec.Boot.FirmwareArtifact) + } + } + + if err := (&metalmachineops.Reconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Site: site, + PowerClients: &metalmachineops.RedfishPowerClientFactory{Reader: mgr.GetClient(), Pool: redfishPool}, + Sessions: sessionManager, + HTTPBootURL: resolver.HTTPBootURL, + SessionHTTPBootURL: sessionHTTPBootURL, + MaxConcurrentMachines: operationMaxConcurrentMachines, + MaxAttempts: operationMaxAttempts, + PollInterval: operationPollInterval, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setting up MachineOperation reconciler: %w", err) + } } if dhcpInterface != "" && dhcpAutoInterface { @@ -269,52 +319,90 @@ func ServePXECmd() *cobra.Command { dhcpServerIP = ifIP } - dhcpServer := &dhcp.Server{ - Interface: dhcpInterface, - Port: dhcpPort, - Reader: mgr.GetClient(), - ServerIP: dhcpServerIP, - OCICache: ociCache, - ServeURL: serveURL, - DefaultNetbootRef: defaultNetbootImage, - } - if err := mgr.Add(dhcpServer); err != nil { - return fmt.Errorf("adding DHCP server: %w", err) + if components.dhcp { + dhcpServer := &dhcp.Server{ + Interface: dhcpInterface, + Port: dhcpPort, + Reader: mgr.GetClient(), + ServerIP: dhcpServerIP, + OCICache: ociCache, + ServeURL: serveURL, + DefaultNetbootRef: defaultNetbootImage, + } + if err := mgr.Add(dhcpServer); err != nil { + return fmt.Errorf("adding DHCP server: %w", err) + } } - if err := mgr.Add(statusQueue); err != nil { - return fmt.Errorf("adding status queue: %w", err) + if statusQueue != nil { + if err := mgr.Add(statusQueue); err != nil { + return fmt.Errorf("adding status queue: %w", err) + } } - tftpServer := &netboot.TFTPServer{ - BindAddr: bindAddress, - FileResolver: resolver, - StatusRecorder: statusQueue, - } - if err := mgr.Add(tftpServer); err != nil { - return fmt.Errorf("adding TFTP server: %w", err) + if components.tftp { + tftpServer := &netboot.TFTPServer{ + BindAddr: bindAddress, + FileResolver: resolver, + StatusRecorder: statusQueue, + } + if err := mgr.Add(tftpServer); err != nil { + return fmt.Errorf("adding TFTP server: %w", err) + } } - attestHandler := &attestation.Handler{ - Clientset: clientset, - ClusterCA: clusterCA, - LookupNodeByIP: resolver.LookupNodeByIP, - StatusUpdater: &StatusUpdater{Client: mgr.GetClient()}, - } + if components.http { + httpMux := http.NewServeMux() - httpMux := http.NewServeMux() - httpMux.HandleFunc("POST /attest", attestHandler.Attest) + var attestHandler *attestation.Handler + if components.attestation { + attestHandler = &attestation.Handler{ + Clientset: clientset, + ClusterCA: clusterCA, + LookupNodeByIP: resolver.LookupNodeByIP, + StatusUpdater: &StatusUpdater{Client: mgr.GetClient()}, + } + } - httpServer := &netboot.HTTPServer{ - BindAddr: bindAddress, - Port: httpPort, - Client: mgr.GetClient(), - Mux: httpMux, - FileResolver: resolver, - StatusRecorder: statusQueue, - } - if err := mgr.Add(httpServer); err != nil { - return fmt.Errorf("adding HTTP server: %w", err) + if components.sessionHTTP { + capabilityKey, err := os.ReadFile(capabilityKeyFile) + if err != nil { + return fmt.Errorf("reading capability key: %w", err) + } + + capabilities, err := netboot.NewCapabilitySigner(capabilityKey, capabilityKeyID, nil) + if err != nil { + return fmt.Errorf("creating capability signer: %w", err) + } + + (&netboot.SessionHTTPServer{ + Client: mgr.GetClient(), + Cache: ociCache, + Capabilities: capabilities, + StatusRecorder: &metalmachineops.SessionStatusRecorder{Client: mgr.GetClient()}, + Attestation: attestHandler, + EdgeAuthenticator: &netboot.TokenReviewEdgeAuthenticator{ + Client: clientset.AuthenticationV1(), + ServiceAccountName: edgeServiceAccount, + }, + }).RegisterHandlers(httpMux) + } + + if attestHandler != nil && !components.sessionHTTP { + httpMux.HandleFunc("POST /attest", attestHandler.Attest) + } + + httpServer := &netboot.HTTPServer{ + BindAddr: bindAddress, + Port: httpPort, + Client: mgr.GetClient(), + Mux: httpMux, + FileResolver: resolver, + StatusRecorder: statusQueue, + } + if err := mgr.Add(httpServer); err != nil { + return fmt.Errorf("adding HTTP server: %w", err) + } } siteDisplay := site @@ -323,7 +411,8 @@ func ServePXECmd() *cobra.Command { } PrintConfig("site", siteDisplay) - PrintConfig("leader-election", leID) + PrintConfig("role", string(role)) + PrintConfig("leader-election", fmt.Sprintf("%t", components.leaderElection)) PrintConfig("serve-url", serveURL) PrintConfig("default-netboot-image", defaultNetbootImage) PrintConfig("cache-dir", cacheDir) @@ -331,15 +420,26 @@ func ServePXECmd() *cobra.Command { PrintConfig("dhcp-port", fmt.Sprintf("%d", dhcpPort)) fmt.Println() - if dhcpInterface != "" { - PrintService("DHCP", fmt.Sprintf("%s:%d", dhcpInterface, dhcpPort)) - } else { - PrintService("DHCP", fmt.Sprintf("0.0.0.0:%d (relay)", dhcpPort)) + if components.dhcp { + if dhcpInterface != "" { + PrintService("DHCP", fmt.Sprintf("%s:%d", dhcpInterface, dhcpPort)) + } else { + PrintService("DHCP", fmt.Sprintf("0.0.0.0:%d (relay)", dhcpPort)) + } + } + + if components.tftp { + PrintService("TFTP", fmt.Sprintf("%s:69", bindAddress)) + } + + if components.http { + PrintService("HTTP", fmt.Sprintf("%s:%d", bindAddress, httpPort)) + } + + if components.redfish { + PrintService("Redfish", "reconciler") } - PrintService("TFTP", fmt.Sprintf("%s:69", bindAddress)) - PrintService("HTTP", fmt.Sprintf("%s:%d", bindAddress, httpPort)) - PrintService("Redfish", "reconciler") PrintReady() return mgr.Start(ctx) @@ -364,6 +464,15 @@ func ServePXECmd() *cobra.Command { cmd.Flags().StringVar(&defaultNetbootImage, "default-netboot-image", DefaultNetbootImage, "Default OCI image containing PXE netboot artifacts") cmd.Flags().StringVar(&defaultNetbootPullSecret, "default-netboot-pull-secret", "", "Namespaced Secret reference (namespace/name) for pulling the default netboot OCI image") + if components.sessionHTTP || components.sessionManager { + cmd.Flags().StringVar(&capabilityKeyFile, "capability-key-file", "/var/run/secrets/metalman/capability.key", "File containing the shared capability HMAC key") + cmd.Flags().StringVar(&capabilityKeyID, "capability-key-id", "v1", "Identifier for the active capability HMAC key") + } + + if components.sessionHTTP { + cmd.Flags().StringVar(&edgeServiceAccount, "edge-service-account", "metalman-edge", "ServiceAccount name accepted by internal edge APIs") + } + return cmd } diff --git a/internal/metalman/commands/server.go b/internal/metalman/commands/server.go new file mode 100644 index 000000000..54924978a --- /dev/null +++ b/internal/metalman/commands/server.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package commands + +import "github.com/spf13/cobra" + +// ServerCmd runs Metalman's replicated artifact and callback server. +func ServerCmd() *cobra.Command { + return newMetalmanRoleCmd(metalmanRoleServer, "Run the Metalman artifact and control server") +} diff --git a/internal/metalman/dhcp/backend.go b/internal/metalman/dhcp/backend.go new file mode 100644 index 000000000..aa9114fd8 --- /dev/null +++ b/internal/metalman/dhcp/backend.go @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package dhcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + pathpkg "path" + "strings" +) + +// HTTPDecisionProvider asks the Metalman server for an immutable session DHCP +// decision. It has no Kubernetes or OCI cache dependency. +type HTTPDecisionProvider struct { + backendURL *url.URL + endpoint string + token string + tokenFile string + client *http.Client +} + +func NewHTTPDecisionProviderFromTokenFile(backendURL, endpoint, tokenFile string, client *http.Client) (*HTTPDecisionProvider, error) { + if strings.TrimSpace(tokenFile) == "" { + return nil, errors.New("edge authentication token file is required") + } + + provider, err := newHTTPDecisionProvider(backendURL, endpoint, client) + if err != nil { + return nil, err + } + + provider.tokenFile = tokenFile + + return provider, nil +} + +func NewHTTPDecisionProvider(backendURL, endpoint, token string, client *http.Client) (*HTTPDecisionProvider, error) { + provider, err := newHTTPDecisionProvider(backendURL, endpoint, client) + if err != nil { + return nil, err + } + + if strings.TrimSpace(token) == "" { + return nil, errors.New("edge authentication token is required") + } + + provider.token = token + + return provider, nil +} + +func newHTTPDecisionProvider(backendURL, endpoint string, client *http.Client) (*HTTPDecisionProvider, error) { + backend, err := url.Parse(backendURL) + if err != nil { + return nil, fmt.Errorf("parsing backend URL: %w", err) + } + + if (backend.Scheme != "http" && backend.Scheme != "https") || backend.Host == "" { + return nil, errors.New("backend URL must use HTTP or HTTPS and include a host") + } + + if strings.TrimSpace(endpoint) == "" { + return nil, errors.New("netboot endpoint name is required") + } + + if client == nil { + client = http.DefaultClient + } + + return &HTTPDecisionProvider{backendURL: backend, endpoint: endpoint, client: client}, nil +} + +func (p *HTTPDecisionProvider) Decide(ctx context.Context, mac string, httpClient bool) (*Decision, error) { + requestURL := *p.backendURL + requestURL.Path = pathpkg.Join(requestURL.Path, "v1/netboot/endpoints", p.endpoint, "dhcp", mac) + query := requestURL.Query() + query.Set("httpClient", fmt.Sprintf("%t", httpClient)) + requestURL.RawQuery = query.Encode() + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("creating DHCP decision request: %w", err) + } + + token := p.token + if p.tokenFile != "" { + tokenBytes, err := os.ReadFile(p.tokenFile) + if err != nil { + return nil, fmt.Errorf("reading edge authentication token: %w", err) + } + + token = strings.TrimSpace(string(tokenBytes)) + } + + request.Header.Set("Authorization", "Bearer "+token) + + response, err := p.client.Do(request) + if err != nil { + return nil, fmt.Errorf("requesting DHCP decision: %w", err) + } + defer response.Body.Close() //nolint:errcheck // Response body is discarded after decoding. + + if response.StatusCode == http.StatusNotFound { + return nil, nil + } + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("DHCP decision server returned %s", response.Status) + } + + var decision Decision + if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { + return nil, fmt.Errorf("decoding DHCP decision: %w", err) + } + + return &decision, nil +} diff --git a/internal/metalman/dhcp/backend_test.go b/internal/metalman/dhcp/backend_test.go new file mode 100644 index 000000000..42ebf132a --- /dev/null +++ b/internal/metalman/dhcp/backend_test.go @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package dhcp + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +func TestHTTPDecisionProviderAuthenticatesEndpointRequest(t *testing.T) { + t.Parallel() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Path; got != "/v1/netboot/endpoints/edge-1/dhcp/aa:bb:cc:dd:ee:ff" { + t.Errorf("path = %q", got) + } + + if got := r.URL.Query().Get("httpClient"); got != "true" { + t.Errorf("httpClient = %q", got) + } + + if got := r.Header.Get("Authorization"); got != "Bearer edge-token" { + t.Errorf("Authorization = %q", got) + } + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"lease":{"mac":"aa:bb:cc:dd:ee:ff","ipv4":"10.0.1.20","subnetMask":"255.255.255.0"},"transport":"HTTP","bootFile":"https://boot.example/shim.efi"}`)) + })) + defer backend.Close() + + provider, err := NewHTTPDecisionProvider(backend.URL, "edge-1", "edge-token", backend.Client()) + if err != nil { + t.Fatal(err) + } + + decision, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", true) + if err != nil { + t.Fatal(err) + } + + if decision.Transport != v1alpha3.NetbootTransportHTTP || decision.Lease.IPv4 != "10.0.1.20" { + t.Fatalf("decision = %#v", decision) + } +} + +func TestHTTPDecisionProviderTreatsMissingSessionAsNoDecision(t *testing.T) { + t.Parallel() + + backend := httptest.NewServer(http.NotFoundHandler()) + defer backend.Close() + + provider, err := NewHTTPDecisionProvider(backend.URL, "edge-1", "edge-token", backend.Client()) + if err != nil { + t.Fatal(err) + } + + decision, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false) + if err != nil { + t.Fatal(err) + } + + if decision != nil { + t.Fatalf("decision = %#v, want nil", decision) + } +} + +func TestHTTPDecisionProviderReloadsProjectedToken(t *testing.T) { + t.Parallel() + + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte("first"), 0o600); err != nil { + t.Fatal(err) + } + + wantToken := "first" + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer "+wantToken { + t.Errorf("Authorization = %q, want token %q", got, wantToken) + } + + http.NotFound(w, r) + })) + defer backend.Close() + + provider, err := NewHTTPDecisionProviderFromTokenFile(backend.URL, "edge-1", tokenFile, backend.Client()) + if err != nil { + t.Fatal(err) + } + + if _, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false); err != nil { + t.Fatal(err) + } + + wantToken = "second" + if err := os.WriteFile(tokenFile, []byte(wantToken), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := provider.Decide(t.Context(), "aa:bb:cc:dd:ee:ff", false); err != nil { + t.Fatal(err) + } +} diff --git a/internal/metalman/dhcp/dhcp.go b/internal/metalman/dhcp/dhcp.go index 43ad373ad..45cfa8247 100644 --- a/internal/metalman/dhcp/dhcp.go +++ b/internal/metalman/dhcp/dhcp.go @@ -25,12 +25,26 @@ type Server struct { Interface string Port int Reader client.Reader + DecisionProvider DecisionProvider ServerIP net.IP OCICache *netboot.OCICache ServeURL string DefaultNetbootRef string } +// Decision is the immutable lease and boot information returned by the +// Metalman server for one ready netboot session. +type Decision struct { + Lease v1alpha3.DHCPLease `json:"lease"` + Transport v1alpha3.NetbootTransport `json:"transport"` + BootFile string `json:"bootFile"` +} + +// DecisionProvider resolves the active ready session for a DHCP client. +type DecisionProvider interface { + Decide(ctx context.Context, mac string, httpClient bool) (*Decision, error) +} + func (s *Server) NeedLeaderElection() bool { // Responding to unicast packets is always safe. // But we need election when binding to an interface (e.g. listening for multicast) @@ -125,33 +139,17 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { ctx := context.Background() - var list v1alpha3.MachineList - if err := s.Reader.List(ctx, &list, client.MatchingFields{indexing.IndexNodeByMAC: mac}); err != nil { - log.Error("listing Machines by MAC", "err", err) - return - } - - if len(list.Items) == 0 { + decision, node, err := s.decision(ctx, mac, isHTTPClientRequest(m)) + if err != nil { + log.Error("resolving DHCP decision", "err", err) return } - node := &list.Items[0] - if node.Spec.Netboot() == nil { + if decision == nil { return } - var lease *v1alpha3.DHCPLease - - for i := range node.Spec.Netboot().DHCPLeases { - if strings.EqualFold(node.Spec.Netboot().DHCPLeases[i].MAC, mac) { - lease = &node.Spec.Netboot().DHCPLeases[i] - break - } - } - - if lease == nil { - return - } + lease := &decision.Lease clientIP := net.ParseIP(lease.IPv4).To4() if clientIP == nil { @@ -189,18 +187,35 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { } } - netbootImage := node.Spec.Netboot().NetbootImage - if netbootImage == "" { - netbootImage = s.DefaultNetbootRef - } + if s.DecisionProvider != nil { + switch decision.Transport { + case v1alpha3.NetbootTransportHTTP: + if isHTTPClientRequest(m) && decision.BootFile != "" { + resp.UpdateOption(dhcpv4.OptBootFileName(decision.BootFile)) + } + case v1alpha3.NetbootTransportTFTP: + if decision.BootFile != "" { + resp.UpdateOption(dhcpv4.OptTFTPServerName(s.ServerIP.String())) + resp.UpdateOption(dhcpv4.OptBootFileName(decision.BootFile)) + resp.ServerIPAddr = s.ServerIP + } + } + } else if node != nil { + netbootImage := node.Spec.Netboot().NetbootImage + if netbootImage == "" { + netbootImage = s.DefaultNetbootRef + } + + if netbootImage == "" || s.OCICache == nil { + goto send + } - if netbootImage != "" && s.OCICache != nil { architecture := node.Spec.Netboot().TargetArchitecture() meta, err := s.OCICache.MetadataForRefArchitecture(netbootImage, architecture) if err != nil { log.Warn("OCI image metadata not available", "image", netbootImage, "architecture", architecture, "err", err) - } else if node.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP { + } else if node.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP { if isHTTPClientRequest(m) { bootURL, err := netboot.JoinServeURLPath(s.ServeURL, netboot.HTTPBootPathFromMetadata(meta)) if err != nil { @@ -216,6 +231,7 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { } } +send: switch m.MessageType() { case dhcpv4.MessageTypeDiscover: resp.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeOffer)) @@ -230,13 +246,43 @@ func (s *Server) handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { dest = peer } - log.Info("sending DHCP response", "node", node.Name, "ip", lease.IPv4, "response", resp.MessageType().String()) + log.Info("sending DHCP response", "ip", lease.IPv4, "response", resp.MessageType().String()) if _, err := conn.WriteTo(resp.ToBytes(), dest); err != nil { log.Error("sending DHCP response", "err", err) } } +func (s *Server) decision(ctx context.Context, mac string, httpClient bool) (*Decision, *v1alpha3.Machine, error) { + if s.DecisionProvider != nil { + decision, err := s.DecisionProvider.Decide(ctx, mac, httpClient) + return decision, nil, err + } + + if s.Reader == nil { + return nil, nil, errors.New("DHCP decision provider is not configured") + } + + var list v1alpha3.MachineList + if err := s.Reader.List(ctx, &list, client.MatchingFields{indexing.IndexNodeByMAC: mac}); err != nil { + return nil, nil, fmt.Errorf("listing Machines by MAC: %w", err) + } + + if len(list.Items) == 0 || list.Items[0].Spec.Netboot() == nil { + return nil, nil, nil + } + + node := &list.Items[0] + for i := range node.Spec.Netboot().DHCPLeases { + lease := &node.Spec.Netboot().DHCPLeases[i] + if strings.EqualFold(lease.MAC, mac) { + return &Decision{Lease: *lease}, node, nil + } + } + + return nil, nil, nil +} + func isHTTPClientRequest(m *dhcpv4.DHCPv4) bool { return strings.HasPrefix(m.ClassIdentifier(), "HTTPClient") } diff --git a/internal/metalman/dhcp/dhcp_test.go b/internal/metalman/dhcp/dhcp_test.go index 3a1bc164f..a7d7d52fb 100644 --- a/internal/metalman/dhcp/dhcp_test.go +++ b/internal/metalman/dhcp/dhcp_test.go @@ -4,6 +4,7 @@ package dhcp import ( + "context" "net" "os" "path/filepath" @@ -266,7 +267,7 @@ func TestDHCPHandlerHTTPBootSuppressesPXEBootOptions(t *testing.T) { Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ NetbootImage: netbootImageRef, - BootProtocol: v1alpha3.PXEBootProtocolHTTP, + Transport: v1alpha3.NetbootTransportHTTP, DHCPLeases: []v1alpha3.DHCPLease{{ MAC: "aa:bb:cc:dd:ee:f3", IPv4: "10.0.1.13", @@ -341,7 +342,7 @@ func TestDHCPHandlerHTTPBootClientGetsHTTPBootURL(t *testing.T) { Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ NetbootImage: netbootImageRef, - BootProtocol: v1alpha3.PXEBootProtocolHTTP, + Transport: v1alpha3.NetbootTransportHTTP, DHCPLeases: []v1alpha3.DHCPLease{{ MAC: "aa:bb:cc:dd:ee:f4", IPv4: "10.0.1.14", @@ -413,6 +414,67 @@ func TestDHCPHandlerHTTPBootClientGetsHTTPBootURL(t *testing.T) { } } +func TestDHCPHandlerUsesBackendSessionDecision(t *testing.T) { + t.Parallel() + + mac, err := net.ParseMAC("aa:bb:cc:dd:ee:f5") + if err != nil { + t.Fatal(err) + } + + serverIP := net.ParseIP("10.0.1.254").To4() + provider := &fakeDecisionProvider{decision: &Decision{ + Lease: v1alpha3.DHCPLease{ + MAC: mac.String(), + IPv4: "10.0.1.15", + SubnetMask: "255.255.255.0", + Gateway: "10.0.1.1", + }, + Transport: v1alpha3.NetbootTransportHTTP, + BootFile: "https://boot.example/v1/netboot/sessions/session/capability/artifacts/shimx64.efi", + }} + srv := &Server{ + Interface: "eth0", + ServerIP: serverIP, + DecisionProvider: provider, + } + + discover, err := dhcpv4.NewDiscovery(mac) + if err != nil { + t.Fatal(err) + } + + discover.UpdateOption(dhcpv4.OptClassIdentifier("HTTPClient:Arch:00016:UNDI:003016")) + + conn := &fakePacketConn{} + srv.handler(conn, &net.UDPAddr{IP: net.ParseIP("10.0.1.15"), Port: 68}, discover) + + if provider.mac != mac.String() { + t.Errorf("provider MAC = %q, want %q", provider.mac, mac.String()) + } + + if !provider.httpClient { + t.Error("provider did not receive HTTP client identity") + } + + if conn.written == nil { + t.Fatal("expected DHCP response, got none") + } + + response, err := dhcpv4.FromBytes(conn.written) + if err != nil { + t.Fatal(err) + } + + if got := response.BootFileNameOption(); got != provider.decision.BootFile { + t.Errorf("bootfile = %q, want %q", got, provider.decision.BootFile) + } + + if got := response.YourIPAddr.String(); got != provider.decision.Lease.IPv4 { + t.Errorf("lease IP = %q, want %q", got, provider.decision.Lease.IPv4) + } +} + func TestDHCPHandlerUnknownMAC(t *testing.T) { mac, _ := net.ParseMAC("ff:ff:ff:ff:ff:ff") serverIP := net.ParseIP("10.0.1.254").To4() @@ -772,6 +834,19 @@ type fakePacketConn struct { dest net.Addr } +type fakeDecisionProvider struct { + decision *Decision + mac string + httpClient bool +} + +func (f *fakeDecisionProvider) Decide(_ context.Context, mac string, httpClient bool) (*Decision, error) { + f.mac = mac + f.httpClient = httpClient + + return f.decision, nil +} + func (f *fakePacketConn) ReadFrom(b []byte) (int, net.Addr, error) { return 0, nil, nil } func (f *fakePacketConn) WriteTo(b []byte, addr net.Addr) (int, error) { f.written = make([]byte, len(b)) diff --git a/internal/metalman/machineops/controller.go b/internal/metalman/machineops/controller.go index 636c3a28a..b3d73d075 100644 --- a/internal/metalman/machineops/controller.go +++ b/internal/metalman/machineops/controller.go @@ -64,6 +64,12 @@ type PowerClientFactory interface { ForMachine(ctx context.Context, machine *v1alpha3.Machine) (PowerClient, error) } +// SessionManager creates or retrieves the immutable session for one +// HostReplace target. +type SessionManager interface { + Ensure(ctx context.Context, operation *v1alpha3.MachineOperation, machine *v1alpha3.Machine) (*v1alpha3.NetbootSession, error) +} + // Reconciler reconciles metalman-owned host MachineOperations. type Reconciler struct { client.Client @@ -71,7 +77,9 @@ type Reconciler struct { Site string PowerClients PowerClientFactory + Sessions SessionManager HTTPBootURL func(*v1alpha3.Machine) (string, error) + SessionHTTPBootURL func(*v1alpha3.NetbootSession) (string, error) MaxConcurrentMachines int MaxAttempts int32 PollInterval time.Duration @@ -83,6 +91,9 @@ type Reconciler struct { // +kubebuilder:rbac:groups=unbounded-cloud.io,resources=machineoperations/status,verbs=get;update;patch // +kubebuilder:rbac:groups=unbounded-cloud.io,resources=machines,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=unbounded-cloud.io,resources=machines/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=unbounded-cloud.io,resources=netbootendpoints,verbs=get;list;watch +// +kubebuilder:rbac:groups=unbounded-cloud.io,resources=netbootsessions,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=unbounded-cloud.io,resources=netbootsessions/status,verbs=get;update;patch // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -387,7 +398,45 @@ func (r *Reconciler) advanceTarget(ctx context.Context, op *v1alpha3.MachineOper case v1alpha3.OperationHostReboot: return r.advanceReboot(ctx, op, &machine, target, now) case v1alpha3.OperationHostReplace: - return r.advanceReplace(ctx, op, &machine, target, now) + var session *v1alpha3.NetbootSession + + if r.Sessions != nil { + var err error + + session, err = r.Sessions.Ensure(ctx, op, &machine) + if err != nil { + if errors.Is(err, netboot.ErrNotYetDownloaded) { + target.Message = "waiting for OCI images to resolve to immutable digests" + + return targetChange{target: target} + } + + return retryTarget(target, fmt.Errorf("ensure netboot session: %w", err), now, r.maxAttempts()) + } + + if target.Input == nil { + target.Input = &v1alpha3.MachineOperationTargetInput{} + } + + if target.Input.NetbootSessionRef == nil { + target.Input.NetbootSessionRef = &v1alpha3.NetbootSessionReference{Name: session.Name, UID: session.UID} + target.Message = fmt.Sprintf("persisted netboot session %s", session.Name) + + return targetChange{target: target} + } + + if target.Input.NetbootSessionRef.Name != session.Name || target.Input.NetbootSessionRef.UID != session.UID { + return failTarget(target, reasonExecutionFailed, "persisted netboot session identity changed", now) + } + + if session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive { + target.Message = fmt.Sprintf("waiting for netboot session %s to become ready", session.Name) + + return targetChange{target: target} + } + } + + return r.advanceReplace(ctx, op, &machine, session, target, now) default: return failTarget(target, reasonUnsupportedTarget, fmt.Sprintf("%s is not handled by metalman", op.Spec.OperationKind), now) } @@ -563,12 +612,12 @@ func (r *Reconciler) waitForPowerAction(target v1alpha3.MachineOperationTargetSt return targetChange{}, false } -func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOperation, machine *v1alpha3.Machine, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { - if !apimeta.IsStatusConditionTrue(op.Status.Conditions, v1alpha3.MachineOperationConditionBootImageWritten) { - return r.waitForRepaveBoot(ctx, machine, target, now) +func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOperation, machine *v1alpha3.Machine, session *v1alpha3.NetbootSession, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { + if !apimeta.IsStatusConditionTrue(target.Conditions, v1alpha3.MachineOperationConditionBootImageWritten) { + return r.waitForRepaveBoot(ctx, machine, session, target, now) } - if change, done := cloudInitReplaceStatus(op, target, now); done { + if change, done := cloudInitReplaceStatus(target, now); done { return change } @@ -589,14 +638,29 @@ func (r *Reconciler) advanceReplace(ctx context.Context, op *v1alpha3.MachineOpe return completeTarget(target, "HostReplace completed", now) } -func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, machine *v1alpha3.Machine) error { - if machine.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP { - bootURL, staticConfig, err := r.httpBootConfig(machine) +func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, machine *v1alpha3.Machine, session *v1alpha3.NetbootSession) error { + netbootSpec := machine.Spec.Netboot() + transport := netbootSpec.TargetTransport() + configurationSource := targetConfigurationSource(netbootSpec) + networkMode := targetNetworkMode(netbootSpec) + + if session != nil { + transport = session.Spec.Boot.Transport + configurationSource = session.Spec.Boot.ConfigurationSource + networkMode = session.Spec.Boot.NetworkMode + } + + if transport == v1alpha3.NetbootTransportHTTP { + if configurationSource == v1alpha3.NetbootConfigurationSourceDHCP { + return pc.SetBootOverride(ctx, redfish.BootTargetUefiHTTP, redfish.BootOnce) + } + + bootURL, staticConfig, err := r.httpBootConfig(machine, session) if err != nil { return err } - if err := setHTTPBootOverride(ctx, pc, bootURL, staticConfig); err != nil { + if err := setHTTPBootOverride(ctx, pc, bootURL, staticConfig, networkMode == v1alpha3.NetbootNetworkModeStatic); err != nil { return err } } else if err := pc.SetBootOverride(ctx, redfish.BootTargetPxe, redfish.BootContinuous); err != nil { @@ -606,30 +670,54 @@ func (r *Reconciler) configureRepaveBoot(ctx context.Context, pc PowerClient, ma return nil } -func (r *Reconciler) httpBootConfig(machine *v1alpha3.Machine) (string, redfish.StaticIPv4Config, error) { - if r.HTTPBootURL == nil { - return "", redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot URL resolver is not configured") +func (r *Reconciler) httpBootConfig(machine *v1alpha3.Machine, session *v1alpha3.NetbootSession) (string, redfish.StaticIPv4Config, error) { + var ( + bootURL string + err error + ) + + if session != nil { + if r.SessionHTTPBootURL == nil { + return "", redfish.StaticIPv4Config{}, fmt.Errorf("session HTTP boot URL resolver is not configured") + } + + bootURL, err = r.SessionHTTPBootURL(session) + } else { + if r.HTTPBootURL == nil { + return "", redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot URL resolver is not configured") + } + + bootURL, err = r.HTTPBootURL(machine) } - bootURL, err := r.HTTPBootURL(machine) if err != nil { return "", redfish.StaticIPv4Config{}, err } - staticConfig, err := httpBootStaticNetworkConfig(machine) - if err != nil { - return "", redfish.StaticIPv4Config{}, err + staticConfig := redfish.StaticIPv4Config{} + if session == nil || session.Spec.Boot.NetworkMode == v1alpha3.NetbootNetworkModeStatic { + staticConfig, err = httpBootStaticNetworkConfig(machine, session) + if err != nil { + return "", redfish.StaticIPv4Config{}, err + } } return bootURL, staticConfig, nil } -func httpBootStaticNetworkConfig(machine *v1alpha3.Machine) (redfish.StaticIPv4Config, error) { - if machine.Spec.Netboot() == nil || len(machine.Spec.Netboot().DHCPLeases) == 0 { +func httpBootStaticNetworkConfig(machine *v1alpha3.Machine, session *v1alpha3.NetbootSession) (redfish.StaticIPv4Config, error) { + var leases []v1alpha3.DHCPLease + if session != nil { + leases = session.Spec.Boot.DHCPLeases + } else if machine.Spec.Netboot() != nil { + leases = machine.Spec.Netboot().DHCPLeases + } + + if len(leases) == 0 { return redfish.StaticIPv4Config{}, fmt.Errorf("HTTP boot requires at least one static lease in spec.host.netboot.dhcpLeases") } - lease := machine.Spec.Netboot().DHCPLeases[0] + lease := leases[0] config := redfish.StaticIPv4Config{ MAC: lease.MAC, Address: lease.IPv4, @@ -649,22 +737,24 @@ func httpBootStaticNetworkConfig(machine *v1alpha3.Machine) (redfish.StaticIPv4C return config, nil } -func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config) error { +func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config, configureStaticNetwork bool) error { config, err := pc.GetBootConfig(ctx) if err != nil { return err } if !config.HasHTTPBootURI { - return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig) + return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig, configureStaticNetwork) } - if err := pc.SetStaticIPv4(ctx, staticConfig); err != nil { - if !errors.Is(err, redfish.ErrUnsupported) { - return err - } + if configureStaticNetwork { + if err := pc.SetStaticIPv4(ctx, staticConfig); err != nil { + if !errors.Is(err, redfish.ErrUnsupported) { + return err + } - return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig) + return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig, true) + } } if err := pc.SetHTTPBootOverride(ctx, bootURL); err != nil { @@ -672,12 +762,14 @@ func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, st return err } - return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig) + return setBIOSHTTPBootOverride(ctx, pc, bootURL, staticConfig, configureStaticNetwork) } // Some BMCs expose both locations but boot from the vendor BIOS setting. - if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil && !errors.Is(err, redfish.ErrUnsupported) { - return err + if configureStaticNetwork { + if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil && !errors.Is(err, redfish.ErrUnsupported) { + return err + } } if err := pc.SetBIOSHTTPBootURI(ctx, bootURL); err != nil && !errors.Is(err, redfish.ErrUnsupported) { @@ -687,9 +779,11 @@ func setHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, st return nil } -func setBIOSHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config) error { - if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil { - return err +func setBIOSHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string, staticConfig redfish.StaticIPv4Config, configureStaticNetwork bool) error { + if configureStaticNetwork { + if err := pc.SetBIOSStaticIPv4(ctx, staticConfig); err != nil { + return err + } } if err := pc.SetBIOSHTTPBootURI(ctx, bootURL); err != nil { @@ -699,7 +793,7 @@ func setBIOSHTTPBootOverride(ctx context.Context, pc PowerClient, bootURL string return pc.SetBootOverride(ctx, redfish.BootTargetUefiHTTP, redfish.BootOnce) } -func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Machine, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { +func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Machine, session *v1alpha3.NetbootSession, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) targetChange { if target.Stage == v1alpha3.OperationStageWaitingRepave && target.LastAttemptAt != nil { if now.Sub(target.LastAttemptAt.Time) < r.powerActionTimeout() { target.Message = "waiting for PXE installer to write the boot image" @@ -715,8 +809,13 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma target.LastAttemptAt = nil } - if machine.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP { - if _, _, err := r.httpBootConfig(machine); err != nil { + transport := machine.Spec.Netboot().TargetTransport() + if session != nil { + transport = session.Spec.Boot.Transport + } + + if transport == v1alpha3.NetbootTransportHTTP { + if _, _, err := r.httpBootConfig(machine, session); err != nil { if errors.Is(err, netboot.ErrNotYetDownloaded) { target.Message = "waiting for OCI image to become available" @@ -757,7 +856,7 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma return targetChange{target: target} } - if err := r.configureRepaveBoot(ctx, pc, machine); err != nil { + if err := r.configureRepaveBoot(ctx, pc, machine, session); err != nil { return retryTarget(target, err, now, r.maxAttempts()) } @@ -773,8 +872,8 @@ func (r *Reconciler) waitForRepaveBoot(ctx context.Context, machine *v1alpha3.Ma return targetChange{target: target} } -func cloudInitReplaceStatus(op *v1alpha3.MachineOperation, target v1alpha3.MachineOperationTargetStatus, now metav1.Time) (targetChange, bool) { - cond := apimeta.FindStatusCondition(op.Status.Conditions, v1alpha3.MachineOperationConditionCloudInitDone) +func cloudInitReplaceStatus(target v1alpha3.MachineOperationTargetStatus, now metav1.Time) (targetChange, bool) { + cond := apimeta.FindStatusCondition(target.Conditions, v1alpha3.MachineOperationConditionCloudInitDone) if cond != nil { switch cond.Status { case metav1.ConditionTrue: diff --git a/internal/metalman/machineops/controller_test.go b/internal/metalman/machineops/controller_test.go index ad662d3d7..8e05a852f 100644 --- a/internal/metalman/machineops/controller_test.go +++ b/internal/metalman/machineops/controller_test.go @@ -368,8 +368,8 @@ func TestReconcilerRequestsHostReplaceOnceAndCompletesAfterRepave(t *testing.T) require.Equal(t, metav1.ConditionUnknown, cloudInitCond.Status) require.Equal(t, "Pending", cloudInitCond.Reason) - markOperationCondition(t, c, op.Name, v1alpha3.MachineOperationConditionBootImageWritten, metav1.ConditionTrue, "Succeeded", "boot image written") - markOperationCondition(t, c, op.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") + markTargetCondition(t, c, op.Name, machine.Name, v1alpha3.MachineOperationConditionBootImageWritten, metav1.ConditionTrue, "Succeeded", "boot image written") + markTargetCondition(t, c, op.Name, machine.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") require.NoError(t, c.Create(context.Background(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}})) _, err = reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) @@ -380,6 +380,171 @@ func TestReconcilerRequestsHostReplaceOnceAndCompletesAfterRepave(t *testing.T) require.Equal(t, v1alpha3.OperationPhaseComplete, completed.Status.Phase) } +func TestReconcilerPersistsReadySessionBeforeHostReplaceSideEffects(t *testing.T) { + t.Parallel() + + s := testScheme(t) + machine := testBareMetalMachine("machine-session", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 3 + op := testOperation("op-session", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 2 + op.Spec.MachineRef = machine.Name + + session := &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: "netboot-session", UID: "session-uid"}, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } + sessions := &recordingSessionManager{session: session} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() + power := &recordingPowerClient{states: map[string]redfish.PowerState{machine.Name: redfish.PowerOff}} + reconciler := testReconciler(c, power, "rack-a") + reconciler.Sessions = sessions + + _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + require.Empty(t, power.calls) + + var preparing v1alpha3.MachineOperation + require.NoError(t, c.Get(t.Context(), client.ObjectKey{Name: op.Name}, &preparing)) + require.Equal(t, &v1alpha3.NetbootSessionReference{Name: session.Name, UID: session.UID}, preparing.Status.Targets[0].Input.NetbootSessionRef) + require.Equal(t, "persisted netboot session netboot-session", preparing.Status.Targets[0].Message) + + _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + require.Equal(t, []string{"machine-session:SetBootOverride:Pxe:Continuous", "machine-session:On"}, power.calls) +} + +func TestReconcilerUsesSessionCapabilityURLForHTTPBoot(t *testing.T) { + t.Parallel() + + s := testScheme(t) + machine := testBareMetalMachine("machine-session-http", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 3 + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportTFTP + machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:ff", IPv4: "192.0.2.99"}} + op := testOperation("op-session-http", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 2 + op.Spec.MachineRef = machine.Name + session := &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: "netboot-session", UID: "session-uid"}, + Spec: v1alpha3.NetbootSessionSpec{ + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{ExternalURL: "https://boot.example.com"}, + Boot: v1alpha3.NetbootSessionBoot{ + Transport: v1alpha3.NetbootTransportHTTP, + FirmwareArtifact: "bootx64.efi", + DHCPLeases: []v1alpha3.DHCPLease{httpBootLease()}, + }, + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } + c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() + power := &recordingPowerClient{states: map[string]redfish.PowerState{machine.Name: redfish.PowerOff}} + reconciler := testReconciler(c, power, "rack-a") + reconciler.Sessions = &recordingSessionManager{session: session} + reconciler.SessionHTTPBootURL = func(got *v1alpha3.NetbootSession) (string, error) { + require.Equal(t, session, got) + + return "https://boot.example.com/v1/netboot/sessions/netboot-session/capability/artifacts/bootx64.efi", nil + } + + _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + _, err = reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) + require.NoError(t, err) + + require.Contains(t, power.calls, "machine-session-http:SetHTTPBootOverride:https://boot.example.com/v1/netboot/sessions/netboot-session/capability/artifacts/bootx64.efi") +} + +func TestConfigureRepaveBootSupportsIndependentBootAxes(t *testing.T) { + t.Parallel() + + const bootURL = "https://boot.example.com/v1/netboot/sessions/session/capability/artifacts/bootx64.efi" + + tests := []struct { + name string + transport v1alpha3.NetbootTransport + configurationSource v1alpha3.NetbootConfigurationSource + networkMode v1alpha3.NetbootNetworkMode + wantCalls []string + wantURLResolution bool + }{ + { + name: "TFTP configured by DHCP", + transport: v1alpha3.NetbootTransportTFTP, + configurationSource: v1alpha3.NetbootConfigurationSourceDHCP, + networkMode: v1alpha3.NetbootNetworkModeDHCP, + wantCalls: []string{"machine:SetBootOverride:Pxe:Continuous"}, + }, + { + name: "HTTP configured by DHCP", + transport: v1alpha3.NetbootTransportHTTP, + configurationSource: v1alpha3.NetbootConfigurationSourceDHCP, + networkMode: v1alpha3.NetbootNetworkModeDHCP, + wantCalls: []string{"machine:SetBootOverride:UefiHttp:Once"}, + }, + { + name: "HTTP URL configured by Redfish with DHCP networking", + transport: v1alpha3.NetbootTransportHTTP, + configurationSource: v1alpha3.NetbootConfigurationSourceRedfish, + networkMode: v1alpha3.NetbootNetworkModeDHCP, + wantURLResolution: true, + wantCalls: []string{ + "machine:GetBootConfig", + "machine:SetHTTPBootOverride:" + bootURL, + "machine:SetBIOSHTTPBootURI:" + bootURL, + }, + }, + { + name: "HTTP URL and static network configured by Redfish", + transport: v1alpha3.NetbootTransportHTTP, + configurationSource: v1alpha3.NetbootConfigurationSourceRedfish, + networkMode: v1alpha3.NetbootNetworkModeStatic, + wantURLResolution: true, + wantCalls: []string{ + "machine:GetBootConfig", + "machine:SetStaticIPv4:aa:bb:cc:dd:ee:01:10.0.0.20:255.255.255.0:10.0.0.1:10.0.0.53", + "machine:SetHTTPBootOverride:" + bootURL, + "machine:SetBIOSStaticIPv4:10.0.0.20:255.255.255.0:10.0.0.1", + "machine:SetBIOSHTTPBootURI:" + bootURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + power := &recordingPowerClient{} + client, err := power.ForMachine(t.Context(), &v1alpha3.Machine{ObjectMeta: metav1.ObjectMeta{Name: "machine"}}) + require.NoError(t, err) + + resolved := false + reconciler := &Reconciler{SessionHTTPBootURL: func(*v1alpha3.NetbootSession) (string, error) { + resolved = true + + return bootURL, nil + }} + session := &v1alpha3.NetbootSession{Spec: v1alpha3.NetbootSessionSpec{Boot: v1alpha3.NetbootSessionBoot{ + Transport: tt.transport, + ConfigurationSource: tt.configurationSource, + NetworkMode: tt.networkMode, + DHCPLeases: []v1alpha3.DHCPLease{httpBootLease()}, + }}} + + err = reconciler.configureRepaveBoot(t.Context(), client, testBareMetalMachine("machine", "rack-a"), session) + require.NoError(t, err) + require.Equal(t, tt.wantURLResolution, resolved) + require.Equal(t, tt.wantCalls, power.calls) + }) + } +} + func TestReconcilerPowersOnHostReplaceTargetWhenOff(t *testing.T) { t.Parallel() @@ -471,7 +636,9 @@ func TestReconcilerFallsBackToBIOSHTTPBootURIForHostReplace(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -512,7 +679,9 @@ func TestReconcilerUsesBIOSHTTPBootURIWhenStandardURIAbsent(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-bios", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -553,7 +722,9 @@ func TestReconcilerFallsBackToBIOSWhenStaticInterfaceIsReadOnly(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-read-only-nic", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -600,7 +771,7 @@ func TestSetHTTPBootOverrideRefreshesStandardAndBIOSURLs(t *testing.T) { DNS: []string{"10.0.0.53"}, } - require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", staticConfig)) + require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", staticConfig, true)) require.Equal(t, []string{ "machine-1:GetBootConfig", "machine-1:SetStaticIPv4:aa:bb:cc:dd:ee:01:10.0.0.20:255.255.255.0:10.0.0.1:10.0.0.53", @@ -616,12 +787,10 @@ func TestSetHTTPBootOverrideAllowsUnsupportedBIOSURL(t *testing.T) { power := &recordingPowerClient{biosHTTPBootUnsupported: map[string]bool{"machine-1": true}} client := &recordingMachinePowerClient{parent: power, machine: "machine-1"} - require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", redfish.StaticIPv4Config{})) + require.NoError(t, setHTTPBootOverride(t.Context(), client, "http://10.0.0.10:8880/http/shimx64.efi", redfish.StaticIPv4Config{}, false)) require.Equal(t, []string{ "machine-1:GetBootConfig", - "machine-1:SetStaticIPv4:::::", "machine-1:SetHTTPBootOverride:http://10.0.0.10:8880/http/shimx64.efi", - "machine-1:SetBIOSStaticIPv4:::", "machine-1:SetBIOSHTTPBootURI:http://10.0.0.10:8880/http/shimx64.efi", }, power.calls) } @@ -631,7 +800,9 @@ func TestReconcilerRetriesHTTPHostReplaceWithoutStaticLease(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic op := testOperation("op-replace-http-no-lease", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -664,7 +835,9 @@ func TestReconcilerWaitsForHTTPBootImage(t *testing.T) { s := testScheme(t) machine := testBareMetalMachine("machine-1", "rack-a") - machine.Spec.PXE.BootProtocol = v1alpha3.PXEBootProtocolHTTP + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.PXE.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + machine.Spec.PXE.NetworkMode = v1alpha3.NetbootNetworkModeStatic machine.Spec.PXE.DHCPLeases = []v1alpha3.DHCPLease{httpBootLease()} op := testOperation("op-replace-http-wait-image", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name @@ -764,12 +937,12 @@ func TestReconcilerKeepsHostReplaceInProgressUntilNodeExists(t *testing.T) { op.Spec.MachineRef = machine.Name op.Spec.TTLSecondsAfterFinished = &ttl op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() @@ -796,12 +969,12 @@ func TestReconcilerKeepsHostReplaceInProgressUntilCloudInitCompletes(t *testing. op := testOperation("op-replace-wait-cloudinit", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}}).WithStatusSubresource(op, machine).Build() @@ -819,7 +992,7 @@ func TestReconcilerKeepsHostReplaceInProgressUntilCloudInitCompletes(t *testing. require.Equal(t, v1alpha3.OperationStageWaitingCloudInit, waiting.Status.Targets[0].Stage) require.Equal(t, "waiting for first-boot cloud-init to complete", waiting.Status.Targets[0].Message) - markOperationCondition(t, c, op.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") + markTargetCondition(t, c, op.Name, machine.Name, v1alpha3.MachineOperationConditionCloudInitDone, metav1.ConditionTrue, "Succeeded", "cloud-init completed successfully") _, err = reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: op.Name}}) require.NoError(t, err) @@ -838,12 +1011,12 @@ func TestReconcilerFailsHostReplaceWhenCloudInitFails(t *testing.T) { op := testOperation("op-replace-cloudinit-failed", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionFalse) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionFalse), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}}).WithStatusSubresource(op, machine).Build() @@ -867,8 +1040,8 @@ func TestReconcilerTimesOutHostReplaceCloudInitCondition(t *testing.T) { op := testOperation("op-replace-cloudinit-timeout", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown) - apimeta.SetStatusCondition(&op.Status.Conditions, metav1.Condition{ + conditions := hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionUnknown) + apimeta.SetStatusCondition(&conditions, metav1.Condition{ Type: v1alpha3.MachineOperationConditionCloudInitDone, Status: metav1.ConditionFalse, Reason: "Running", @@ -880,6 +1053,7 @@ func TestReconcilerTimesOutHostReplaceCloudInitCondition(t *testing.T) { Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingCloudInit, ObservedGeneration: machine.Generation, + Conditions: conditions, }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: machine.Name}}).WithStatusSubresource(op, machine).Build() @@ -904,12 +1078,12 @@ func TestReconcilerUsesKubernetesNodeRefForHostReplaceCompletion(t *testing.T) { op := testOperation("op-replace-custom-node", v1alpha3.OperationHostReplace) op.Spec.MachineRef = machine.Name op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue) op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{{ MachineRef: machine.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, ObservedGeneration: machine.Generation, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(machine, op, testRedfishSecret()).WithStatusSubresource(op, machine).Build() @@ -941,21 +1115,18 @@ func TestReconcilerCompletesAllHostReplaceTargetsWhenOperationMilestonesComplete op := testOperation("op-replace-multi", v1alpha3.OperationHostReplace) op.Spec.MachineSelector = &metav1.LabelSelector{MatchLabels: map[string]string{siteLabel: "rack-a"}} op.Status.Phase = v1alpha3.OperationPhaseInProgress - op.Status.Conditions = []metav1.Condition{ - {Type: v1alpha3.MachineOperationConditionBootLoaderDownloaded, Status: metav1.ConditionUnknown, Reason: "Pending"}, - {Type: v1alpha3.MachineOperationConditionBootImageWritten, Status: metav1.ConditionTrue, Reason: "Succeeded"}, - {Type: v1alpha3.MachineOperationConditionCloudInitDone, Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "machine-a completed cloud-init"}, - } op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{ { MachineRef: machineA.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingRepave, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }, { MachineRef: machineB.Name, Phase: v1alpha3.OperationPhaseInProgress, Stage: v1alpha3.OperationStageWaitingCloudInit, + Conditions: hostReplaceConditions(metav1.ConditionTrue, metav1.ConditionTrue), }, } @@ -1169,6 +1340,14 @@ type recordingPowerClient struct { calls []string } +type recordingSessionManager struct { + session *v1alpha3.NetbootSession +} + +func (m *recordingSessionManager) Ensure(_ context.Context, _ *v1alpha3.MachineOperation, _ *v1alpha3.Machine) (*v1alpha3.NetbootSession, error) { + return m.session.DeepCopy(), nil +} + func (r *recordingPowerClient) ForMachine(_ context.Context, machine *v1alpha3.Machine) (PowerClient, error) { r.mu.Lock() defer r.mu.Unlock() @@ -1356,19 +1535,30 @@ func testRedfishSecret() *corev1.Secret { } } -func markOperationCondition(t *testing.T, c client.Client, opName, conditionType string, status metav1.ConditionStatus, reason, message string) { +func markTargetCondition(t *testing.T, c client.Client, opName, machineName, conditionType string, status metav1.ConditionStatus, reason, message string) { t.Helper() var op v1alpha3.MachineOperation require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: opName}, &op)) - apimeta.SetStatusCondition(&op.Status.Conditions, metav1.Condition{ - Type: conditionType, - Status: status, - Reason: reason, - Message: message, - ObservedGeneration: op.Generation, - }) - require.NoError(t, c.Status().Update(context.Background(), &op)) + + for i := range op.Status.Targets { + if op.Status.Targets[i].MachineRef != machineName { + continue + } + + apimeta.SetStatusCondition(&op.Status.Targets[i].Conditions, metav1.Condition{ + Type: conditionType, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: op.Status.Targets[i].ObservedGeneration, + }) + require.NoError(t, c.Status().Update(context.Background(), &op)) + + return + } + + t.Fatalf("operation %s has no target %s", opName, machineName) } func hostReplaceConditions(bootImage, cloudInit metav1.ConditionStatus) []metav1.Condition { diff --git a/internal/metalman/machineops/session_manager.go b/internal/metalman/machineops/session_manager.go new file mode 100644 index 000000000..b33a11d3e --- /dev/null +++ b/internal/metalman/machineops/session_manager.go @@ -0,0 +1,357 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "context" + "errors" + "fmt" + "maps" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/metalman/netboot" +) + +const ( + sessionOperationUIDLabel = "metalman.unbounded-cloud.io/operation-uid" + sessionMachineUIDLabel = "metalman.unbounded-cloud.io/machine-uid" + defaultSessionTTL = 24 * time.Hour +) + +// KubernetesSessionManager persists immutable sessions in the Kubernetes API. +type KubernetesSessionManager struct { + Client client.Client + Cache *netboot.OCICache + DefaultNetbootRef string + DefaultNetbootPullSecret *v1alpha3.NamespacedSecretReference + Cluster netboot.ClusterInfoProvider + KubernetesVersion string + ClusterDNS string + ProviderLabels map[string]string + Now func() metav1.Time +} + +// Ensure returns the durable session for one operation target, creating it +// only after both OCI references have resolved to immutable digests. +func (m *KubernetesSessionManager) Ensure(ctx context.Context, operation *v1alpha3.MachineOperation, machine *v1alpha3.Machine) (*v1alpha3.NetbootSession, error) { + if operation.UID == "" || machine.UID == "" { + return nil, errors.New("operation and Machine UIDs are required for netboot session identity") + } + + name := sessionName(operation.UID, machine.UID) + + var existing v1alpha3.NetbootSession + if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, &existing); err == nil { + if existing.Spec.Operation.UID != operation.UID || existing.Spec.Machine.UID != machine.UID { + return nil, fmt.Errorf("netboot session %s belongs to different objects", name) + } + + if existing.Status.Phase == v1alpha3.NetbootSessionPhasePreparing { + return m.refreshEndpointReadiness(ctx, &existing) + } + + return &existing, nil + } else if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("get netboot session %s: %w", name, err) + } + + netbootSpec := machine.Spec.Netboot() + if netbootSpec == nil { + return nil, errors.New("machine has no netboot configuration") + } + + var endpoint v1alpha3.NetbootEndpoint + if err := m.Client.Get(ctx, client.ObjectKey{Name: netbootSpec.EndpointRef}, &endpoint); err != nil { + return nil, fmt.Errorf("get NetbootEndpoint %s: %w", netbootSpec.EndpointRef, err) + } + + architecture := netbootSpec.TargetArchitecture() + + machineDigest := m.Cache.DigestForArchitecture(netbootSpec.Image, architecture) + if machineDigest == "" { + return nil, fmt.Errorf("%w: machine image %q for architecture %q", netboot.ErrNotYetDownloaded, netbootSpec.Image, architecture) + } + + netbootRef := netbootSpec.NetbootImage + netbootPullSecret := netbootSpec.NetbootPullSecretRef + + if netbootRef == "" { + netbootRef = m.DefaultNetbootRef + netbootPullSecret = m.DefaultNetbootPullSecret + } + + netbootDigest := m.Cache.DigestForArchitecture(netbootRef, architecture) + if netbootDigest == "" { + return nil, fmt.Errorf("%w: netboot image %q for architecture %q", netboot.ErrNotYetDownloaded, netbootRef, architecture) + } + + metadata, err := m.Cache.MetadataForArchitecture(netbootDigest, architecture) + if err != nil { + return nil, fmt.Errorf("read netboot image metadata: %w", err) + } + + firmwareArtifact := firmwareArtifactForTransport(metadata, netbootSpec.TargetTransport()) + if firmwareArtifact == "" { + return nil, fmt.Errorf("netboot image %q has no firmware artifact for %s", netbootRef, netbootSpec.TargetTransport()) + } + + now := m.now() + + userData, err := m.resolveUserData(ctx, netbootSpec) + if err != nil { + return nil, err + } + + clusterInfo := netboot.ClusterInfo{} + if m.Cluster != nil { + clusterInfo = m.Cluster.ClusterInfo() + } + + session := &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + sessionOperationUIDLabel: string(operation.UID), + sessionMachineUIDLabel: string(machine.UID), + }, + }, + Spec: v1alpha3.NetbootSessionSpec{ + Machine: objectSnapshot(machine.Name, machine.UID, machine.Generation), + Operation: objectSnapshot(operation.Name, operation.UID, operation.Generation), + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{ + Name: endpoint.Name, + UID: endpoint.UID, + ExternalURL: endpoint.Spec.ExternalURL, + }, + Boot: v1alpha3.NetbootSessionBoot{ + Transport: netbootSpec.TargetTransport(), + ConfigurationSource: targetConfigurationSource(netbootSpec), + NetworkMode: targetNetworkMode(netbootSpec), + FirmwareArtifact: firmwareArtifact, + Architecture: architecture, + DHCPLeases: append([]v1alpha3.DHCPLease(nil), netbootSpec.DHCPLeases...), + TargetDisk: netbootSpec.TargetDisk, + }, + Provisioning: v1alpha3.NetbootSessionProvisioning{ + Cluster: v1alpha3.NetbootSessionCluster{ + APIServerURL: clusterInfo.ApiserverURL, + CACertBase64: clusterInfo.CACertBase64, + DNS: m.ClusterDNS, + KubernetesVersion: m.KubernetesVersion, + }, + Kubernetes: machine.Spec.Kubernetes.DeepCopy(), + Agent: machine.Spec.Agent.DeepCopy(), + ProviderLabels: maps.Clone(m.ProviderLabels), + UserData: userData, + }, + Artifacts: v1alpha3.NetbootSessionArtifacts{ + MachineImage: v1alpha3.NetbootSessionImage{ + Reference: netbootSpec.Image, + Digest: machineDigest, + PullSecretRef: netbootSpec.PullSecretRef.DeepCopy(), + }, + NetbootImage: v1alpha3.NetbootSessionImage{ + Reference: netbootRef, + Digest: netbootDigest, + PullSecretRef: netbootPullSecret.DeepCopy(), + }, + Files: sessionArtifacts(firmwareArtifact), + }, + ExpiresAt: metav1.NewTime(now.Add(defaultSessionTTL)), + }, + } + + if err := m.Client.Create(ctx, session); err != nil { + if apierrors.IsAlreadyExists(err) { + if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, session); err != nil { + return nil, fmt.Errorf("get concurrently created netboot session %s: %w", name, err) + } + + return session, nil + } + + return nil, fmt.Errorf("create netboot session %s: %w", name, err) + } + + return m.refreshEndpointReadinessWithEndpoint(ctx, session, &endpoint) +} + +func (m *KubernetesSessionManager) resolveUserData(ctx context.Context, spec *v1alpha3.PXESpec) (string, error) { + if spec.CloudInit == nil || spec.CloudInit.UserDataConfigMapRef == nil { + return "#cloud-config\n", nil + } + + ref := spec.CloudInit.UserDataConfigMapRef + + var configMap corev1.ConfigMap + if err := m.Client.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, &configMap); err != nil { + return "", fmt.Errorf("get cloud-init user-data ConfigMap %s/%s: %w", ref.Namespace, ref.Name, err) + } + + key := ref.Key + if key == "" { + key = "user-data" + } + + if value, ok := configMap.Data[key]; ok { + return value, nil + } + + if value, ok := configMap.BinaryData[key]; ok { + return string(value), nil + } + + return "", fmt.Errorf("cloud-init user-data key %q not found in ConfigMap %s/%s", key, ref.Namespace, ref.Name) +} + +func (m *KubernetesSessionManager) refreshEndpointReadiness(ctx context.Context, session *v1alpha3.NetbootSession) (*v1alpha3.NetbootSession, error) { + var endpoint v1alpha3.NetbootEndpoint + if err := m.Client.Get(ctx, client.ObjectKey{Name: session.Spec.Endpoint.Name}, &endpoint); err != nil { + return nil, fmt.Errorf("get NetbootEndpoint %s: %w", session.Spec.Endpoint.Name, err) + } + + if endpoint.UID != session.Spec.Endpoint.UID { + return nil, fmt.Errorf("NetbootEndpoint %s identity changed", endpoint.Name) + } + + return m.refreshEndpointReadinessWithEndpoint(ctx, session, &endpoint) +} + +func (m *KubernetesSessionManager) refreshEndpointReadinessWithEndpoint(ctx context.Context, session *v1alpha3.NetbootSession, endpoint *v1alpha3.NetbootEndpoint) (*v1alpha3.NetbootSession, error) { + now := m.now() + + ready := endpoint.Status.ObservedGeneration == endpoint.Generation && apimeta.IsStatusConditionTrue(endpoint.Status.Conditions, "Ready") + + session.Status.Phase = v1alpha3.NetbootSessionPhasePreparing + if ready { + session.Status.Phase = v1alpha3.NetbootSessionPhaseReady + } + + apimeta.SetStatusCondition(&session.Status.Conditions, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionPrepared, + Status: metav1.ConditionTrue, + Reason: "ArtifactsResolved", + Message: "OCI references resolved to immutable digests", + ObservedGeneration: session.Generation, + LastTransitionTime: now, + }) + apimeta.SetStatusCondition(&session.Status.Conditions, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionEndpointReady, + Status: conditionStatus(ready), + Reason: endpointReadyReason(ready), + Message: fmt.Sprintf("NetbootEndpoint %s readiness", endpoint.Name), + ObservedGeneration: session.Generation, + LastTransitionTime: now, + }) + + if err := m.Client.Status().Update(ctx, session); err != nil { + return nil, fmt.Errorf("update netboot session %s status: %w", session.Name, err) + } + + return session, nil +} + +func (m *KubernetesSessionManager) now() metav1.Time { + if m.Now != nil { + return m.Now() + } + + return metav1.Now() +} + +func sessionName(operationUID, machineUID types.UID) string { + return fmt.Sprintf("netboot-%s-%s", operationUID, machineUID) +} + +func objectSnapshot(name string, uid types.UID, generation int64) v1alpha3.NetbootSessionObjectSnapshot { + return v1alpha3.NetbootSessionObjectSnapshot{Name: name, UID: uid, Generation: generation} +} + +func targetConfigurationSource(spec *v1alpha3.PXESpec) v1alpha3.NetbootConfigurationSource { + if spec.ConfigurationSource == "" { + return v1alpha3.NetbootConfigurationSourceDHCP + } + + return spec.ConfigurationSource +} + +func targetNetworkMode(spec *v1alpha3.PXESpec) v1alpha3.NetbootNetworkMode { + if spec.NetworkMode == "" { + return v1alpha3.NetbootNetworkModeDHCP + } + + return spec.NetworkMode +} + +func sessionArtifacts(firmwareArtifact string) []v1alpha3.NetbootSessionArtifact { + artifacts := []v1alpha3.NetbootSessionArtifact{ + {Name: "disk.img.gz", Source: "MachineImage", Path: "/disk/disk.img.gz"}, + {Name: "metadata.yaml", Source: "NetbootImage", Path: "/disk/metadata.yaml"}, + {Name: firmwareArtifact, Source: "NetbootImage", Path: "/disk/" + firmwareArtifact}, + {Name: "vmlinuz", Source: "NetbootImage", Path: "/disk/vmlinuz"}, + {Name: "initrd", Source: "NetbootImage", Path: "/disk/initrd"}, + {Name: "init.cpio", Source: "NetbootImage", Path: "/disk/init.cpio"}, + {Name: "grub/grub.cfg", Source: "NetbootImage", Path: "/disk/grub/grub.cfg"}, + {Name: "cloud-init/meta-data", Source: "NetbootImage", Path: "/disk/cloud-init/meta-data"}, + {Name: "cloud-init/user-data", Source: "Session", Path: "/session/cloud-init/user-data"}, + {Name: "cloud-init/vendor-data", Source: "NetbootImage", Path: "/disk/cloud-init/vendor-data"}, + {Name: "cloud-init/network-config", Source: "NetbootImage", Path: "/disk/cloud-init/network-config"}, + } + if strings.Contains(firmwareArtifact, "aa64") { + artifacts = append(artifacts, v1alpha3.NetbootSessionArtifact{Name: "grubaa64.efi", Source: "NetbootImage", Path: "/disk/grubaa64.efi"}) + } else { + artifacts = append(artifacts, v1alpha3.NetbootSessionArtifact{Name: "grubx64.efi", Source: "NetbootImage", Path: "/disk/grubx64.efi"}) + } + + unique := make([]v1alpha3.NetbootSessionArtifact, 0, len(artifacts)) + + names := make(map[string]struct{}, len(artifacts)) + for _, artifact := range artifacts { + if _, exists := names[artifact.Name]; exists { + continue + } + + names[artifact.Name] = struct{}{} + unique = append(unique, artifact) + } + + return unique +} + +func firmwareArtifactForTransport(metadata *netboot.ImageMetadata, transport v1alpha3.NetbootTransport) string { + if metadata == nil { + return "" + } + + if transport == v1alpha3.NetbootTransportHTTP { + return netboot.HTTPBootPathFromMetadata(metadata) + } + + return strings.TrimPrefix(metadata.DHCPBootImageName, "/") +} + +func conditionStatus(ready bool) metav1.ConditionStatus { + if ready { + return metav1.ConditionTrue + } + + return metav1.ConditionFalse +} + +func endpointReadyReason(ready bool) string { + if ready { + return "Ready" + } + + return "NotReady" +} diff --git a/internal/metalman/machineops/session_manager_test.go b/internal/metalman/machineops/session_manager_test.go new file mode 100644 index 000000000..7c784937f --- /dev/null +++ b/internal/metalman/machineops/session_manager_test.go @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/metalman/netboot" +) + +func TestSessionManagerSnapshotsDigestsAndReusesSession(t *testing.T) { + t.Parallel() + + machine := testBareMetalMachine("machine-session", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 4 + machine.Spec.PXE.EndpointRef = "rack-a-edge" + machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" + machine.Spec.PXE.Transport = v1alpha3.NetbootTransportHTTP + machine.Spec.Kubernetes = &v1alpha3.KubernetesSpec{ + Version: "v1.35.0", + NodeLabels: map[string]string{"user-label": "original"}, + RegisterWithTaints: []string{"dedicated=metal:NoSchedule"}, + } + machine.Spec.Agent = &v1alpha3.AgentSpec{Image: "ghcr.io/test/agent:v1", Version: "v1.2.3"} + machine.Spec.PXE.CloudInit = &v1alpha3.CloudInitSpec{UserDataConfigMapRef: &v1alpha3.ConfigMapKeySelector{ + Name: "machine-session-user-data", Namespace: "default", + }} + op := testOperation("replace-session", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 2 + endpoint := readyTestEndpoint() + + cache := netboot.NewOCICache(t.TempDir()) + machineDigest := "sha256:" + stringOf('a', 64) + netbootDigest := "sha256:" + stringOf('b', 64) + + cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, machineDigest) + cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, netbootDigest) + require.NoError(t, writeSessionMetadata(cache, netbootDigest, "http/bootx64.efi")) + + s := testScheme(t) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(endpoint, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "machine-session-user-data", Namespace: "default"}, + Data: map[string]string{"user-data": "#cloud-config\nhostname: original\n"}, + }).WithStatusSubresource(&v1alpha3.NetbootSession{}).Build() + manager := &KubernetesSessionManager{ + Client: c, + Cache: cache, + Cluster: &netboot.StaticClusterInfo{Info: netboot.ClusterInfo{ApiserverURL: "https://api.example.com:6443", CACertBase64: "cluster-ca"}}, + KubernetesVersion: "v1.34.0", + ClusterDNS: "10.96.0.10", + ProviderLabels: map[string]string{"provider-label": "original"}, + Now: func() metav1.Time { return fixedNow() }, + } + + session, err := manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, v1alpha3.NetbootSessionPhaseReady, session.Status.Phase) + require.Equal(t, "sha256:"+stringOf('a', 64), session.Spec.Artifacts.MachineImage.Digest) + require.Equal(t, "sha256:"+stringOf('b', 64), session.Spec.Artifacts.NetbootImage.Digest) + require.Equal(t, machine.UID, session.Spec.Machine.UID) + require.Equal(t, op.UID, session.Spec.Operation.UID) + require.Equal(t, endpoint.Spec.ExternalURL, session.Spec.Endpoint.ExternalURL) + require.Equal(t, "http/bootx64.efi", session.Spec.Boot.FirmwareArtifact) + require.Equal(t, "https://api.example.com:6443", session.Spec.Provisioning.Cluster.APIServerURL) + require.Equal(t, "cluster-ca", session.Spec.Provisioning.Cluster.CACertBase64) + require.Equal(t, "10.96.0.10", session.Spec.Provisioning.Cluster.DNS) + require.Equal(t, "v1.34.0", session.Spec.Provisioning.Cluster.KubernetesVersion) + require.Equal(t, "original", session.Spec.Provisioning.Kubernetes.NodeLabels["user-label"]) + require.Equal(t, "ghcr.io/test/agent:v1", session.Spec.Provisioning.Agent.Image) + require.Equal(t, "original", session.Spec.Provisioning.ProviderLabels["provider-label"]) + require.Equal(t, "#cloud-config\nhostname: original\n", session.Spec.Provisioning.UserData) + require.Contains(t, session.Spec.Artifacts.Files, v1alpha3.NetbootSessionArtifact{ + Name: "http/bootx64.efi", + Source: "NetbootImage", + Path: "/disk/http/bootx64.efi", + }) + + for _, name := range []string{ + "vmlinuz", "initrd", "init.cpio", "grub/grub.cfg", "grubx64.efi", + "cloud-init/meta-data", "cloud-init/user-data", "cloud-init/vendor-data", "cloud-init/network-config", + } { + require.Condition(t, func() bool { + for _, artifact := range session.Spec.Artifacts.Files { + if artifact.Name == name { + return true + } + } + + return false + }, "session artifact list is missing %s", name) + } + + require.True(t, session.Spec.ExpiresAt.Time.Equal(fixedNow().Add(24*time.Hour))) + + machine.Spec.PXE.Image = "ghcr.io/test/changed:v2" + machine.Spec.Kubernetes.NodeLabels["user-label"] = "changed" + machine.Spec.Agent.Image = "ghcr.io/test/agent:changed" + manager.ProviderLabels["provider-label"] = "changed" + reused, err := manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, session.Name, reused.Name) + require.Equal(t, "ghcr.io/test/host:v1", reused.Spec.Artifacts.MachineImage.Reference) + require.Equal(t, "original", reused.Spec.Provisioning.Kubernetes.NodeLabels["user-label"]) + require.Equal(t, "ghcr.io/test/agent:v1", reused.Spec.Provisioning.Agent.Image) + require.Equal(t, "original", reused.Spec.Provisioning.ProviderLabels["provider-label"]) + + var sessions v1alpha3.NetbootSessionList + require.NoError(t, c.List(t.Context(), &sessions, client.MatchingLabels{sessionOperationUIDLabel: string(op.UID)})) + require.Len(t, sessions.Items, 1) +} + +func TestSessionManagerWaitsForImmutableDigests(t *testing.T) { + t.Parallel() + + machine := testBareMetalMachine("machine-pending", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 1 + machine.Spec.PXE.EndpointRef = "rack-a-edge" + machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" + op := testOperation("replace-pending", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 1 + + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(readyTestEndpoint()).WithStatusSubresource(&v1alpha3.NetbootSession{}).Build() + manager := &KubernetesSessionManager{Client: c, Cache: netboot.NewOCICache(t.TempDir())} + + _, err := manager.Ensure(t.Context(), op, machine) + require.ErrorIs(t, err, netboot.ErrNotYetDownloaded) + + var sessions v1alpha3.NetbootSessionList + require.NoError(t, c.List(t.Context(), &sessions)) + require.Empty(t, sessions.Items) +} + +func TestSessionManagerPromotesExistingSessionWhenEndpointBecomesReady(t *testing.T) { + t.Parallel() + + machine := testBareMetalMachine("machine-endpoint", "rack-a") + machine.UID = "machine-uid" + machine.Generation = 1 + machine.Spec.PXE.EndpointRef = "rack-a-edge" + machine.Spec.PXE.NetbootImage = "ghcr.io/test/netboot:v1" + op := testOperation("replace-endpoint", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Generation = 1 + endpoint := readyTestEndpoint() + endpoint.Status.Conditions[0].Status = metav1.ConditionFalse + + cache := netboot.NewOCICache(t.TempDir()) + cache.SetDigestForArchitecture(machine.Spec.PXE.Image, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('a', 64)) + cache.SetDigestForArchitecture(machine.Spec.PXE.NetbootImage, v1alpha3.DefaultPXEArchitecture, "sha256:"+stringOf('b', 64)) + require.NoError(t, writeSessionMetadata(cache, "sha256:"+stringOf('b', 64), "bootx64.efi")) + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(endpoint).WithStatusSubresource(endpoint, &v1alpha3.NetbootSession{}).Build() + manager := &KubernetesSessionManager{Client: c, Cache: cache} + + session, err := manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, v1alpha3.NetbootSessionPhasePreparing, session.Status.Phase) + + endpoint.Status.Conditions[0].Status = metav1.ConditionTrue + require.NoError(t, c.Status().Update(t.Context(), endpoint)) + + session, err = manager.Ensure(t.Context(), op, machine) + require.NoError(t, err) + require.Equal(t, v1alpha3.NetbootSessionPhaseReady, session.Status.Phase) +} + +func writeSessionMetadata(cache *netboot.OCICache, digest, httpBootPath string) error { + diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.DefaultPXEArchitecture) + if err := os.MkdirAll(diskDir, 0o755); err != nil { + return err + } + + return os.WriteFile(filepath.Join(diskDir, "metadata.yaml"), []byte("dhcpBootImageName: "+httpBootPath+"\nhttpBootPath: "+httpBootPath+"\n"), 0o600) +} + +func readyTestEndpoint() *v1alpha3.NetbootEndpoint { + return &v1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a-edge", UID: "endpoint-uid", Generation: 3}, + Spec: v1alpha3.NetbootEndpointSpec{ + SiteRef: "rack-a", + Type: v1alpha3.NetbootEndpointTypeExternalL2, + ExternalURL: "http://192.0.2.10:8880", + TLS: v1alpha3.NetbootEndpointTLS{ + Trust: v1alpha3.NetbootEndpointTrustTrustedLAN, + Mode: v1alpha3.NetbootEndpointTLSDisabled, + }, + }, + Status: v1alpha3.NetbootEndpointStatus{ + ObservedGeneration: 3, + Conditions: []metav1.Condition{{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "Available", + }}, + }, + } +} + +func stringOf(value byte, count int) string { + result := make([]byte, count) + for i := range result { + result[i] = value + } + + return string(result) +} diff --git a/internal/metalman/machineops/session_status.go b/internal/metalman/machineops/session_status.go new file mode 100644 index 000000000..80a4e4a8d --- /dev/null +++ b/internal/metalman/machineops/session_status.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +// SessionStatusRecorder durably records a server-observed milestone on the +// exact immutable session and its exact MachineOperation target. +type SessionStatusRecorder struct { + Client client.Client + Now func() metav1.Time +} + +func (r *SessionStatusRecorder) RecordCondition(ctx context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error { + if r == nil || r.Client == nil || sessionName == "" || sessionUID == "" { + return fmt.Errorf("session status recorder is not configured") + } + + var session v1alpha3.NetbootSession + if err := r.Client.Get(ctx, client.ObjectKey{Name: sessionName}, &session); err != nil { + return fmt.Errorf("get NetbootSession %s: %w", sessionName, err) + } + + if session.UID != sessionUID { + return fmt.Errorf("NetbootSession %s identity changed", sessionName) + } + + condition.ObservedGeneration = session.Generation + condition.LastTransitionTime = r.now() + + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var latest v1alpha3.NetbootSession + if err := r.Client.Get(ctx, client.ObjectKey{Name: sessionName}, &latest); err != nil { + return err + } + + if latest.UID != sessionUID { + return fmt.Errorf("NetbootSession %s identity changed", sessionName) + } + + if apimeta.IsStatusConditionTrue(latest.Status.Conditions, condition.Type) { + return nil + } + + apimeta.SetStatusCondition(&latest.Status.Conditions, condition) + + if latest.Status.Phase == v1alpha3.NetbootSessionPhaseReady { + latest.Status.Phase = v1alpha3.NetbootSessionPhaseActive + } + + return r.Client.Status().Update(ctx, &latest) + }); err != nil { + return fmt.Errorf("update NetbootSession %s status: %w", sessionName, err) + } + + return r.recordTargetCondition(ctx, &session, condition) +} + +func (r *SessionStatusRecorder) recordTargetCondition(ctx context.Context, session *v1alpha3.NetbootSession, condition metav1.Condition) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + var operation v1alpha3.MachineOperation + if err := r.Client.Get(ctx, client.ObjectKey{Name: session.Spec.Operation.Name}, &operation); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + if operation.UID != session.Spec.Operation.UID { + return fmt.Errorf("MachineOperation %s identity changed", operation.Name) + } + + for i := range operation.Status.Targets { + target := &operation.Status.Targets[i] + if target.Input == nil || target.Input.NetbootSessionRef == nil || target.Input.NetbootSessionRef.Name != session.Name || target.Input.NetbootSessionRef.UID != session.UID { + continue + } + + if apimeta.IsStatusConditionTrue(target.Conditions, condition.Type) { + return nil + } + + condition.ObservedGeneration = target.ObservedGeneration + apimeta.SetStatusCondition(&target.Conditions, condition) + + return r.Client.Status().Update(ctx, &operation) + } + + return fmt.Errorf("MachineOperation %s has no target for NetbootSession %s", operation.Name, session.Name) + }) +} + +func (r *SessionStatusRecorder) now() metav1.Time { + if r.Now != nil { + return r.Now() + } + + return metav1.Now() +} diff --git a/internal/metalman/machineops/session_status_test.go b/internal/metalman/machineops/session_status_test.go new file mode 100644 index 000000000..e6d7fd6cf --- /dev/null +++ b/internal/metalman/machineops/session_status_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package machineops + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +func TestSessionStatusRecorderUpdatesOnlyExactSessionAndTarget(t *testing.T) { + t.Parallel() + + sessionA := statusTestSession("session-a", "session-a-uid", "operation-uid", "machine-a") + sessionB := statusTestSession("session-b", "session-b-uid", "operation-uid", "machine-b") + op := testOperation("operation-a", v1alpha3.OperationHostReplace) + op.UID = "operation-uid" + op.Status.Phase = v1alpha3.OperationPhaseInProgress + op.Status.Targets = []v1alpha3.MachineOperationTargetStatus{ + {MachineRef: "machine-a", Phase: v1alpha3.OperationPhaseInProgress, Input: &v1alpha3.MachineOperationTargetInput{NetbootSessionRef: &v1alpha3.NetbootSessionReference{Name: sessionA.Name, UID: sessionA.UID}}}, + {MachineRef: "machine-b", Phase: v1alpha3.OperationPhaseInProgress, Input: &v1alpha3.MachineOperationTargetInput{NetbootSessionRef: &v1alpha3.NetbootSessionReference{Name: sessionB.Name, UID: sessionB.UID}}}, + } + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(sessionA, sessionB, op).WithStatusSubresource(sessionA, sessionB, op).Build() + recorder := &SessionStatusRecorder{Client: c, Now: fixedNow} + + require.NoError(t, recorder.RecordCondition(context.Background(), sessionA.Name, sessionA.UID, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionBootImageWritten, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "image written", + })) + + var updatedA v1alpha3.NetbootSession + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: sessionA.Name}, &updatedA)) + require.True(t, apimeta.IsStatusConditionTrue(updatedA.Status.Conditions, v1alpha3.NetbootSessionConditionBootImageWritten)) + + var updatedB v1alpha3.NetbootSession + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: sessionB.Name}, &updatedB)) + require.Nil(t, apimeta.FindStatusCondition(updatedB.Status.Conditions, v1alpha3.NetbootSessionConditionBootImageWritten)) + + var updatedOp v1alpha3.MachineOperation + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Name: op.Name}, &updatedOp)) + require.True(t, apimeta.IsStatusConditionTrue(updatedOp.Status.Targets[0].Conditions, v1alpha3.MachineOperationConditionBootImageWritten)) + require.Nil(t, apimeta.FindStatusCondition(updatedOp.Status.Targets[1].Conditions, v1alpha3.MachineOperationConditionBootImageWritten)) + require.Nil(t, apimeta.FindStatusCondition(updatedOp.Status.Conditions, v1alpha3.MachineOperationConditionBootImageWritten)) +} + +func TestSessionStatusRecorderRejectsStaleSessionUID(t *testing.T) { + t.Parallel() + + session := statusTestSession("session-a", "current-uid", "operation-uid", "machine-a") + c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(session).WithStatusSubresource(session).Build() + recorder := &SessionStatusRecorder{Client: c} + + err := recorder.RecordCondition(context.Background(), session.Name, types.UID("stale-uid"), metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionCloudInitDone, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + }) + require.ErrorContains(t, err, "identity changed") +} + +func statusTestSession(name string, uid, operationUID types.UID, machineName string) *v1alpha3.NetbootSession { + return &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: uid}, + Spec: v1alpha3.NetbootSessionSpec{ + Machine: v1alpha3.NetbootSessionObjectSnapshot{Name: machineName, UID: types.UID(machineName + "-uid"), Generation: 1}, + Operation: v1alpha3.NetbootSessionObjectSnapshot{Name: "operation-a", UID: operationUID, Generation: 1}, + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseActive}, + } +} diff --git a/internal/metalman/netboot/capability.go b/internal/metalman/netboot/capability.go new file mode 100644 index 000000000..944434c73 --- /dev/null +++ b/internal/metalman/netboot/capability.go @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +const minimumCapabilityKeySize = 32 + +type capabilityClaims struct { + KeyID string `json:"kid"` + Session string `json:"session"` + SessionUID string `json:"sessionUID"` + Expires int64 `json:"expires"` +} + +// CapabilitySigner issues operation-scoped bearer capabilities without +// persisting the bearer value in Kubernetes. +type CapabilitySigner struct { + key []byte + keyID string + now func() time.Time +} + +func NewCapabilitySigner(key []byte, keyID string, now func() time.Time) (*CapabilitySigner, error) { + if len(key) < minimumCapabilityKeySize { + return nil, fmt.Errorf("capability HMAC key must be at least %d bytes", minimumCapabilityKeySize) + } + + if strings.TrimSpace(keyID) == "" { + return nil, errors.New("capability key ID is required") + } + + if now == nil { + now = time.Now + } + + return &CapabilitySigner{key: append([]byte(nil), key...), keyID: keyID, now: now}, nil +} + +func (s *CapabilitySigner) Sign(session *v1alpha3.NetbootSession) (string, error) { + if session == nil || session.Name == "" || session.UID == "" { + return "", errors.New("session name and UID are required") + } + + claims := capabilityClaims{ + KeyID: s.keyID, + Session: session.Name, + SessionUID: string(session.UID), + Expires: session.Spec.ExpiresAt.Unix(), + } + + payload, err := json.Marshal(claims) + if err != nil { + return "", fmt.Errorf("marshal capability claims: %w", err) + } + + encoded := base64.RawURLEncoding.EncodeToString(payload) + + return encoded + "." + s.signature(encoded), nil +} + +func (s *CapabilitySigner) Verify(session *v1alpha3.NetbootSession, capability string) error { + encoded, signature, ok := strings.Cut(capability, ".") + if !ok || encoded == "" || signature == "" || !hmac.Equal([]byte(signature), []byte(s.signature(encoded))) { + return errors.New("invalid capability") + } + + payload, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return errors.New("invalid capability") + } + + var claims capabilityClaims + if err := json.Unmarshal(payload, &claims); err != nil { + return errors.New("invalid capability") + } + + if session == nil || claims.KeyID != s.keyID || claims.Session != session.Name || claims.SessionUID != string(session.UID) || claims.Expires != session.Spec.ExpiresAt.Unix() { + return errors.New("invalid capability") + } + + if !s.now().Before(time.Unix(claims.Expires, 0)) { + return errors.New("expired capability") + } + + return nil +} + +func (s *CapabilitySigner) signature(payload string) string { + mac := hmac.New(sha256.New, s.key) + _, _ = mac.Write([]byte(payload)) + + return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +func (s *CapabilitySigner) IsExpired(session *v1alpha3.NetbootSession) bool { + return session == nil || !s.now().Before(session.Spec.ExpiresAt.Time) +} diff --git a/internal/metalman/netboot/edge_auth.go b/internal/metalman/netboot/edge_auth.go new file mode 100644 index 000000000..da9fe57b1 --- /dev/null +++ b/internal/metalman/netboot/edge_auth.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "context" + "net/http" + "slices" + "strings" + + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1" +) + +const EdgeTokenAudience = "metalman-edge" + +// TokenReviewEdgeAuthenticator validates audience-bound Kubernetes ServiceAccount tokens. +type TokenReviewEdgeAuthenticator struct { + Client authenticationclient.AuthenticationV1Interface + ServiceAccountName string +} + +func (a *TokenReviewEdgeAuthenticator) Authenticate(ctx context.Context, request *http.Request) bool { + if a == nil || a.Client == nil || strings.TrimSpace(a.ServiceAccountName) == "" { + return false + } + + token, ok := strings.CutPrefix(request.Header.Get("Authorization"), "Bearer ") + if !ok || strings.TrimSpace(token) == "" { + return false + } + + review, err := a.Client.TokenReviews().Create(ctx, &authenticationv1.TokenReview{ + Spec: authenticationv1.TokenReviewSpec{Token: token, Audiences: []string{EdgeTokenAudience}}, + }, metav1.CreateOptions{}) + if err != nil || !review.Status.Authenticated || !slices.Contains(review.Status.Audiences, EdgeTokenAudience) { + return false + } + + prefix := "system:serviceaccount:" + if !strings.HasPrefix(review.Status.User.Username, prefix) { + return false + } + + parts := strings.Split(strings.TrimPrefix(review.Status.User.Username, prefix), ":") + + return len(parts) == 2 && parts[1] == a.ServiceAccountName +} diff --git a/internal/metalman/netboot/edge_auth_test.go b/internal/metalman/netboot/edge_auth_test.go new file mode 100644 index 000000000..64638080b --- /dev/null +++ b/internal/metalman/netboot/edge_auth_test.go @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "net/http/httptest" + "testing" + + authenticationv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" +) + +func TestTokenReviewEdgeAuthenticatorRequiresAudienceAndServiceAccount(t *testing.T) { + t.Parallel() + + clientset := fake.NewClientset() + clientset.PrependReactor("create", "tokenreviews", func(action clienttesting.Action) (bool, runtime.Object, error) { + create := action.(clienttesting.CreateAction) + + review := create.GetObject().(*authenticationv1.TokenReview) + if len(review.Spec.Audiences) != 1 || review.Spec.Audiences[0] != EdgeTokenAudience { + t.Errorf("audiences = %v", review.Spec.Audiences) + } + + return true, &authenticationv1.TokenReview{ + ObjectMeta: metav1.ObjectMeta{}, + Status: authenticationv1.TokenReviewStatus{ + Authenticated: true, + Audiences: []string{EdgeTokenAudience}, + User: authenticationv1.UserInfo{Username: "system:serviceaccount:unbounded-system:metalman-edge"}, + }, + }, nil + }) + authenticator := &TokenReviewEdgeAuthenticator{Client: clientset.AuthenticationV1(), ServiceAccountName: "metalman-edge"} + request := httptest.NewRequest("GET", "/", nil) + request.Header.Set("Authorization", "Bearer edge-token") + + if !authenticator.Authenticate(t.Context(), request) { + t.Fatal("expected edge token to authenticate") + } +} + +func TestTokenReviewEdgeAuthenticatorRejectsWrongAudience(t *testing.T) { + t.Parallel() + + clientset := fake.NewSimpleClientset() + clientset.PrependReactor("create", "tokenreviews", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, &authenticationv1.TokenReview{Status: authenticationv1.TokenReviewStatus{ + Authenticated: true, + Audiences: []string{"other"}, + User: authenticationv1.UserInfo{Username: "system:serviceaccount:unbounded-system:metalman-edge"}, + }}, nil + }) + authenticator := &TokenReviewEdgeAuthenticator{Client: clientset.AuthenticationV1(), ServiceAccountName: "metalman-edge"} + request := httptest.NewRequest("GET", "/", nil) + request.Header.Set("Authorization", "Bearer edge-token") + + if authenticator.Authenticate(t.Context(), request) { + t.Fatal("expected wrong audience to be rejected") + } +} diff --git a/internal/metalman/netboot/http.go b/internal/metalman/netboot/http.go index 6c3e9ee2f..b737681a2 100644 --- a/internal/metalman/netboot/http.go +++ b/internal/metalman/netboot/http.go @@ -159,7 +159,7 @@ func (h *HTTPServer) handleFile(w http.ResponseWriter, r *http.Request) { return } - if node.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP && + if node.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP && h.isHTTPBootLoaderDownload(imageRef, node.Spec.Netboot().TargetArchitecture(), path) { installRequested, err := h.installRequested(r.Context(), node) if err != nil { @@ -187,7 +187,7 @@ func (h *HTTPServer) handleFile(w http.ResponseWriter, r *http.Request) { return } - if node.Spec.Netboot().TargetBootProtocol() == v1alpha3.PXEBootProtocolHTTP && isOptionalShimRevocationsFile(path) { + if node.Spec.Netboot().TargetTransport() == v1alpha3.NetbootTransportHTTP && isOptionalShimRevocationsFile(path) { serveMissingShimRevocationsFile(w, log, node, path) return @@ -422,7 +422,7 @@ func (h *HTTPServer) recordBootImageWritten(ctx context.Context, log *slog.Logge } func (h *HTTPServer) recordHTTPBootLoaderDownloaded(ctx context.Context, log *slog.Logger, node *v1alpha3.Machine, imageRef, path string) { - if h.StatusRecorder == nil || node == nil || node.Spec.Netboot() == nil || node.Spec.Netboot().TargetBootProtocol() != v1alpha3.PXEBootProtocolHTTP { + if h.StatusRecorder == nil || node == nil || node.Spec.Netboot() == nil || node.Spec.Netboot().TargetTransport() != v1alpha3.NetbootTransportHTTP { return } diff --git a/internal/metalman/netboot/netboot.go b/internal/metalman/netboot/netboot.go index bb504552e..4c9b046e9 100644 --- a/internal/metalman/netboot/netboot.go +++ b/internal/metalman/netboot/netboot.go @@ -303,14 +303,18 @@ func (f *FileResolver) resolveUserDataFromConfigMap(ctx context.Context, node *v } type templateData struct { - Machine *v1alpha3.Machine - BootLease *v1alpha3.DHCPLease - ApiserverURL string - ServeURL string - AgentConfigJSON string - InstallScript string - InstallEnv []string - InstallRequested bool + Machine *v1alpha3.Machine + BootLease *v1alpha3.DHCPLease + ApiserverURL string + ServeURL string + ArtifactBaseURL string + BootImageWrittenURL string + CloudInitURL string + InstallLogURL string + AgentConfigJSON string + InstallScript string + InstallEnv []string + InstallRequested bool } func newTemplateData(node *v1alpha3.Machine, ci ClusterInfo, serveURL, agentConfigJSON, requestIP string, installRequested bool) templateData { @@ -324,15 +328,21 @@ func newTemplateData(node *v1alpha3.Machine, ci ClusterInfo, serveURL, agentConf bootLease = selectBootLease(node, requestIP) } + serveURL = strings.TrimRight(serveURL, "/") + return templateData{ - Machine: node, - BootLease: bootLease, - ApiserverURL: ci.ApiserverURL, - ServeURL: serveURL, - AgentConfigJSON: agentConfigJSON, - InstallScript: provision.UnboundedAgentInstallScript(), - InstallEnv: provision.AgentInstallEnv(agent), - InstallRequested: installRequested, + Machine: node, + BootLease: bootLease, + ApiserverURL: ci.ApiserverURL, + ServeURL: serveURL, + ArtifactBaseURL: serveURL, + BootImageWrittenURL: serveURL + "/pxe/disable", + CloudInitURL: serveURL + "/cloudinit/log", + InstallLogURL: serveURL + "/unbounded-agent/install-log", + AgentConfigJSON: agentConfigJSON, + InstallScript: provision.UnboundedAgentInstallScript(), + InstallEnv: provision.AgentInstallEnv(agent), + InstallRequested: installRequested, } } @@ -440,6 +450,23 @@ func indentTemplateBlock(spaces int, value string) string { } func renderTemplate(tmplStr string, data templateData) ([]byte, error) { + serveURL := strings.TrimRight(data.ServeURL, "/") + if data.ArtifactBaseURL == "" { + data.ArtifactBaseURL = serveURL + } + + if data.BootImageWrittenURL == "" { + data.BootImageWrittenURL = serveURL + "/pxe/disable" + } + + if data.CloudInitURL == "" { + data.CloudInitURL = serveURL + "/cloudinit/log" + } + + if data.InstallLogURL == "" { + data.InstallLogURL = serveURL + "/unbounded-agent/install-log" + } + t, err := template.New("").Funcs(templateFuncMap).Parse(tmplStr) if err != nil { return nil, fmt.Errorf("parsing template: %w", err) diff --git a/internal/metalman/netboot/netboot_test.go b/internal/metalman/netboot/netboot_test.go index 62c1fa9cb..e0c53b72b 100644 --- a/internal/metalman/netboot/netboot_test.go +++ b/internal/metalman/netboot/netboot_test.go @@ -237,9 +237,9 @@ func TestHTTPServer_HTTPBootLoaderRequiresActiveInstallOperation(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node-http"}, Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ - Image: "ghcr.io/test/image:v1", - BootProtocol: v1alpha3.PXEBootProtocolHTTP, - DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:10", IPv4: "10.0.1.60", SubnetMask: "255.255.255.0"}}, + Image: "ghcr.io/test/image:v1", + Transport: v1alpha3.NetbootTransportHTTP, + DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:10", IPv4: "10.0.1.60", SubnetMask: "255.255.255.0"}}, }, }, } @@ -314,9 +314,9 @@ func TestHTTPServer_MissingOptionalShimRevocationsFilesHTTPBoot(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "node-revocations"}, Spec: v1alpha3.MachineSpec{ PXE: &v1alpha3.PXESpec{ - Image: "ghcr.io/test/image:v1", - BootProtocol: v1alpha3.PXEBootProtocolHTTP, - DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:11", IPv4: "10.0.1.61", SubnetMask: "255.255.255.0"}}, + Image: "ghcr.io/test/image:v1", + Transport: v1alpha3.NetbootTransportHTTP, + DHCPLeases: []v1alpha3.DHCPLease{{MAC: "aa:bb:cc:dd:ee:11", IPv4: "10.0.1.61", SubnetMask: "255.255.255.0"}}, }, }, } diff --git a/internal/metalman/netboot/oci_cache.go b/internal/metalman/netboot/oci_cache.go index 5c9fe4d69..8f53137dd 100644 --- a/internal/metalman/netboot/oci_cache.go +++ b/internal/metalman/netboot/oci_cache.go @@ -193,6 +193,20 @@ func (c *OCICache) ResolvePathForArchitecture(imageRef, architecture, reqPath st return "", false, fmt.Errorf("image %q for architecture %q not yet pulled", imageRef, normalizeArchitecture(architecture)) } + return c.ResolveDigestPathForArchitecture(digest, architecture, reqPath) +} + +// ResolveDigestPathForArchitecture resolves a file from an immutable cached +// digest without consulting the mutable image-reference index. +func (c *OCICache) ResolveDigestPathForArchitecture(digest, architecture, reqPath string) (diskPath string, isTemplate bool, err error) { + if digest == "" { + return "", false, fmt.Errorf("image digest is required") + } + + if !c.IsCachedForArchitecture(digest, architecture) { + return "", false, fmt.Errorf("%w: image digest %q for architecture %q is not cached", ErrNotYetDownloaded, digest, normalizeArchitecture(architecture)) + } + // Reject absolute paths and Windows-style volume names. if filepath.IsAbs(reqPath) || filepath.VolumeName(reqPath) != "" { return "", false, fmt.Errorf("invalid request path %q: must be relative", reqPath) @@ -223,7 +237,7 @@ func (c *OCICache) ResolvePathForArchitecture(imageRef, architecture, reqPath st return cleanedBase, false, nil } - return "", false, fmt.Errorf("file not found in image %q: %s", imageRef, reqPath) + return "", false, fmt.Errorf("file not found in image digest %q: %s", digest, reqPath) } // InvalidateRef removes the digest mapping for an image reference, diff --git a/internal/metalman/netboot/session_dhcp_test.go b/internal/metalman/netboot/session_dhcp_test.go new file mode 100644 index 000000000..a337c6699 --- /dev/null +++ b/internal/metalman/netboot/session_dhcp_test.go @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/metalman/dhcp" + "github.com/Azure/unbounded/internal/metalman/netboot" +) + +func TestSessionDHCPDecisionUsesReadyImmutableSession(t *testing.T) { + t.Parallel() + + session := readyDHCPSession() + server := newSessionDHCPTestServer(t, session) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) + } + + var decision dhcp.Decision + if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { + t.Fatal(err) + } + + if decision.Lease.IPv4 != "10.0.1.20" { + t.Errorf("lease IP = %q", decision.Lease.IPv4) + } + + if decision.Transport != v1alpha3.NetbootTransportHTTP { + t.Errorf("transport = %q", decision.Transport) + } + + wantPrefix := "https://boot.example/v1/netboot/sessions/session-1/" + if len(decision.BootFile) <= len(wantPrefix) || decision.BootFile[:len(wantPrefix)] != wantPrefix { + t.Errorf("boot file = %q, want prefix %q", decision.BootFile, wantPrefix) + } +} + +func TestSessionDHCPDecisionRequiresEdgeAuthentication(t *testing.T) { + t.Parallel() + + server := newSessionDHCPTestServer(t, readyDHCPSession()) + server.EdgeAuthenticator = headerAuthenticator("Bearer edge-token") + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", response.Code, http.StatusUnauthorized) + } + + request = httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + request.Header.Set("Authorization", "Bearer edge-token") + + response = httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("authenticated status = %d, want %d", response.Code, http.StatusOK) + } +} + +func TestSessionDHCPDecisionRejectsAmbiguousReadySessions(t *testing.T) { + t.Parallel() + + first := readyDHCPSession() + second := first.DeepCopy() + second.Name = "session-2" + second.UID = types.UID("session-uid-2") + server := newSessionDHCPTestServer(t, first, second) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict) + } +} + +func TestSessionDHCPDecisionOmitsBootFileForRedfishConfiguration(t *testing.T) { + t.Parallel() + + session := readyDHCPSession() + session.Spec.Boot.ConfigurationSource = v1alpha3.NetbootConfigurationSourceRedfish + server := newSessionDHCPTestServer(t, session) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff?httpClient=true", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", response.Code, http.StatusOK) + } + + var decision dhcp.Decision + if err := json.NewDecoder(response.Body).Decode(&decision); err != nil { + t.Fatal(err) + } + + if decision.BootFile != "" { + t.Errorf("boot file = %q, want empty", decision.BootFile) + } +} + +func TestSessionDHCPDecisionIgnoresExpiredReadySession(t *testing.T) { + t.Parallel() + + session := readyDHCPSession() + session.Spec.ExpiresAt = metav1.NewTime(time.Unix(999, 0)) + server := newSessionDHCPTestServer(t, session) + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/endpoints/edge/dhcp/aa:bb:cc:dd:ee:ff", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", response.Code, http.StatusNotFound) + } +} + +func newSessionDHCPTestServer(t *testing.T, sessions ...*v1alpha3.NetbootSession) *netboot.SessionHTTPServer { + t.Helper() + + scheme := runtime.NewScheme() + if err := v1alpha3.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + objects := make([]runtime.Object, len(sessions)) + for i := range sessions { + objects[i] = sessions[i] + } + + clientObjects := make([]runtime.Object, 0, len(objects)) + clientObjects = append(clientObjects, objects...) + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(clientObjects...).Build() + + signer, err := netboot.NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test", func() time.Time { + return time.Unix(1000, 0) + }) + if err != nil { + t.Fatal(err) + } + + return &netboot.SessionHTTPServer{Client: client, Capabilities: signer} +} + +func readyDHCPSession() *v1alpha3.NetbootSession { + return &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: "session-1", UID: types.UID("session-uid-1")}, + Spec: v1alpha3.NetbootSessionSpec{ + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{Name: "edge", ExternalURL: "https://boot.example"}, + Boot: v1alpha3.NetbootSessionBoot{ + Transport: v1alpha3.NetbootTransportHTTP, + ConfigurationSource: v1alpha3.NetbootConfigurationSourceDHCP, + FirmwareArtifact: "shimx64.efi", + DHCPLeases: []v1alpha3.DHCPLease{{ + MAC: "aa:bb:cc:dd:ee:ff", + IPv4: "10.0.1.20", + SubnetMask: "255.255.255.0", + }}, + }, + Artifacts: v1alpha3.NetbootSessionArtifacts{Files: []v1alpha3.NetbootSessionArtifact{{Name: "shimx64.efi"}}}, + ExpiresAt: metav1.NewTime(time.Unix(2000, 0)), + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } +} + +type headerAuthenticator string + +func (h headerAuthenticator) Authenticate(_ context.Context, request *http.Request) bool { + return request.Header.Get("Authorization") == string(h) +} diff --git a/internal/metalman/netboot/session_http.go b/internal/metalman/netboot/session_http.go new file mode 100644 index 000000000..548c5597f --- /dev/null +++ b/internal/metalman/netboot/session_http.go @@ -0,0 +1,586 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + pathpkg "path" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/internal/provision" +) + +// SessionHTTPServer serves only immutable artifacts authorized by a +// session-scoped capability. +type SessionHTTPServer struct { + Client client.Reader + Cache *OCICache + Capabilities *CapabilitySigner + StatusRecorder SessionConditionRecorder + EdgeAuthenticator EdgeAuthenticator + Attestation SessionAttester +} + +// EdgeAuthenticator validates access to internal edge-only routes. +type EdgeAuthenticator interface { + Authenticate(ctx context.Context, request *http.Request) bool +} + +// SessionConditionRecorder persists a milestone for an exact session identity. +type SessionConditionRecorder interface { + RecordCondition(ctx context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error +} + +// SessionAttester performs TPM attestation for an exact authenticated Machine. +type SessionAttester interface { + AttestMachine(w http.ResponseWriter, r *http.Request, machine *v1alpha3.Machine) +} + +// SessionArtifactURL returns the externally advertised capability URL for one +// artifact listed by the immutable session. +func SessionArtifactURL(signer *CapabilitySigner, session *v1alpha3.NetbootSession, artifactName string) (string, error) { + if signer == nil || session == nil { + return "", errors.New("capability signer and session are required") + } + + if _, ok := sessionArtifact(session, artifactName); !ok { + return "", fmt.Errorf("artifact %q is not listed by session %s", artifactName, session.Name) + } + + cleanArtifact := strings.TrimPrefix(artifactName, "/") + if cleanArtifact == "" || pathpkg.Clean(cleanArtifact) != cleanArtifact || strings.HasPrefix(artifactName, "/") { + return "", fmt.Errorf("invalid artifact name %q", artifactName) + } + + baseURL, err := SessionBaseURL(signer, session) + if err != nil { + return "", err + } + + return JoinServeURLPath(baseURL, pathpkg.Join("artifacts", cleanArtifact)) +} + +// SessionBaseURL returns the externally advertised capability root for a +// session's artifacts and callbacks. +func SessionBaseURL(signer *CapabilitySigner, session *v1alpha3.NetbootSession) (string, error) { + if signer == nil || session == nil { + return "", errors.New("capability signer and session are required") + } + + capability, err := signer.Sign(session) + if err != nil { + return "", err + } + + return JoinServeURLPath(session.Spec.Endpoint.ExternalURL, pathpkg.Join("v1/netboot/sessions", session.Name, capability)) +} + +func (s *SessionHTTPServer) Handler() http.Handler { + mux := http.NewServeMux() + s.RegisterHandlers(mux) + + return mux +} + +// RegisterHandlers adds authenticated session artifact and callback routes. +func (s *SessionHTTPServer) RegisterHandlers(mux *http.ServeMux) { + mux.HandleFunc("GET /v1/netboot/sessions/{session}/{capability}/artifacts/{artifact...}", s.handleArtifact) + mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/callbacks/{milestone}", s.handleCallback) + mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/logs/agent-install", s.handleInstallLog) + mux.HandleFunc("POST /v1/netboot/sessions/{session}/{capability}/attest", s.handleAttest) + mux.HandleFunc("GET /v1/netboot/endpoints/{endpoint}/dhcp/{mac}", s.handleDHCPDecision) +} + +func (s *SessionHTTPServer) handleDHCPDecision(w http.ResponseWriter, r *http.Request) { + if s.Client == nil || s.Capabilities == nil { + http.Error(w, "session server unavailable", http.StatusServiceUnavailable) + return + } + + if s.EdgeAuthenticator != nil && !s.EdgeAuthenticator.Authenticate(r.Context(), r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + endpoint := r.PathValue("endpoint") + mac := strings.ToLower(r.PathValue("mac")) + + var sessions v1alpha3.NetbootSessionList + if err := s.Client.List(r.Context(), &sessions); err != nil { + http.Error(w, "loading sessions", http.StatusServiceUnavailable) + return + } + + matches := make([]*v1alpha3.NetbootSession, 0, 1) + + for i := range sessions.Items { + session := &sessions.Items[i] + if session.Spec.Endpoint.Name != endpoint || s.Capabilities.IsExpired(session) || (session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive) { + continue + } + + for _, lease := range session.Spec.Boot.DHCPLeases { + if strings.EqualFold(lease.MAC, mac) { + matches = append(matches, session) + break + } + } + } + + if len(matches) == 0 { + http.NotFound(w, r) + return + } + + if len(matches) != 1 { + http.Error(w, "multiple ready sessions match DHCP client", http.StatusConflict) + return + } + + session := matches[0] + + var lease *v1alpha3.DHCPLease + + for i := range session.Spec.Boot.DHCPLeases { + if strings.EqualFold(session.Spec.Boot.DHCPLeases[i].MAC, mac) { + lease = &session.Spec.Boot.DHCPLeases[i] + break + } + } + + if lease == nil { + http.NotFound(w, r) + return + } + + bootFile := "" + + if session.Spec.Boot.ConfigurationSource == v1alpha3.NetbootConfigurationSourceDHCP { + var err error + + bootFile, err = SessionArtifactURL(s.Capabilities, session, session.Spec.Boot.FirmwareArtifact) + if err != nil { + http.Error(w, "building firmware URL", http.StatusServiceUnavailable) + return + } + + if session.Spec.Boot.Transport == v1alpha3.NetbootTransportTFTP { + capability, err := s.Capabilities.Sign(session) + if err != nil { + http.Error(w, "building firmware capability", http.StatusServiceUnavailable) + return + } + + bootFile = pathpkg.Join("v1/netboot/sessions", session.Name, capability, "artifacts", session.Spec.Boot.FirmwareArtifact) + } + } + + w.Header().Set("Content-Type", "application/json") + + if err := json.NewEncoder(w).Encode(struct { + Lease v1alpha3.DHCPLease `json:"lease"` + Transport v1alpha3.NetbootTransport `json:"transport"` + BootFile string `json:"bootFile"` + }{Lease: *lease, Transport: session.Spec.Boot.Transport, BootFile: bootFile}); err != nil { + slog.Warn("encoding DHCP decision", "err", err) + } +} + +func (s *SessionHTTPServer) handleCallback(w http.ResponseWriter, r *http.Request) { + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + + if s.StatusRecorder == nil { + http.Error(w, "session status unavailable", http.StatusServiceUnavailable) + return + } + + milestone := r.PathValue("milestone") + if milestone == "cloud-init" { + s.handleSessionCloudInit(w, r, session) + return + } + + conditionType, ok := sessionMilestoneCondition(milestone) + if !ok { + http.NotFound(w, r) + return + } + + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, metav1.Condition{ + Type: conditionType, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "provisioning client reported milestone", + }); err != nil { + http.Error(w, "recording session status", http.StatusServiceUnavailable) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func (s *SessionHTTPServer) handleSessionCloudInit(w http.ResponseWriter, r *http.Request, session *v1alpha3.NetbootSession) { + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, "reading cloud-init event", http.StatusBadRequest) + return + } + + var event cloudInitEvent + if err := json.Unmarshal(body, &event); err != nil { + http.Error(w, "invalid cloud-init event", http.StatusBadRequest) + return + } + + condition := buildCloudInitCondition(&event, session.Spec.Machine.Generation) + if condition != nil { + condition.Type = v1alpha3.NetbootSessionConditionCloudInitDone + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, *condition); err != nil { + http.Error(w, "recording session status", http.StatusServiceUnavailable) + return + } + } + + w.WriteHeader(http.StatusNoContent) +} + +func (s *SessionHTTPServer) handleInstallLog(w http.ResponseWriter, r *http.Request) { + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, "reading install log", http.StatusBadRequest) + return + } + + slog.Warn("unbounded-agent install log", "session", session.Name, "body", strings.TrimSpace(string(body))) + w.WriteHeader(http.StatusNoContent) +} + +func (s *SessionHTTPServer) handleAttest(w http.ResponseWriter, r *http.Request) { + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + + if s.Attestation == nil || s.StatusRecorder == nil { + http.Error(w, "session attestation unavailable", http.StatusServiceUnavailable) + return + } + + var machine v1alpha3.Machine + if err := s.Client.Get(r.Context(), client.ObjectKey{Name: session.Spec.Machine.Name}, &machine); err != nil { + if apierrors.IsNotFound(err) { + http.NotFound(w, r) + return + } + + http.Error(w, "loading Machine", http.StatusServiceUnavailable) + + return + } + + if machine.UID != session.Spec.Machine.UID { + http.Error(w, "Machine identity changed", http.StatusConflict) + return + } + + response := newBufferedResponseWriter() + s.Attestation.AttestMachine(response, r, &machine) + + if response.status < http.StatusOK || response.status >= http.StatusMultipleChoices { + response.writeTo(w) + return + } + + if err := s.StatusRecorder.RecordCondition(r.Context(), session.Name, session.UID, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionAttested, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "TPM attestation succeeded", + }); err != nil { + http.Error(w, "recording session status", http.StatusServiceUnavailable) + return + } + + response.writeTo(w) +} + +func (s *SessionHTTPServer) handleArtifact(w http.ResponseWriter, r *http.Request) { + if s.Cache == nil { + http.Error(w, "session server unavailable", http.StatusServiceUnavailable) + return + } + + session, ok := s.authorizeSession(w, r) + if !ok { + return + } + + artifact, ok := sessionArtifact(session, r.PathValue("artifact")) + if !ok { + http.NotFound(w, r) + return + } + + if artifact.Source == "Session" { + if artifact.Name != "cloud-init/user-data" { + http.NotFound(w, r) + return + } + + http.ServeContent(w, r, artifact.Name, session.CreationTimestamp.Time, strings.NewReader(session.Spec.Provisioning.UserData)) + + return + } + + image, ok := sessionArtifactImage(session, artifact.Source) + if !ok { + http.NotFound(w, r) + return + } + + reqPath := strings.TrimPrefix(artifact.Path, "/disk/") + if reqPath == artifact.Path { + http.NotFound(w, r) + return + } + + diskPath, isTemplate, err := s.Cache.ResolveDigestPathForArchitecture(image.Digest, session.Spec.Boot.Architecture, reqPath) + if err != nil { + if errors.Is(err, ErrNotYetDownloaded) { + w.Header().Set("Retry-After", "5") + http.Error(w, "artifact unavailable", http.StatusServiceUnavailable) + + return + } + + http.NotFound(w, r) + + return + } + + if isTemplate { + data, err := s.renderSessionTemplate(diskPath, session) + if err != nil { + http.Error(w, "rendering artifact", http.StatusServiceUnavailable) + return + } + + http.ServeContent(w, r, artifact.Name, session.CreationTimestamp.Time, bytes.NewReader(data)) + s.recordFirmwareDownloaded(r.Context(), session, artifact.Name) + + return + } + + http.ServeFile(w, r, diskPath) + s.recordFirmwareDownloaded(r.Context(), session, artifact.Name) +} + +func (s *SessionHTTPServer) recordFirmwareDownloaded(ctx context.Context, session *v1alpha3.NetbootSession, artifactName string) { + if s.StatusRecorder == nil || artifactName != session.Spec.Boot.FirmwareArtifact { + return + } + + if err := s.StatusRecorder.RecordCondition(ctx, session.Name, session.UID, metav1.Condition{ + Type: v1alpha3.NetbootSessionConditionBootLoaderDownloaded, + Status: metav1.ConditionTrue, + Reason: "Succeeded", + Message: "firmware artifact downloaded", + }); err != nil { + slog.Warn("recording firmware artifact download", "session", session.Name, "err", err) + } +} + +func (s *SessionHTTPServer) renderSessionTemplate(templatePath string, session *v1alpha3.NetbootSession) ([]byte, error) { + content, err := os.ReadFile(templatePath) + if err != nil { + return nil, fmt.Errorf("reading template: %w", err) + } + + baseURL, err := SessionBaseURL(s.Capabilities, session) + if err != nil { + return nil, err + } + + machine := sessionMachine(session) + cluster := session.Spec.Provisioning.Cluster + agentConfig := provision.BuildAgentConfig(provision.BuildAgentConfigParams{ + Machine: machine, + Cluster: provision.ClusterEndpoint{ + APIServer: cluster.APIServerURL, + CACertBase64: cluster.CACertBase64, + ClusterDNS: cluster.DNS, + KubeVersion: cluster.KubernetesVersion, + }, + ProviderLabels: session.Spec.Provisioning.ProviderLabels, + AttestURL: baseURL, + }) + + agentConfigJSON, err := json.MarshalIndent(agentConfig, " ", " ") + if err != nil { + return nil, fmt.Errorf("marshaling agent config: %w", err) + } + + data := newTemplateData(machine, ClusterInfo{ApiserverURL: cluster.APIServerURL, CACertBase64: cluster.CACertBase64}, baseURL, string(agentConfigJSON), "", true) + + data.ArtifactBaseURL, err = JoinServeURLPath(baseURL, "artifacts") + if err != nil { + return nil, err + } + + data.BootImageWrittenURL, err = JoinServeURLPath(baseURL, "callbacks/boot-image-written") + if err != nil { + return nil, err + } + + data.CloudInitURL, err = JoinServeURLPath(baseURL, "callbacks/cloud-init") + if err != nil { + return nil, err + } + + data.InstallLogURL, err = JoinServeURLPath(baseURL, "logs/agent-install") + if err != nil { + return nil, err + } + + return renderTemplate(string(content), data) +} + +func sessionMachine(session *v1alpha3.NetbootSession) *v1alpha3.Machine { + return &v1alpha3.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: session.Spec.Machine.Name, UID: session.Spec.Machine.UID, Generation: session.Spec.Machine.Generation}, + Spec: v1alpha3.MachineSpec{ + Host: &v1alpha3.HostSpec{Netboot: &v1alpha3.PXESpec{ + Transport: session.Spec.Boot.Transport, + ConfigurationSource: session.Spec.Boot.ConfigurationSource, + NetworkMode: session.Spec.Boot.NetworkMode, + Architecture: session.Spec.Boot.Architecture, + DHCPLeases: append([]v1alpha3.DHCPLease(nil), session.Spec.Boot.DHCPLeases...), + TargetDisk: session.Spec.Boot.TargetDisk, + }}, + Kubernetes: session.Spec.Provisioning.Kubernetes.DeepCopy(), + Agent: session.Spec.Provisioning.Agent.DeepCopy(), + }, + } +} + +func (s *SessionHTTPServer) authorizeSession(w http.ResponseWriter, r *http.Request) (*v1alpha3.NetbootSession, bool) { + if s.Client == nil || s.Capabilities == nil { + http.Error(w, "session server unavailable", http.StatusServiceUnavailable) + return nil, false + } + + var session v1alpha3.NetbootSession + if err := s.Client.Get(r.Context(), client.ObjectKey{Name: r.PathValue("session")}, &session); err != nil { + if apierrors.IsNotFound(err) { + http.NotFound(w, r) + return nil, false + } + + http.Error(w, "loading session", http.StatusServiceUnavailable) + + return nil, false + } + + if err := s.Capabilities.Verify(&session, r.PathValue("capability")); err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return nil, false + } + + if session.Status.Phase != v1alpha3.NetbootSessionPhaseReady && session.Status.Phase != v1alpha3.NetbootSessionPhaseActive { + http.Error(w, "session unavailable", http.StatusServiceUnavailable) + return nil, false + } + + return &session, true +} + +func sessionMilestoneCondition(milestone string) (string, bool) { + switch milestone { + case "boot-loader-downloaded": + return v1alpha3.NetbootSessionConditionBootLoaderDownloaded, true + case "boot-image-written": + return v1alpha3.NetbootSessionConditionBootImageWritten, true + case "cloud-init-done": + return v1alpha3.NetbootSessionConditionCloudInitDone, true + default: + return "", false + } +} + +type bufferedResponseWriter struct { + header http.Header + body bytes.Buffer + status int +} + +func newBufferedResponseWriter() *bufferedResponseWriter { + return &bufferedResponseWriter{header: make(http.Header), status: http.StatusOK} +} + +func (w *bufferedResponseWriter) Header() http.Header { + return w.header +} + +func (w *bufferedResponseWriter) WriteHeader(status int) { + w.status = status +} + +func (w *bufferedResponseWriter) Write(data []byte) (int, error) { + return w.body.Write(data) +} + +func (w *bufferedResponseWriter) writeTo(destination http.ResponseWriter) { + for key, values := range w.header { + destination.Header()[key] = append([]string(nil), values...) + } + + destination.WriteHeader(w.status) + + if _, err := destination.Write(w.body.Bytes()); err != nil { + slog.Warn("writing buffered attestation response", "err", err) + } +} + +func sessionArtifact(session *v1alpha3.NetbootSession, name string) (v1alpha3.NetbootSessionArtifact, bool) { + for _, artifact := range session.Spec.Artifacts.Files { + if artifact.Name == name { + return artifact, true + } + } + + return v1alpha3.NetbootSessionArtifact{}, false +} + +func sessionArtifactImage(session *v1alpha3.NetbootSession, source string) (v1alpha3.NetbootSessionImage, bool) { + switch source { + case "MachineImage": + return session.Spec.Artifacts.MachineImage, true + case "NetbootImage": + return session.Spec.Artifacts.NetbootImage, true + default: + return v1alpha3.NetbootSessionImage{}, false + } +} diff --git a/internal/metalman/netboot/session_http_test.go b/internal/metalman/netboot/session_http_test.go new file mode 100644 index 000000000..de31688ae --- /dev/null +++ b/internal/metalman/netboot/session_http_test.go @@ -0,0 +1,406 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" +) + +func TestSessionHTTPServesImmutableArtifactWithCapabilityAndRange(t *testing.T) { + t.Parallel() + + const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + cache := setupOCICache(t, "unused.example/image:latest", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", map[string][]byte{ + "disk.img.gz": []byte("0123456789"), + }) + require.NoError(t, populateOCICache(cache.CacheDir, "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", map[string][]byte{ + "disk.img.gz": []byte("mutable-tag-content"), + })) + cache.SetDigest("unused.example/image:latest", "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") + + session := testNetbootSession("session-a", digest) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-a/"+capability+"/artifacts/disk.img.gz", nil) + request.Header.Set("Range", "bytes=2-5") + + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusPartialContent, response.Code) + require.Equal(t, "bytes 2-5/10", response.Header().Get("Content-Range")) + require.Equal(t, "2345", response.Body.String()) +} + +func TestSessionHTTPRejectsInvalidExpiredAndUnlistedCapabilities(t *testing.T) { + t.Parallel() + + const digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + cache := setupOCICache(t, "unused.example/image:latest", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", map[string][]byte{ + "disk.img.gz": []byte("disk"), + "secret": []byte("secret"), + }) + session := testNetbootSession("session-b", digest) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + now := time.Unix(1_700_000_000, 0) + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { return now }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() + + for name, test := range map[string]struct { + path string + wantStatus int + }{ + "invalid capability": {path: "/v1/netboot/sessions/session-b/not-a-capability/artifacts/disk.img.gz", wantStatus: http.StatusUnauthorized}, + "tampered capability": {path: "/v1/netboot/sessions/session-b/" + capability + "x/artifacts/disk.img.gz", wantStatus: http.StatusUnauthorized}, + "unlisted artifact": {path: "/v1/netboot/sessions/session-b/" + capability + "/artifacts/secret", wantStatus: http.StatusNotFound}, + } { + t.Run(name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, test.path, nil)) + require.Equal(t, test.wantStatus, response.Code) + }) + } + + now = session.Spec.ExpiresAt.Add(time.Second) + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-b/"+capability+"/artifacts/disk.img.gz", nil)) + require.Equal(t, http.StatusUnauthorized, response.Code) + body, err := io.ReadAll(response.Result().Body) + require.NoError(t, err) + require.NotContains(t, string(body), capability) +} + +func TestSessionHTTPRecordsAuthenticatedSessionCallback(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-c", "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + recorder := &recordingSessionConditionRecorder{} + handler := (&SessionHTTPServer{Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, StatusRecorder: recorder}).Handler() + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/v1/netboot/sessions/session-c/"+capability+"/callbacks/boot-image-written", nil)) + + require.Equal(t, http.StatusNoContent, response.Code) + require.Equal(t, session.Name, recorder.sessionName) + require.Equal(t, session.UID, recorder.sessionUID) + require.Equal(t, v1alpha3.NetbootSessionConditionBootImageWritten, recorder.condition.Type) + require.Equal(t, metav1.ConditionTrue, recorder.condition.Status) +} + +func TestSessionHTTPRecordsCloudInitCompletionOnlyForFinalSuccess(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-cloud-init", "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd") + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + recorder := &recordingSessionConditionsRecorder{} + handler := (&SessionHTTPServer{Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, StatusRecorder: recorder}).Handler() + path := "/v1/netboot/sessions/session-cloud-init/" + capability + "/callbacks/cloud-init" + + for _, body := range []string{ + `{"event_type":"start","name":"modules-final","description":"running"}`, + `{"event_type":"finish","name":"modules-config","description":"done","result":"SUCCESS"}`, + `{"event_type":"finish","name":"modules-final","description":"done","result":"SUCCESS"}`, + } { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))) + require.Equal(t, http.StatusNoContent, response.Code) + } + + require.Len(t, recorder.conditions, 3) + require.Equal(t, metav1.ConditionFalse, recorder.conditions[0].Status) + require.Equal(t, metav1.ConditionFalse, recorder.conditions[1].Status) + require.Equal(t, metav1.ConditionTrue, recorder.conditions[2].Status) + + for _, condition := range recorder.conditions { + require.Equal(t, v1alpha3.NetbootSessionConditionCloudInitDone, condition.Type) + } +} + +func TestSessionHTTPAttestsExactSessionMachineAndRecordsMilestone(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-attest", "sha256:dededededededededededededededededededededededededededededededede") + machine := &v1alpha3.Machine{ObjectMeta: metav1.ObjectMeta{Name: session.Spec.Machine.Name, UID: session.Spec.Machine.UID}} + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session, machine).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + attester := &recordingSessionAttester{} + recorder := &recordingSessionConditionRecorder{} + handler := (&SessionHTTPServer{ + Client: client, Cache: NewOCICache(t.TempDir()), Capabilities: signer, + StatusRecorder: recorder, Attestation: attester, + }).Handler() + + request := httptest.NewRequest(http.MethodPost, "/v1/netboot/sessions/session-attest/"+capability+"/attest", strings.NewReader(`{"ekPub":"a2V5","srkPub":"a2V5"}`)) + request.RemoteAddr = "198.51.100.25:12345" + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code) + require.Equal(t, machine.Name, attester.machine.Name) + require.Equal(t, machine.UID, attester.machine.UID) + require.Equal(t, v1alpha3.NetbootSessionConditionAttested, recorder.condition.Type) + require.Equal(t, session.UID, recorder.sessionUID) +} + +func TestSessionHTTPRecordsFirmwareDownloadForExactSession(t *testing.T) { + t.Parallel() + + const digest = "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + + cache := setupOCICache(t, "unused.example/netboot:latest", strings.TrimPrefix(digest, "sha256:"), map[string][]byte{ + "bootx64.efi": []byte("firmware"), + }) + session := testNetbootSession("session-firmware", digest) + session.Spec.Boot.FirmwareArtifact = "bootx64.efi" + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, v1alpha3.NetbootSessionArtifact{ + Name: "bootx64.efi", Source: "NetbootImage", Path: "/disk/bootx64.efi", + }) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + recorder := &recordingSessionConditionRecorder{} + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-firmware/"+capability+"/artifacts/bootx64.efi", nil) + (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer, StatusRecorder: recorder}).Handler().ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code) + require.Equal(t, session.Name, recorder.sessionName) + require.Equal(t, session.UID, recorder.sessionUID) + require.Equal(t, v1alpha3.NetbootSessionConditionBootLoaderDownloaded, recorder.condition.Type) +} + +func TestSessionArtifactURLUsesEndpointAndCapability(t *testing.T) { + t.Parallel() + + session := testNetbootSession("session-url", "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd") + session.Spec.Endpoint.ExternalURL = "https://boot.example.com/base/" + session.Spec.Boot.FirmwareArtifact = "http/bootx64.efi" + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, v1alpha3.NetbootSessionArtifact{ + Name: "http/bootx64.efi", Source: "NetbootImage", Path: "/disk/http/bootx64.efi", + }) + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + bootURL, err := SessionArtifactURL(signer, session, session.Spec.Boot.FirmwareArtifact) + require.NoError(t, err) + require.Equal(t, "https://boot.example.com/base/v1/netboot/sessions/session-url/"+capability+"/artifacts/http/bootx64.efi", bootURL) +} + +func TestSessionHTTPRendersBootArtifactsFromImmutableSnapshot(t *testing.T) { + t.Parallel() + + const digest = "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + + cache := NewOCICache(t.TempDir()) + diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.PXEArchitectureAMD64) + require.NoError(t, os.MkdirAll(filepath.Join(diskDir, "grub"), 0o755)) + templateContent, err := os.ReadFile(filepath.Join("..", "..", "..", "images", "netboot", "assets", "grub.cfg.tmpl")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(diskDir, "grub", "grub.cfg.tmpl"), templateContent, 0o600)) + + session := testNetbootSession("session-render", digest) + session.Spec.Boot = v1alpha3.NetbootSessionBoot{ + Architecture: v1alpha3.PXEArchitectureAMD64, + TargetDisk: "/dev/sda", + DHCPLeases: []v1alpha3.DHCPLease{{ + MAC: "aa:bb:cc:dd:ee:ff", IPv4: "192.0.2.20", SubnetMask: "255.255.255.0", Gateway: "192.0.2.1", + }}, + } + session.Spec.Provisioning = v1alpha3.NetbootSessionProvisioning{ + Cluster: v1alpha3.NetbootSessionCluster{APIServerURL: "https://api.snapshot.example:6443"}, + } + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, + v1alpha3.NetbootSessionArtifact{Name: "grub/grub.cfg", Source: "NetbootImage", Path: "/disk/grub/grub.cfg"}, + v1alpha3.NetbootSessionArtifact{Name: "vmlinuz", Source: "NetbootImage", Path: "/disk/vmlinuz"}, + v1alpha3.NetbootSessionArtifact{Name: "initrd", Source: "NetbootImage", Path: "/disk/initrd"}, + v1alpha3.NetbootSessionArtifact{Name: "init.cpio", Source: "NetbootImage", Path: "/disk/init.cpio"}, + ) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-render/"+capability+"/artifacts/grub/grub.cfg", nil) + (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler().ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code) + body := response.Body.String() + capabilityBase := "https://boot.example.com/v1/netboot/sessions/session-render/" + capability + require.Contains(t, body, "linux "+capabilityBase+"/artifacts/vmlinuz") + require.Contains(t, body, "initrd "+capabilityBase+"/artifacts/initrd "+capabilityBase+"/artifacts/init.cpio") + require.Contains(t, body, "unbounded.image_url="+capabilityBase+"/artifacts/disk.img.gz") + require.Contains(t, body, "unbounded.serve_url="+capabilityBase) + require.Contains(t, body, "unbounded.ds_url="+capabilityBase+"/artifacts/cloud-init/") + require.Contains(t, body, "unbounded.apiserver_url=https://api.snapshot.example:6443") + require.Contains(t, body, "unbounded.disk=/dev/sda") +} + +func TestSessionHTTPRendersCapabilityScopedCallbacks(t *testing.T) { + t.Parallel() + + const digest = "sha256:abababababababababababababababababababababababababababababababab" + + cache := NewOCICache(t.TempDir()) + diskDir := cache.DiskDirForArchitecture(digest, v1alpha3.PXEArchitectureAMD64) + require.NoError(t, os.MkdirAll(filepath.Join(diskDir, "cloud-init"), 0o755)) + + for _, artifact := range []string{"grub.cfg", "vendor-data"} { + source, err := os.ReadFile(filepath.Join("..", "..", "..", "images", "netboot", "assets", artifact+".tmpl")) + require.NoError(t, err) + + destination := filepath.Join(diskDir, artifact+".tmpl") + if artifact == "vendor-data" { + destination = filepath.Join(diskDir, "cloud-init", artifact+".tmpl") + } + + require.NoError(t, os.WriteFile(destination, source, 0o600)) + } + + session := testNetbootSession("session-callbacks", digest) + session.Spec.Boot.Architecture = v1alpha3.PXEArchitectureAMD64 + session.Spec.Artifacts.Files = append(session.Spec.Artifacts.Files, + v1alpha3.NetbootSessionArtifact{Name: "grub.cfg", Source: "NetbootImage", Path: "/disk/grub.cfg"}, + v1alpha3.NetbootSessionArtifact{Name: "cloud-init/vendor-data", Source: "NetbootImage", Path: "/disk/cloud-init/vendor-data"}, + ) + client := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(session).Build() + signer, err := NewCapabilitySigner([]byte("01234567890123456789012345678901"), "test-key", func() time.Time { + return time.Unix(1_700_000_000, 0) + }) + require.NoError(t, err) + capability, err := signer.Sign(session) + require.NoError(t, err) + + handler := (&SessionHTTPServer{Client: client, Cache: cache, Capabilities: signer}).Handler() + capabilityBase := "https://boot.example.com/v1/netboot/sessions/session-callbacks/" + capability + + grubResponse := httptest.NewRecorder() + handler.ServeHTTP(grubResponse, httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-callbacks/"+capability+"/artifacts/grub.cfg", nil)) + require.Equal(t, http.StatusOK, grubResponse.Code) + require.Contains(t, grubResponse.Body.String(), "unbounded.boot_image_written_url="+capabilityBase+"/callbacks/boot-image-written") + require.NotContains(t, grubResponse.Body.String(), "/pxe/disable") + + vendorResponse := httptest.NewRecorder() + handler.ServeHTTP(vendorResponse, httptest.NewRequest(http.MethodGet, "/v1/netboot/sessions/session-callbacks/"+capability+"/artifacts/cloud-init/vendor-data", nil)) + require.Equal(t, http.StatusOK, vendorResponse.Code) + require.Contains(t, vendorResponse.Body.String(), "endpoint: "+capabilityBase+"/callbacks/cloud-init") + require.Contains(t, vendorResponse.Body.String(), `"URL": "`+capabilityBase+`"`) + require.Contains(t, vendorResponse.Body.String(), `"`+capabilityBase+`/logs/agent-install"`) +} + +type recordingSessionConditionRecorder struct { + sessionName string + sessionUID types.UID + condition metav1.Condition +} + +type recordingSessionConditionsRecorder struct { + conditions []metav1.Condition +} + +type recordingSessionAttester struct { + machine *v1alpha3.Machine +} + +func (a *recordingSessionAttester) AttestMachine(w http.ResponseWriter, _ *http.Request, machine *v1alpha3.Machine) { + a.machine = machine.DeepCopy() + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"credentialBlob":"YQ=="}`)) +} + +func (r *recordingSessionConditionsRecorder) RecordCondition(_ context.Context, _ string, _ types.UID, condition metav1.Condition) error { + r.conditions = append(r.conditions, condition) + + return nil +} + +func (r *recordingSessionConditionRecorder) RecordCondition(_ context.Context, sessionName string, sessionUID types.UID, condition metav1.Condition) error { + r.sessionName = sessionName + r.sessionUID = sessionUID + r.condition = condition + + return nil +} + +func testNetbootSession(name, digest string) *v1alpha3.NetbootSession { + return &v1alpha3.NetbootSession{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid")}, + Spec: v1alpha3.NetbootSessionSpec{ + Machine: v1alpha3.NetbootSessionObjectSnapshot{Name: "machine-a", UID: "machine-uid", Generation: 1}, + Operation: v1alpha3.NetbootSessionObjectSnapshot{Name: "operation-a", UID: "operation-uid", Generation: 1}, + Endpoint: v1alpha3.NetbootSessionEndpointSnapshot{Name: "endpoint-a", UID: "endpoint-uid", ExternalURL: "https://boot.example.com"}, + Boot: v1alpha3.NetbootSessionBoot{Architecture: v1alpha3.PXEArchitectureAMD64}, + Artifacts: v1alpha3.NetbootSessionArtifacts{ + MachineImage: v1alpha3.NetbootSessionImage{Reference: "unused.example/image:latest", Digest: digest}, + NetbootImage: v1alpha3.NetbootSessionImage{Reference: "unused.example/netboot:latest", Digest: digest}, + Files: []v1alpha3.NetbootSessionArtifact{{Name: "disk.img.gz", Source: "MachineImage", Path: "/disk/disk.img.gz"}}, + }, + ExpiresAt: metav1.NewTime(time.Unix(1_700_003_600, 0)), + }, + Status: v1alpha3.NetbootSessionStatus{Phase: v1alpha3.NetbootSessionPhaseReady}, + } +} diff --git a/internal/metalman/netboot/tftp.go b/internal/metalman/netboot/tftp.go index b268709da..a3e989039 100644 --- a/internal/metalman/netboot/tftp.go +++ b/internal/metalman/netboot/tftp.go @@ -20,8 +20,18 @@ import ( type TFTPServer struct { BindAddr string + Port int FileResolver StatusRecorder TFTPStatusRecorder + Backend TFTPBackend +} + +type TFTPBackend interface { + Open(ctx context.Context, filename string) (io.ReadCloser, error) +} + +type TFTPBackendStatusRecorder interface { + RecordBootLoaderDownloaded(ctx context.Context, filename string) error } type TFTPStatusRecorder interface { @@ -34,7 +44,12 @@ func (t *TFTPServer) Start(ctx context.Context) error { s := tftp.NewServer(t.readHandler, nil) s.SetAnticipate(0) - addr := net.JoinHostPort(t.BindAddr, "69") + port := t.Port + if port == 0 { + port = 69 + } + + addr := net.JoinHostPort(t.BindAddr, fmt.Sprint(port)) conn, err := net.ListenPacket("udp", addr) if err != nil { @@ -55,7 +70,31 @@ func (t *TFTPServer) readHandler(filename string, rf io.ReaderFrom) error { ctx := context.Background() ip := rf.(tftp.OutgoingTransfer).RemoteAddr().IP.String() //nolint:errcheck // Type is guaranteed by the tftp library. filename = strings.TrimPrefix(filename, "/") + log := slog.With("proto", "tftp", "filename", filename, "ip", ip) + if t.Backend != nil { + if !validSessionArtifactPath(filename) { + return fmt.Errorf("invalid session artifact path %q", filename) + } + + reader, err := t.Backend.Open(ctx, filename) + if err != nil { + return fmt.Errorf("opening backend artifact: %w", err) + } + defer reader.Close() //nolint:errcheck // Backend stream is no longer needed. + + if _, err := rf.ReadFrom(reader); err != nil { + return fmt.Errorf("transferring backend artifact: %w", err) + } + + if recorder, ok := t.Backend.(TFTPBackendStatusRecorder); ok { + if err := recorder.RecordBootLoaderDownloaded(ctx, filename); err != nil { + return fmt.Errorf("recording backend transfer: %w", err) + } + } + + return nil + } node, err := t.LookupNodeByIP(ctx, ip) if err != nil { @@ -112,6 +151,11 @@ func (t *TFTPServer) readHandler(filename string, rf io.ReaderFrom) error { return nil } +func validSessionArtifactPath(filename string) bool { + parts := strings.Split(filename, "/") + return len(parts) >= 7 && parts[0] == "v1" && parts[1] == "netboot" && parts[2] == "sessions" && parts[3] != "" && parts[4] != "" && parts[5] == "artifacts" && parts[6] != "" +} + func (t *TFTPServer) recordBootLoaderDownloaded(ctx context.Context, log *slog.Logger, node *v1alpha3.Machine, imageRef, filename string) { if node == nil || t.StatusRecorder == nil || !t.isInitialBootLoaderDownload(imageRef, node.Spec.Netboot().TargetArchitecture(), filename) { return diff --git a/internal/metalman/netboot/tftp_backend.go b/internal/metalman/netboot/tftp_backend.go new file mode 100644 index 000000000..70e41ce1b --- /dev/null +++ b/internal/metalman/netboot/tftp_backend.go @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + pathpkg "path" + "strings" +) + +const maxTFTPArtifactBackendAttempts = 3 + +// HTTPArtifactBackend streams immutable session artifacts from a Metalman +// server and resumes a truncated backend response with exact byte ranges. +type HTTPArtifactBackend struct { + backendURL *url.URL + client *http.Client +} + +func NewHTTPArtifactBackend(backendURL string, client *http.Client) (*HTTPArtifactBackend, error) { + backend, err := url.Parse(backendURL) + if err != nil { + return nil, fmt.Errorf("parsing artifact backend URL: %w", err) + } + + if (backend.Scheme != "http" && backend.Scheme != "https") || backend.Host == "" { + return nil, errors.New("artifact backend URL must use HTTP or HTTPS and include a host") + } + + if client == nil { + client = http.DefaultClient + } + + return &HTTPArtifactBackend{backendURL: backend, client: client}, nil +} + +func (b *HTTPArtifactBackend) Open(ctx context.Context, filename string) (io.ReadCloser, error) { + requestURL := *b.backendURL + requestURL.Path = pathpkg.Join(requestURL.Path, filename) + + reader := &resumingArtifactReader{ctx: ctx, client: b.client, url: requestURL.String()} + if err := reader.open(0, -1); err != nil { + return nil, err + } + + return reader, nil +} + +func (b *HTTPArtifactBackend) RecordBootLoaderDownloaded(ctx context.Context, filename string) error { + parts := strings.Split(strings.TrimPrefix(filename, "/"), "/") + if len(parts) < 7 || parts[5] != "artifacts" { + return fmt.Errorf("invalid session artifact path %q", filename) + } + + requestURL := *b.backendURL + requestURL.Path = pathpkg.Join(requestURL.Path, pathpkg.Join(parts[:5]...), "callbacks", "boot-loader-downloaded") + + request, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), nil) + if err != nil { + return fmt.Errorf("creating TFTP milestone request: %w", err) + } + + response, err := b.client.Do(request) + if err != nil { + return fmt.Errorf("reporting TFTP milestone: %w", err) + } + defer response.Body.Close() //nolint:errcheck // Response body is not reused. + + if response.StatusCode != http.StatusNoContent { + return fmt.Errorf("reporting TFTP milestone: backend returned %s", response.Status) + } + + return nil +} + +type resumingArtifactReader struct { + ctx context.Context + client *http.Client + url string + body io.ReadCloser + offset int64 + end int64 + attempts int +} + +func (r *resumingArtifactReader) Read(buffer []byte) (int, error) { + for { + n, err := r.body.Read(buffer) + r.offset += int64(n) + + truncated := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) + if !truncated || r.offset > r.end { + return n, err + } + + if n > 0 { + return n, nil + } + + if r.attempts >= maxTFTPArtifactBackendAttempts { + return 0, io.ErrUnexpectedEOF + } + + if err := r.open(r.offset, r.end); err != nil { + if r.attempts >= maxTFTPArtifactBackendAttempts { + return 0, err + } + } + } +} + +func (r *resumingArtifactReader) Close() error { + if r.body == nil { + return nil + } + + return r.body.Close() +} + +func (r *resumingArtifactReader) open(start, end int64) error { + r.attempts++ + + request, err := http.NewRequestWithContext(r.ctx, http.MethodGet, r.url, nil) + if err != nil { + return err + } + + if end >= 0 { + request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + } + + response, err := r.client.Do(request) + if err != nil { + return fmt.Errorf("requesting TFTP artifact backend: %w", err) + } + + wantStatus := http.StatusOK + if end >= 0 { + wantStatus = http.StatusPartialContent + } + + if response.StatusCode != wantStatus || response.ContentLength <= 0 { + response.Body.Close() //nolint:errcheck // Invalid response. + return fmt.Errorf("TFTP artifact backend returned %s", response.Status) + } + + if end < 0 { + end = response.ContentLength - 1 + } else if response.ContentLength != end-start+1 || response.Header.Get("Content-Range") != fmt.Sprintf("bytes %d-%d/%d", start, end, end+1) { + response.Body.Close() //nolint:errcheck // Invalid range response. + return errors.New("TFTP artifact backend returned a mismatched range") + } + + if r.body != nil { + r.body.Close() //nolint:errcheck // Previous response reached EOF. + } + + r.body = response.Body + r.end = end + + return nil +} diff --git a/internal/metalman/netboot/tftp_backend_test.go b/internal/metalman/netboot/tftp_backend_test.go new file mode 100644 index 000000000..388b0e1fc --- /dev/null +++ b/internal/metalman/netboot/tftp_backend_test.go @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package netboot + +import ( + "bytes" + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/pin/tftp/v3" +) + +func TestTFTPServerFetchesTokenizedSessionArtifactFromBackend(t *testing.T) { + t.Parallel() + + backend := &recordingTFTPBackend{data: []byte("firmware")} + server := &TFTPServer{Backend: backend} + transfer := &memoryOutgoingTransfer{} + filename := "v1/netboot/sessions/session-1/capability/artifacts/shimx64.efi" + + if err := server.readHandler(filename, transfer); err != nil { + t.Fatal(err) + } + + if backend.filename != filename { + t.Errorf("backend filename = %q, want %q", backend.filename, filename) + } + + if got := transfer.String(); got != "firmware" { + t.Errorf("transfer = %q", got) + } +} + +func TestTFTPServerRejectsLegacySourceIPFilenameWithBackend(t *testing.T) { + t.Parallel() + + backend := &recordingTFTPBackend{data: []byte("firmware")} + + server := &TFTPServer{Backend: backend} + if err := server.readHandler("shimx64.efi", &memoryOutgoingTransfer{}); err == nil { + t.Fatal("expected legacy filename to be rejected") + } + + if backend.filename != "" { + t.Errorf("backend filename = %q, want empty", backend.filename) + } +} + +func TestTFTPServerReportsCompletedSessionTransfer(t *testing.T) { + t.Parallel() + + backend := &recordingTFTPBackend{data: []byte("firmware")} + server := &TFTPServer{Backend: backend} + filename := "v1/netboot/sessions/session-1/capability/artifacts/shimx64.efi" + + if err := server.readHandler(filename, &memoryOutgoingTransfer{}); err != nil { + t.Fatal(err) + } + + if backend.completed != filename { + t.Errorf("completed filename = %q, want %q", backend.completed, filename) + } +} + +type recordingTFTPBackend struct { + filename string + completed string + data []byte +} + +func (b *recordingTFTPBackend) Open(_ context.Context, filename string) (io.ReadCloser, error) { + b.filename = filename + + return io.NopCloser(bytes.NewReader(b.data)), nil +} + +func (b *recordingTFTPBackend) RecordBootLoaderDownloaded(_ context.Context, filename string) error { + b.completed = filename + + return nil +} + +type memoryOutgoingTransfer struct { + bytes.Buffer +} + +func (m *memoryOutgoingTransfer) ReadFrom(reader io.Reader) (int64, error) { + return m.Buffer.ReadFrom(reader) +} + +func (m *memoryOutgoingTransfer) RemoteAddr() net.UDPAddr { + return net.UDPAddr{IP: net.ParseIP("10.0.1.20"), Port: 12345} +} + +func (m *memoryOutgoingTransfer) LocalIP() net.IP { + return net.ParseIP("10.0.1.254") +} + +func (m *memoryOutgoingTransfer) SetSize(int64) {} + +var _ tftp.OutgoingTransfer = (*memoryOutgoingTransfer)(nil) + +func TestHTTPArtifactBackendResumesTruncatedResponse(t *testing.T) { + t.Parallel() + + const artifact = "immutable-firmware" + + var requests atomic.Int32 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("Range = %q", got) + } + + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + http.Error(w, "unexpected", http.StatusInternalServerError) + } + })) + defer server.Close() + + backend, err := NewHTTPArtifactBackend(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + + reader, err := backend.Open(t.Context(), "v1/netboot/sessions/session/capability/artifacts/shimx64.efi") + if err != nil { + t.Fatal(err) + } + defer reader.Close() //nolint:errcheck // Test cleanup. + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + if got := string(data); got != artifact { + t.Errorf("artifact = %q, want %q", got, artifact) + } +} + +func TestHTTPArtifactBackendRetriesFailedResumeRequest(t *testing.T) { + t.Parallel() + + const artifact = "immutable-firmware" + + var requests atomic.Int32 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch requests.Add(1) { + case 1: + w.Header().Set("Content-Length", "18") + _, _ = io.WriteString(w, artifact[:9]) + case 2: + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijacking failed resume request: %v", err) + return + } + + _ = conn.Close() + case 3: + if got := r.Header.Get("Range"); got != "bytes=9-17" { + t.Errorf("Range = %q", got) + } + + w.Header().Set("Content-Length", "9") + w.Header().Set("Content-Range", "bytes 9-17/18") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, artifact[9:]) + default: + http.Error(w, "unexpected", http.StatusInternalServerError) + } + })) + defer server.Close() + + backend, err := NewHTTPArtifactBackend(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + + reader, err := backend.Open(t.Context(), "v1/netboot/sessions/session/capability/artifacts/shimx64.efi") + if err != nil { + t.Fatal(err) + } + defer reader.Close() //nolint:errcheck // Test cleanup. + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + if got := string(data); got != artifact { + t.Errorf("artifact = %q, want %q", got, artifact) + } + + if got := requests.Load(); got != 3 { + t.Errorf("backend requests = %d, want 3", got) + } +} + +func TestHTTPArtifactBackendReportsSessionBootLoaderMilestone(t *testing.T) { + t.Parallel() + + var callbackPath string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callbackPath = r.URL.Path + + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + backend, err := NewHTTPArtifactBackend(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + + artifactPath := "v1/netboot/sessions/session/capability/artifacts/shimx64.efi" + if err := backend.RecordBootLoaderDownloaded(t.Context(), artifactPath); err != nil { + t.Fatal(err) + } + + if want := "/v1/netboot/sessions/session/capability/callbacks/boot-loader-downloaded"; callbackPath != want { + t.Errorf("callback path = %q, want %q", callbackPath, want) + } +} diff --git a/cmd/unbounded-net-node/bootstrap_helpers.go b/internal/net/nodeagent/bootstrap_helpers.go similarity index 99% rename from cmd/unbounded-net-node/bootstrap_helpers.go rename to internal/net/nodeagent/bootstrap_helpers.go index fcd4af565..df5af285a 100644 --- a/cmd/unbounded-net-node/bootstrap_helpers.go +++ b/internal/net/nodeagent/bootstrap_helpers.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/bootstrap_helpers_test.go b/internal/net/nodeagent/bootstrap_helpers_test.go similarity index 99% rename from cmd/unbounded-net-node/bootstrap_helpers_test.go rename to internal/net/nodeagent/bootstrap_helpers_test.go index 0bc3a1e8c..b27fb0683 100644 --- a/cmd/unbounded-net-node/bootstrap_helpers_test.go +++ b/internal/net/nodeagent/bootstrap_helpers_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/bpf_status.go b/internal/net/nodeagent/bpf_status.go similarity index 99% rename from cmd/unbounded-net-node/bpf_status.go rename to internal/net/nodeagent/bpf_status.go index 0fae10870..27c2a5f10 100644 --- a/cmd/unbounded-net-node/bpf_status.go +++ b/internal/net/nodeagent/bpf_status.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "fmt" diff --git a/cmd/unbounded-net-node/encapsulation.go b/internal/net/nodeagent/encapsulation.go similarity index 99% rename from cmd/unbounded-net-node/encapsulation.go rename to internal/net/nodeagent/encapsulation.go index 1e6406b2c..4a8897fc0 100644 --- a/cmd/unbounded-net-node/encapsulation.go +++ b/internal/net/nodeagent/encapsulation.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( unboundednetv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" diff --git a/cmd/unbounded-net-node/encapsulation_test.go b/internal/net/nodeagent/encapsulation_test.go similarity index 99% rename from cmd/unbounded-net-node/encapsulation_test.go rename to internal/net/nodeagent/encapsulation_test.go index 5fc7b021d..eea9e5f82 100644 --- a/cmd/unbounded-net-node/encapsulation_test.go +++ b/internal/net/nodeagent/encapsulation_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "testing" diff --git a/internal/net/nodeagent/external.go b/internal/net/nodeagent/external.go new file mode 100644 index 000000000..4ad1ef17d --- /dev/null +++ b/internal/net/nodeagent/external.go @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodeagent + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/rest" +) + +// ExternalGatewayOptions identifies a temporary external gateway dataplane. +type ExternalGatewayOptions struct { + NodeName string + RuntimeDir string + RESTConfig *rest.Config +} + +func externalGatewayConfig(options ExternalGatewayOptions) (*config, error) { + if options.NodeName == "" { + return nil, fmt.Errorf("external gateway node name is required") + } + + if options.RuntimeDir == "" { + return nil, fmt.Errorf("external gateway runtime directory is required") + } + + if !filepath.IsAbs(options.RuntimeDir) { + return nil, fmt.Errorf("external gateway runtime directory must be absolute") + } + + cfg := defaultConfig() + cfg.NodeName = options.NodeName + cfg.CNIConfDir = filepath.Join(options.RuntimeDir, "cni") + cfg.WireGuardDir = filepath.Join(options.RuntimeDir, "wireguard") + cfg.StatusPushEnabled = false + cfg.StatusWSEnabled = false + cfg.KubeProxyHealthInterval = 0 + cfg.HealthPort = 0 + cfg.RouteTableID = 254 + cfg.RemoveConfigurationOnShutdown = true + cfg.PreferredPublicEncap = "WireGuard" + cfg.RESTConfig = options.RESTConfig + + return cfg, nil +} + +// RunExternalGateway runs the normal node dataplane with external-gateway-safe +// defaults until ctx is cancelled. +func RunExternalGateway(ctx context.Context, options ExternalGatewayOptions) error { + cfg, err := externalGatewayConfig(options) + if err != nil { + return err + } + + if err := os.MkdirAll(cfg.WireGuardDir, 0o700); err != nil { + return fmt.Errorf("create WireGuard runtime directory: %w", err) + } + + if err := os.MkdirAll(cfg.CNIConfDir, 0o700); err != nil { + return fmt.Errorf("create CNI runtime directory: %w", err) + } + + return run(ctx, cfg) +} diff --git a/internal/net/nodeagent/external_test.go b/internal/net/nodeagent/external_test.go new file mode 100644 index 000000000..79602d897 --- /dev/null +++ b/internal/net/nodeagent/external_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodeagent + +import ( + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/client-go/rest" +) + +func TestExternalGatewayConfigUsesSafeStandaloneDefaults(t *testing.T) { + restConfig := &rest.Config{Host: "https://api.example.test"} + cfg, err := externalGatewayConfig(ExternalGatewayOptions{ + NodeName: "bootstrap-gateway", + RuntimeDir: "/run/unbounded-netboot/bootstrap-gateway", + RESTConfig: restConfig, + }) + require.NoError(t, err) + require.Equal(t, "bootstrap-gateway", cfg.NodeName) + require.Equal(t, "/run/unbounded-netboot/bootstrap-gateway/wireguard", cfg.WireGuardDir) + require.Equal(t, "/run/unbounded-netboot/bootstrap-gateway/cni", cfg.CNIConfDir) + require.Same(t, restConfig, cfg.RESTConfig) + require.False(t, cfg.StatusPushEnabled) + require.False(t, cfg.StatusWSEnabled) + require.Zero(t, cfg.KubeProxyHealthInterval) + require.Zero(t, cfg.HealthPort) + require.Equal(t, 254, cfg.RouteTableID) + require.True(t, cfg.RemoveConfigurationOnShutdown) + require.Equal(t, "WireGuard", cfg.PreferredPublicEncap) +} + +func TestExternalGatewayConfigRequiresIdentityAndRuntimeDirectory(t *testing.T) { + _, err := externalGatewayConfig(ExternalGatewayOptions{RuntimeDir: "/run/unbounded-netboot/gateway"}) + require.ErrorContains(t, err, "node name") + + _, err = externalGatewayConfig(ExternalGatewayOptions{NodeName: "gateway"}) + require.ErrorContains(t, err, "runtime directory") + + _, err = externalGatewayConfig(ExternalGatewayOptions{ + NodeName: "gateway", + RuntimeDir: "relative/path", + }) + require.ErrorContains(t, err, "absolute") +} diff --git a/cmd/unbounded-net-node/gateway_routes.go b/internal/net/nodeagent/gateway_routes.go similarity index 99% rename from cmd/unbounded-net-node/gateway_routes.go rename to internal/net/nodeagent/gateway_routes.go index 73ec34e6b..2692dc88a 100644 --- a/cmd/unbounded-net-node/gateway_routes.go +++ b/internal/net/nodeagent/gateway_routes.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/gateway_routes_test.go b/internal/net/nodeagent/gateway_routes_test.go similarity index 99% rename from cmd/unbounded-net-node/gateway_routes_test.go rename to internal/net/nodeagent/gateway_routes_test.go index 6e6a5aff3..fde76fcbd 100644 --- a/cmd/unbounded-net-node/gateway_routes_test.go +++ b/internal/net/nodeagent/gateway_routes_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/kube_proxy_monitor.go b/internal/net/nodeagent/kube_proxy_monitor.go similarity index 99% rename from cmd/unbounded-net-node/kube_proxy_monitor.go rename to internal/net/nodeagent/kube_proxy_monitor.go index 9b701e846..12b2117ca 100644 --- a/cmd/unbounded-net-node/kube_proxy_monitor.go +++ b/internal/net/nodeagent/kube_proxy_monitor.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/kube_proxy_monitor_test.go b/internal/net/nodeagent/kube_proxy_monitor_test.go similarity index 99% rename from cmd/unbounded-net-node/kube_proxy_monitor_test.go rename to internal/net/nodeagent/kube_proxy_monitor_test.go index 26742b721..5fff00985 100644 --- a/cmd/unbounded-net-node/kube_proxy_monitor_test.go +++ b/internal/net/nodeagent/kube_proxy_monitor_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/link_stats_monitor.go b/internal/net/nodeagent/link_stats_monitor.go similarity index 99% rename from cmd/unbounded-net-node/link_stats_monitor.go rename to internal/net/nodeagent/link_stats_monitor.go index 569d30309..42a16c85b 100644 --- a/cmd/unbounded-net-node/link_stats_monitor.go +++ b/internal/net/nodeagent/link_stats_monitor.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/link_stats_monitor_test.go b/internal/net/nodeagent/link_stats_monitor_test.go similarity index 99% rename from cmd/unbounded-net-node/link_stats_monitor_test.go rename to internal/net/nodeagent/link_stats_monitor_test.go index c2196af4b..b1f80d807 100644 --- a/cmd/unbounded-net-node/link_stats_monitor_test.go +++ b/internal/net/nodeagent/link_stats_monitor_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "testing" diff --git a/cmd/unbounded-net-node/main_config_test.go b/internal/net/nodeagent/main_config_test.go similarity index 99% rename from cmd/unbounded-net-node/main_config_test.go rename to internal/net/nodeagent/main_config_test.go index a3af27346..183140180 100644 --- a/cmd/unbounded-net-node/main_config_test.go +++ b/internal/net/nodeagent/main_config_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "os" diff --git a/cmd/unbounded-net-node/main_test.go b/internal/net/nodeagent/main_test.go similarity index 99% rename from cmd/unbounded-net-node/main_test.go rename to internal/net/nodeagent/main_test.go index 3eef81cf1..9685bf821 100644 --- a/cmd/unbounded-net-node/main_test.go +++ b/internal/net/nodeagent/main_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/main_update_test.go b/internal/net/nodeagent/main_update_test.go similarity index 99% rename from cmd/unbounded-net-node/main_update_test.go rename to internal/net/nodeagent/main_update_test.go index 68633f98d..22463cd86 100644 --- a/cmd/unbounded-net-node/main_update_test.go +++ b/internal/net/nodeagent/main_update_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/metrics.go b/internal/net/nodeagent/metrics.go similarity index 99% rename from cmd/unbounded-net-node/metrics.go rename to internal/net/nodeagent/metrics.go index bffd758ee..3265058a4 100644 --- a/cmd/unbounded-net-node/metrics.go +++ b/internal/net/nodeagent/metrics.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "github.com/prometheus/client_golang/prometheus" diff --git a/internal/net/nodeagent/node.go b/internal/net/nodeagent/node.go new file mode 100644 index 000000000..0875b3d4b --- /dev/null +++ b/internal/net/nodeagent/node.go @@ -0,0 +1,795 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package nodeagent + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + configpkg "github.com/Azure/unbounded/internal/net/config" + "github.com/Azure/unbounded/internal/net/metrics" + unboundednetnetlink "github.com/Azure/unbounded/internal/net/netlink" + "github.com/Azure/unbounded/internal/version" +) + +// CNIConfig represents the CNI configuration file structure +type CNIConfig struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Plugins []PluginConf `json:"plugins"` +} + +// PluginConf represents a CNI plugin configuration +type PluginConf struct { + Type string `json:"type"` + Bridge string `json:"bridge,omitempty"` + IsGateway bool `json:"isGateway,omitempty"` + IsDefaultGW bool `json:"isDefaultGateway,omitempty"` + ForceAddress bool `json:"forceAddress,omitempty"` + IPMasq bool `json:"ipMasq,omitempty"` + HairpinMode bool `json:"hairpinMode,omitempty"` + MTU int `json:"mtu,omitempty"` + IPAM *IPAMConfig `json:"ipam,omitempty"` + Capabilities *Caps `json:"capabilities,omitempty"` +} + +// IPAMConfig represents the IPAM configuration +type IPAMConfig struct { + Type string `json:"type"` + Ranges [][]IPRange `json:"ranges,omitempty"` +} + +// IPRange represents an IP range for IPAM +type IPRange struct { + Subnet string `json:"subnet"` +} + +// Caps represents plugin capabilities +type Caps struct { + PortMappings bool `json:"portMappings,omitempty"` +} + +type config struct { + ConfigFile string + KubeconfigPath string + ApiserverURL string // Override Kubernetes API server URL (empty = use default) + NodeName string + CNIConfDir string + CNIConfFile string + BridgeName string + WireGuardDir string + WireGuardPort int + EnablePolicyRouting bool + MTU int + HealthPort int + InformerResyncPeriod time.Duration + StatusPushEnabled bool // Whether to push status to controller + StatusPushURL string // Controller URL for status push + StatusPushInterval time.Duration // Interval between status pushes to controller + StatusPushAPIServerInterval time.Duration // Interval between status pushes via aggregated API server + StatusPushDelta bool // Whether periodic HTTP pushes use deltas + StatusWSEnabled bool // Whether websocket push is enabled + StatusWSURL string // Controller websocket URL for status push + StatusWSAPIServerMode string // API server websocket mode: never, fallback, preferred + StatusWSAPIServerURL string // API server websocket URL for status push fallback + StatusWSAPIServerStartupDelay time.Duration // Delay before API server fallback is allowed after startup + StatusWSKeepaliveInterval time.Duration // Interval between websocket keepalive pings (0 disables keepalive) + StatusWSKeepaliveFailureCount int // Sequential websocket keepalive ping failures before reconnect + RemoveConfigurationOnShutdown bool // Remove all managed configuration (WireGuard, routes, masquerade, etc.) on shutdown + RemoveWireGuardOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown + CleanupNetlinkOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown + RemoveMasqueradeOnShutdown bool // Deprecated: use RemoveConfigurationOnShutdown + HealthCheckPort int // UDP port for health check probes (default 9997) + BaseMetric int // Base metric for programmed routes (default 1) + RouteTableID int // Route table ID for managed routes (default 252) + CriticalDeltaEvery time.Duration // Maximum critical delta publish frequency; changes are queued up to this interval for batching + StatsDeltaEvery time.Duration // Maximum statistics delta publish frequency; changes are queued up to this interval for batching + FullSyncEvery time.Duration // Forced full status sync interval; ensures controller has complete status periodically + GenevePort int // GENEVE UDP destination port (default 6081) + GeneveVNI int // GENEVE Virtual Network Identifier (default 1) + GeneveInterfaceName string // GENEVE shared tunnel interface name (default geneve0) + VXLANInterfaceName string // VXLAN shared tunnel interface name (default vxlan0) + IPIPInterfaceName string // IPIP shared tunnel interface name (default ipip0) + WireGuardInterfacePrefix string // Prefix for per-port WireGuard interfaces (default "wg"; per-peer name is ) + VXLANPort int // VXLAN UDP destination port (default 4789) + VXLANSrcPortLow int // VXLAN UDP source port range low (default 47891) + VXLANSrcPortHigh int // VXLAN UDP source port range high (default 47922) + PreferredPrivateEncap string // Preferred encap for private/internal networks (GENEVE, IPIP, VXLAN, WireGuard) + PreferredPublicEncap string // Preferred encap for public/external networks (WireGuard, IPIP, GENEVE, VXLAN) + HealthFlapMaxBackoff time.Duration // Maximum backoff duration for health check flap dampening + KubeProxyHealthInterval time.Duration // Interval between kube-proxy health checks (0 to disable) + NetlinkResyncPeriod time.Duration // Interval between full netlink cache resyncs + TunnelDataplaneMapSize int // Maximum LPM trie entries for eBPF tunnel map (default 16384) + TunnelIPFamily string // Tunnel underlay IP family: "IPv4" (default) or "IPv6" + RESTConfig *rest.Config // Explicit client configuration for embedded runners +} + +var siteGVR = schema.GroupVersionResource{ + Group: unboundedv1alpha3.GroupVersion.Group, + Version: unboundedv1alpha3.GroupVersion.Version, + Resource: "sites", +} + +var siteNodeSliceGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "sitenodeslices", +} + +var gatewayPoolGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "gatewaypools", +} + +var gatewayNodeGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "gatewaypoolnodes", +} + +var sitePeeringGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "sitepeerings", +} + +var siteGatewayPoolAssignmentGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "sitegatewaypoolassignments", +} + +var gatewayPoolPeeringGVR = schema.GroupVersionResource{ + Group: "net.unbounded-cloud.io", + Version: "v1alpha1", + Resource: "gatewaypoolpeerings", +} + +const ( + // WireGuard public key annotation on the node + WireGuardPubKeyAnnotation = "net.unbounded-cloud.io/wg-pubkey" + + // TunnelMTUAnnotation is the maximum tunnel MTU this node + // can support, based on its default-route interface MTU minus encapsulation + // overhead. The controller uses this to validate that the configured MTU + // does not exceed what any node in the cluster can handle. + TunnelMTUAnnotation = "net.unbounded-cloud.io/tunnel-mtu" + + // Gateway node taint key - prevents regular workloads from running on gateway nodes + // since they don't have regular pod CIDR routing + GatewayNodeTaintKey = "net.unbounded-cloud.io/gateway-node" + + // gatewayNodeHeartbeatInterval controls how frequently the node agent refreshes + // GatewayNode.status.lastUpdated. Route staleness is derived from this cadence. + gatewayNodeHeartbeatInterval = 10 * time.Second +) + +// NewCommand builds the unbounded-net node-agent command. +func NewCommand() *cobra.Command { + // Initialize klog flags + klog.InitFlags(nil) + + // Add klog flags to pflag + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + + cfg := defaultConfig() + if nodeName := os.Getenv("NODE_NAME"); nodeName != "" { + cfg.NodeName = nodeName + } + + rootCmd := &cobra.Command{ + Use: "unbounded-net-node", + Short: "CNI configuration agent for unbounded-net", + Long: `unbounded-net-node runs on each node as a DaemonSet and configures CNI +networking by writing a CNI configuration file based on the node's podCIDRs. + +It watches the node object in Kubernetes and waits for podCIDRs to be assigned +by the unbounded-net-controller. Once assigned, it writes a CNI configuration +file that sets up pod networking using the bridge plugin with host-local IPAM. + +It also generates WireGuard keys for the node and stores them in /etc/wireguard, +then annotates the node with the public key.`, + Version: version.Version + " (commit: " + version.GitCommit + ")", + SilenceUsage: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + return applyNodeRuntimeConfig(cmd, cfg) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return run(cmd.Context(), cfg) + }, + } + + // Add flags + flags := rootCmd.Flags() + + // Change version flag from -v to -V to avoid conflict with klog's -v flag + rootCmd.Flags().BoolP("version", "V", false, "Print version information") + rootCmd.SetVersionTemplate(`{{printf "%s\n" .Version}}`) + + // General flags + flags.StringVar(&cfg.ConfigFile, "config-file", "/etc/unbounded-net/config.yaml", "Path to runtime YAML config file") + flags.StringVar(&cfg.KubeconfigPath, "kubeconfig", "", "Path to kubeconfig file (uses in-cluster config if not specified)") + flags.StringVar(&cfg.ApiserverURL, "apiserver-url", "", "Override Kubernetes API server URL (empty = use default from kubeconfig or in-cluster config)") + flags.StringVar(&cfg.NodeName, "node-name", os.Getenv("NODE_NAME"), "Name of this node (defaults to NODE_NAME env var)") + flags.IntVar(&cfg.HealthPort, "health-port", 9998, "Port for health check HTTP server (0 to disable)") + flags.DurationVar(&cfg.InformerResyncPeriod, "informer-resync-period", 600*time.Second, "Resync period for Kubernetes informers") + + // CNI configuration flags + flags.StringVar(&cfg.CNIConfDir, "cni-conf-dir", "/etc/cni/net.d", "Directory to write CNI configuration") + flags.StringVar(&cfg.CNIConfFile, "cni-conf-file", "10-unbounded.conflist", "Name of the CNI configuration file") + flags.StringVar(&cfg.BridgeName, "bridge-name", "cbr0", "Name of the bridge interface") + flags.IntVar(&cfg.MTU, "mtu", 1280, "MTU for WireGuard and bridge interfaces (default 1280, the IPv6 minimum)") + + // WireGuard configuration flags + flags.StringVar(&cfg.WireGuardDir, "wireguard-dir", "/etc/wireguard", "Directory to store WireGuard keys") + flags.IntVar(&cfg.WireGuardPort, "wireguard-port", 51820, "WireGuard listen port") + flags.BoolVar(&cfg.EnablePolicyRouting, "enable-policy-routing", false, "Enable policy-based routing on gateway interfaces (deprecated, UNBOUNDED-FORWARD chain rules replace PBR)") + + // Tunnel-interface configuration flags. All three shared tunnel device + // names must be non-empty, distinct, and must not collide with + // "unbounded0" (the agent's eBPF dummy device). Kernel interface names + // are limited to 15 bytes. + flags.IntVar(&cfg.GenevePort, "geneve-port", 6081, "GENEVE UDP destination port") + flags.IntVar(&cfg.GeneveVNI, "geneve-vni", 1, "GENEVE Virtual Network Identifier") + flags.StringVar(&cfg.GeneveInterfaceName, "geneve-interface", "geneve0", "Shared flow-based GENEVE interface name") + flags.StringVar(&cfg.VXLANInterfaceName, "vxlan-interface", "vxlan0", "Shared flow-based VXLAN interface name") + flags.StringVar(&cfg.IPIPInterfaceName, "ipip-interface", "ipip0", "Shared flow-based IPIP interface name") + flags.StringVar(&cfg.WireGuardInterfacePrefix, "wireguard-interface-prefix", "wg", "Prefix for per-port WireGuard interfaces; runtime name is ") + flags.IntVar(&cfg.VXLANPort, "vxlan-port", 4789, "VXLAN UDP destination port") + flags.IntVar(&cfg.VXLANSrcPortLow, "vxlan-src-port-low", 47891, "VXLAN UDP source port range low (narrow range reduces VM flow count in cloud platforms)") + flags.IntVar(&cfg.VXLANSrcPortHigh, "vxlan-src-port-high", 47922, "VXLAN UDP source port range high (narrow range reduces VM flow count in cloud platforms)") + flags.StringVar(&cfg.PreferredPrivateEncap, "preferred-private-encap", "GENEVE", "Preferred encapsulation for private networks (GENEVE, IPIP, VXLAN, WireGuard)") + flags.StringVar(&cfg.PreferredPublicEncap, "preferred-public-encap", "WireGuard", "Preferred encapsulation for public networks (WireGuard, IPIP, GENEVE, VXLAN)") + + // Status push flags + flags.BoolVar(&cfg.StatusPushEnabled, "status-push-enabled", true, "Enable pushing node status to controller") + flags.StringVar(&cfg.StatusPushURL, "status-push-url", "", "Controller URL for status push (default: use UNBOUNDED_NET_CONTROLLER_SERVICE_HOST/PORT)") + flags.DurationVar(&cfg.StatusPushInterval, "status-push-interval", 60*time.Second, "Interval between status pushes to controller") + flags.DurationVar(&cfg.StatusPushAPIServerInterval, "status-push-apiserver-interval", 60*time.Second, "Interval between status pushes via aggregated API server") + flags.BoolVar(&cfg.StatusPushDelta, "status-push-delta", true, "Enable delta mode for periodic HTTP status push") + flags.BoolVar(&cfg.StatusWSEnabled, "status-ws-enabled", true, "Enable websocket status push to controller") + flags.StringVar(&cfg.StatusWSURL, "status-ws-url", "", "Controller websocket URL for status push (default: ws://service/status/nodews)") + flags.StringVar(&cfg.StatusWSAPIServerMode, "status-ws-apiserver-mode", statusWSAPIServerModeFallback, "API server websocket mode: never, fallback, preferred") + flags.StringVar(&cfg.StatusWSAPIServerURL, "status-ws-apiserver-url", "", "API server websocket URL for status push fallback (default: wss://$(KUBERNETES_SERVICE_HOST)/apis/status.net.unbounded-cloud.io/v1alpha1/status/nodews)") + flags.DurationVar(&cfg.StatusWSAPIServerStartupDelay, "status-ws-apiserver-startup-delay", 60*time.Second, "Delay before API server websocket/push fallback is allowed after startup (0 to disable delay)") + flags.DurationVar(&cfg.StatusWSKeepaliveInterval, "status-ws-keepalive-interval", 10*time.Second, "Interval between websocket keepalive pings (0 to disable)") + flags.IntVar(&cfg.StatusWSKeepaliveFailureCount, "status-ws-keepalive-failure-count", 2, "Sequential websocket keepalive ping failures before reconnect") + flags.BoolVar(&cfg.RemoveConfigurationOnShutdown, "remove-configuration-on-shutdown", false, "Remove all managed configuration (WireGuard, routes, masquerade, tunnel interfaces) on shutdown") + flags.BoolVar(&cfg.RemoveWireGuardOnShutdown, "shutdown-remove-wireguard-configuration", false, "Remove WireGuard interfaces/configuration on shutdown (deprecated: use --remove-configuration-on-shutdown)") + flags.BoolVar(&cfg.CleanupNetlinkOnShutdown, "shutdown-cleanup-netlink", false, "Remove managed netlink routes/policy rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") + flags.BoolVar(&cfg.RemoveMasqueradeOnShutdown, "shutdown-remove-masquerade-rules", false, "Remove managed masquerade rules on shutdown (deprecated: use --remove-configuration-on-shutdown)") + flags.IntVar(&cfg.HealthCheckPort, "healthcheck-port", 9997, "UDP port for health check probes") + flags.IntVar(&cfg.BaseMetric, "base-metric", 1, "Base metric for programmed routes") + flags.IntVar(&cfg.RouteTableID, "route-table-id", 252, "Route table ID for managed routes (default 252, set to 254 for main table)") + flags.DurationVar(&cfg.CriticalDeltaEvery, "status-critical-interval", 15*time.Second, "Maximum critical delta publish frequency; changed fields are queued up to this interval for batching") + flags.DurationVar(&cfg.StatsDeltaEvery, "status-stats-interval", 60*time.Second, "Maximum statistics delta publish frequency; changed fields are queued up to this interval for batching") + flags.DurationVar(&cfg.FullSyncEvery, "status-full-sync-interval", 2*time.Minute, "Forced full status sync interval; ensures controller has complete status periodically") + flags.DurationVar(&cfg.HealthFlapMaxBackoff, "health-flap-max-backoff", 120*time.Second, "Maximum backoff duration for health check flap dampening") + flags.DurationVar(&cfg.KubeProxyHealthInterval, "kube-proxy-health-interval", 30*time.Second, "Interval between kube-proxy health checks (0 to disable)") + flags.DurationVar(&cfg.NetlinkResyncPeriod, "netlink-resync-period", 300*time.Second, "Interval between full netlink cache resyncs") + flags.IntVar(&cfg.TunnelDataplaneMapSize, "tunnel-dataplane-map-size", 16384, "Maximum LPM trie entries for eBPF tunnel map") + flags.StringVar(&cfg.TunnelIPFamily, "tunnel-ip-family", "IPv4", "Tunnel underlay IP family: IPv4 (default) or IPv6") + + return rootCmd +} + +func defaultConfig() *config { + return &config{ + ConfigFile: "/etc/unbounded-net/config.yaml", + CNIConfDir: "/etc/cni/net.d", + CNIConfFile: "10-unbounded.conflist", + BridgeName: "cbr0", + WireGuardDir: "/etc/wireguard", + WireGuardPort: 51820, + MTU: 1280, + HealthPort: 9998, + InformerResyncPeriod: 600 * time.Second, + StatusPushEnabled: true, + StatusPushInterval: 10 * time.Second, + StatusPushAPIServerInterval: 30 * time.Second, + StatusPushDelta: true, + StatusWSEnabled: true, + StatusWSAPIServerMode: statusWSAPIServerModeFallback, + StatusWSAPIServerStartupDelay: 60 * time.Second, + StatusWSKeepaliveInterval: 10 * time.Second, + StatusWSKeepaliveFailureCount: 2, + CriticalDeltaEvery: time.Second, + StatsDeltaEvery: 15 * time.Second, + FullSyncEvery: 2 * time.Minute, + GenevePort: 6081, + GeneveVNI: 1, + GeneveInterfaceName: "geneve0", + VXLANInterfaceName: "vxlan0", + IPIPInterfaceName: "ipip0", + WireGuardInterfacePrefix: "wg", + VXLANPort: 4789, + VXLANSrcPortLow: 47891, + VXLANSrcPortHigh: 47922, + PreferredPrivateEncap: "GENEVE", + PreferredPublicEncap: "WireGuard", + NetlinkResyncPeriod: 300 * time.Second, + TunnelDataplaneMapSize: 16384, + TunnelIPFamily: "IPv4", + } +} + +func applyNodeRuntimeConfig(cmd *cobra.Command, cfg *config) error { + runtimeCfg, err := configpkg.LoadRuntimeConfig(cfg.ConfigFile) + if err != nil { + return err + } + + flags := cmd.Flags() + nodeCfg := runtimeCfg.Node + + if !flags.Changed("informer-resync-period") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.InformerResyncPeriod, "node.informerResyncPeriod"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.InformerResyncPeriod = d + } + } + + if !flags.Changed("node-name") && nodeCfg.NodeName != "" { + cfg.NodeName = nodeCfg.NodeName + } + + if !flags.Changed("cni-conf-dir") && nodeCfg.CNIConfDir != "" { + cfg.CNIConfDir = nodeCfg.CNIConfDir + } + + if !flags.Changed("cni-conf-file") && nodeCfg.CNIConfFile != "" { + cfg.CNIConfFile = nodeCfg.CNIConfFile + } + + if !flags.Changed("bridge-name") && nodeCfg.BridgeName != "" { + cfg.BridgeName = nodeCfg.BridgeName + } + + if !flags.Changed("wireguard-dir") && nodeCfg.WireGuardDir != "" { + cfg.WireGuardDir = nodeCfg.WireGuardDir + } + + if !flags.Changed("wireguard-port") && nodeCfg.WireGuardPort != nil { + cfg.WireGuardPort = *nodeCfg.WireGuardPort + } + + if !flags.Changed("enable-policy-routing") && nodeCfg.EnablePolicyRouting != nil { //nolint:staticcheck // intentional use of deprecated field for backward compat + cfg.EnablePolicyRouting = *nodeCfg.EnablePolicyRouting //nolint:staticcheck // intentional use of deprecated field + } + + if !flags.Changed("mtu") && nodeCfg.MTU != nil { + cfg.MTU = *nodeCfg.MTU + } + + if !flags.Changed("health-port") && nodeCfg.HealthPort != nil { + cfg.HealthPort = *nodeCfg.HealthPort + } + + if !flags.Changed("status-push-enabled") && nodeCfg.StatusPushEnabled != nil { + cfg.StatusPushEnabled = *nodeCfg.StatusPushEnabled + } + + if !flags.Changed("status-push-url") && nodeCfg.StatusPushURL != "" { + cfg.StatusPushURL = nodeCfg.StatusPushURL + } + + if !flags.Changed("status-push-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushInterval, "node.statusPushInterval"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.StatusPushInterval = d + } + } + + if !flags.Changed("status-push-apiserver-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusPushAPIServerInterval, "node.statusPushApiserverInterval"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.StatusPushAPIServerInterval = d + } + } + + if !flags.Changed("status-push-delta") && nodeCfg.StatusPushDelta != nil { + cfg.StatusPushDelta = *nodeCfg.StatusPushDelta + } + + if !flags.Changed("status-ws-enabled") && nodeCfg.StatusWSEnabled != nil { + cfg.StatusWSEnabled = *nodeCfg.StatusWSEnabled + } + + if !flags.Changed("status-ws-url") && nodeCfg.StatusWSURL != "" { + cfg.StatusWSURL = nodeCfg.StatusWSURL + } + + if !flags.Changed("status-ws-apiserver-mode") && nodeCfg.StatusWSAPIServerMode != "" { + cfg.StatusWSAPIServerMode = nodeCfg.StatusWSAPIServerMode + } + + if !flags.Changed("status-ws-apiserver-url") && nodeCfg.StatusWSAPIServerURL != "" { + cfg.StatusWSAPIServerURL = nodeCfg.StatusWSAPIServerURL + } + + if !flags.Changed("status-ws-apiserver-startup-delay") && nodeCfg.StatusWSAPIServerStartupDelay != "" { + d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSAPIServerStartupDelay, "node.statusWebsocketApiserverStartupDelay") + if parseErr != nil { + return parseErr + } + + if d < 0 { + return fmt.Errorf("node.statusWebsocketApiserverStartupDelay must be >= 0") + } + + cfg.StatusWSAPIServerStartupDelay = d + } + + if !flags.Changed("status-ws-keepalive-interval") && nodeCfg.StatusWSKeepaliveInterval != "" { + d, parseErr := configpkg.ParseDurationField(nodeCfg.StatusWSKeepaliveInterval, "node.statusWebsocketKeepaliveInterval") + if parseErr != nil { + return parseErr + } + + cfg.StatusWSKeepaliveInterval = d + } + + if !flags.Changed("status-ws-keepalive-failure-count") && nodeCfg.StatusWSKeepaliveFailCount != nil { + cfg.StatusWSKeepaliveFailureCount = *nodeCfg.StatusWSKeepaliveFailCount + } + // New consolidated shutdown cleanup flag. + if !flags.Changed("remove-configuration-on-shutdown") && nodeCfg.RemoveConfigurationOnShutdown != nil { + cfg.RemoveConfigurationOnShutdown = *nodeCfg.RemoveConfigurationOnShutdown + } + // Deprecated individual shutdown cleanup flags (kept for backward compatibility). + if !flags.Changed("shutdown-remove-wireguard-configuration") && nodeCfg.ShutdownRemoveWireGuardConfiguration != nil { + cfg.RemoveWireGuardOnShutdown = *nodeCfg.ShutdownRemoveWireGuardConfiguration + } + + if !flags.Changed("shutdown-cleanup-netlink") && nodeCfg.ShutdownRemoveIPRoutes != nil { + cfg.CleanupNetlinkOnShutdown = *nodeCfg.ShutdownRemoveIPRoutes + } + + if !flags.Changed("shutdown-remove-masquerade-rules") && nodeCfg.ShutdownRemoveMasqueradeRules != nil { + cfg.RemoveMasqueradeOnShutdown = *nodeCfg.ShutdownRemoveMasqueradeRules + } + // If any deprecated flag is true, activate the consolidated flag. + if cfg.RemoveWireGuardOnShutdown || cfg.CleanupNetlinkOnShutdown || cfg.RemoveMasqueradeOnShutdown { + cfg.RemoveConfigurationOnShutdown = true + } + + if _, err := parseStatusWSAPIServerMode(cfg.StatusWSAPIServerMode); err != nil { + return err + } + + if !flags.Changed("status-critical-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.CriticalDeltaEvery, "node.criticalDeltaEvery"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.CriticalDeltaEvery = d + } + } + + if !flags.Changed("status-stats-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.StatsDeltaEvery, "node.statsDeltaEvery"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.StatsDeltaEvery = d + } + } + + if !flags.Changed("status-full-sync-interval") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.FullSyncEvery, "node.fullSyncEvery"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.FullSyncEvery = d + } + } + + if cfg.StatusWSKeepaliveFailureCount < 1 { + return fmt.Errorf("node.statusWsKeepaliveFailureCount must be >= 1") + } + + // Apply preferred encapsulation from config file if not set via CLI. + if !flags.Changed("preferred-private-encap") && nodeCfg.PreferredPrivateNetworkEncapsulation != "" { + cfg.PreferredPrivateEncap = nodeCfg.PreferredPrivateNetworkEncapsulation + } + + if !flags.Changed("preferred-public-encap") && nodeCfg.PreferredPublicNetworkEncapsulation != "" { + cfg.PreferredPublicEncap = nodeCfg.PreferredPublicNetworkEncapsulation + } + + if !flags.Changed("health-flap-max-backoff") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.HealthFlapMaxBackoff, "node.healthFlapMaxBackoff"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.HealthFlapMaxBackoff = d + } + } + + if !flags.Changed("route-table-id") && nodeCfg.RouteTableID != nil { + cfg.RouteTableID = *nodeCfg.RouteTableID + } + + if !flags.Changed("kube-proxy-health-interval") && nodeCfg.KubeProxyHealthInterval != "" { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.KubeProxyHealthInterval, "node.kubeProxyHealthInterval"); parseErr != nil { + return parseErr + } else { + cfg.KubeProxyHealthInterval = d + } + } + + if !flags.Changed("netlink-resync-period") { + if d, parseErr := configpkg.ParseDurationField(nodeCfg.NetlinkResyncPeriod, "node.netlinkResyncPeriod"); parseErr != nil { + return parseErr + } else if d > 0 { + cfg.NetlinkResyncPeriod = d + } + } + + if !flags.Changed("tunnel-dataplane-map-size") && nodeCfg.TunnelDataplaneMapSize != nil { + cfg.TunnelDataplaneMapSize = *nodeCfg.TunnelDataplaneMapSize + } + + if !flags.Changed("tunnel-ip-family") && nodeCfg.TunnelIPFamily != "" { + cfg.TunnelIPFamily = nodeCfg.TunnelIPFamily + } + + if !flags.Changed("vxlan-src-port-low") && nodeCfg.VXLANSrcPortLow != nil { + cfg.VXLANSrcPortLow = *nodeCfg.VXLANSrcPortLow + } + + if !flags.Changed("vxlan-src-port-high") && nodeCfg.VXLANSrcPortHigh != nil { + cfg.VXLANSrcPortHigh = *nodeCfg.VXLANSrcPortHigh + } + + if !flags.Changed("geneve-interface") && nodeCfg.GeneveInterfaceName != "" { + cfg.GeneveInterfaceName = nodeCfg.GeneveInterfaceName + } + + if !flags.Changed("vxlan-interface") && nodeCfg.VXLANInterfaceName != "" { + cfg.VXLANInterfaceName = nodeCfg.VXLANInterfaceName + } + + if !flags.Changed("ipip-interface") && nodeCfg.IPIPInterfaceName != "" { + cfg.IPIPInterfaceName = nodeCfg.IPIPInterfaceName + } + + if !flags.Changed("wireguard-interface-prefix") && nodeCfg.WireGuardInterfacePrefix != "" { + cfg.WireGuardInterfacePrefix = nodeCfg.WireGuardInterfacePrefix + } + + // Validate and default tunnelIPFamily + switch cfg.TunnelIPFamily { + case "IPv4", "IPv6": + // valid + case "": + cfg.TunnelIPFamily = "IPv4" + default: + return fmt.Errorf("invalid tunnel-ip-family %q: must be 'IPv4' or 'IPv6'", cfg.TunnelIPFamily) + } + + if err := validateTunnelInterfaceNames(cfg.GeneveInterfaceName, cfg.VXLANInterfaceName, cfg.IPIPInterfaceName); err != nil { + return err + } + + if err := validateWireGuardInterfacePrefix(cfg.WireGuardInterfacePrefix); err != nil { + return err + } + + // Normalize MTU: treat 0 as 1280 (the IPv6 minimum, safe for all links). + if cfg.MTU == 0 { + cfg.MTU = 1280 + } + + // Apply common config. + if !flags.Changed("apiserver-url") && runtimeCfg.Common.ApiserverURL != "" { + cfg.ApiserverURL = runtimeCfg.Common.ApiserverURL + } + + return nil +} + +func run(ctx context.Context, cfg *config) error { + klog.Infof("unbounded-net-node version=%s commit=%s built=%s", version.Version, version.GitCommit, version.BuildTime) + + // Validate node name + if cfg.NodeName == "" { + return fmt.Errorf("node name is required; set NODE_NAME or use --node-name") + } + + klog.Infof("Running on node: %s", cfg.NodeName) + + if cfg.EnablePolicyRouting { + klog.Info("Policy-based routing on gateway interfaces is enabled") + } else { + klog.Info("Policy-based routing on gateway interfaces is disabled") + } + + // Build Kubernetes client + var ( + restConfig *rest.Config + err error + ) + + if cfg.RESTConfig != nil { + restConfig = rest.CopyConfig(cfg.RESTConfig) + } else if cfg.KubeconfigPath != "" { + restConfig, err = clientcmd.BuildConfigFromFlags(cfg.ApiserverURL, cfg.KubeconfigPath) + } else { + restConfig, err = rest.InClusterConfig() + if err == nil && cfg.ApiserverURL != "" { + restConfig.Host = cfg.ApiserverURL + } + } + + if err != nil { + return fmt.Errorf("build kubeconfig: %w", err) + } + + if cfg.ApiserverURL != "" { + klog.Infof("Using API server URL override: %s", cfg.ApiserverURL) + } + + // Wire client-go metrics into Prometheus before creating clients. + metrics.RegisterClientGoMetrics() + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create Kubernetes client: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create dynamic Kubernetes client: %w", err) + } + + // Generate WireGuard keys and annotate node + pubKey, err := ensureWireGuardKeys(cfg) + if err != nil { + return fmt.Errorf("ensure WireGuard keys: %w", err) + } + + klog.Infof("WireGuard public key: %s", pubKey) + + if err := annotateNodeWithPubKey(ctx, clientset, cfg.NodeName, pubKey); err != nil { + return fmt.Errorf("annotate Node with WireGuard public key: %w", err) + } + + klog.Info("Node annotated with WireGuard public key") + + // Detect and annotate the node's maximum tunnel MTU so the controller + // can validate that the configured MTU is compatible across all nodes. + if detectedMTU := unboundednetnetlink.DetectDefaultRouteMTU(); detectedMTU > 0 { + wgMTU := detectedMTU - unboundednetnetlink.WireGuardMTUOverhead + if err := annotateNodeWithMTU(ctx, clientset, cfg.NodeName, wgMTU); err != nil { + klog.Warningf("Failed to annotate node with tunnel MTU: %v", err) + } else { + klog.Infof("Node annotated with tunnel MTU %d (detected default route MTU %d - %d overhead)", wgMTU, detectedMTU, unboundednetnetlink.WireGuardMTUOverhead) + } + } + + // Watch the config file for dynamic log level changes. + go configpkg.WatchConfigLogLevel(ctx, cfg.ConfigFile) + + // Warn if public network traffic will be sent unencrypted. + if cfg.PreferredPublicEncap != "" && cfg.PreferredPublicEncap != "WireGuard" { + klog.Warningf("WARNING: preferredPublicNetworkEncapsulation is set to %q -- traffic over public networks will be sent UNENCRYPTED", cfg.PreferredPublicEncap) + } + + // Create informers early - before any CRD-based lookups + // This allows us to use the informer cache for all CRD operations + informerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, cfg.InformerResyncPeriod) + sliceInformer := informerFactory.ForResource(siteNodeSliceGVR).Informer() + siteInformer := informerFactory.ForResource(siteGVR).Informer() + gatewayPoolInformer := informerFactory.ForResource(gatewayPoolGVR).Informer() + gatewayNodeInformer := informerFactory.ForResource(gatewayNodeGVR).Informer() + sitePeeringInformer := informerFactory.ForResource(sitePeeringGVR).Informer() + assignmentInformer := informerFactory.ForResource(siteGatewayPoolAssignmentGVR).Informer() + poolPeeringInformer := informerFactory.ForResource(gatewayPoolPeeringGVR).Informer() + + // Start the informers + informerFactory.Start(ctx.Done()) + + // Wait for caches to sync + klog.Info("Waiting for informer caches to sync") + + if !cache.WaitForCacheSync(ctx.Done(), sliceInformer.HasSynced, siteInformer.HasSynced, gatewayPoolInformer.HasSynced, gatewayNodeInformer.HasSynced, sitePeeringInformer.HasSynced, assignmentInformer.HasSynced, poolPeeringInformer.HasSynced) { + return fmt.Errorf("failed to sync informer caches") + } + + klog.Info("Informer caches synced") + + // Start the netlink cache (read-only network state snapshot). + netlinkCache := unboundednetnetlink.NewNetlinkCache(cfg.NetlinkResyncPeriod) + if err := netlinkCache.Start(ctx); err != nil { + return fmt.Errorf("start netlink cache: %w", err) + } + + // Track if CNI is configured for health checks + cniConfigured := false + + // Create shared health state for health server + healthState := &nodeHealthState{ + cniConfigured: &cniConfigured, + informersSynced: []cache.InformerSynced{ + sliceInformer.HasSynced, + siteInformer.HasSynced, + gatewayPoolInformer.HasSynced, + gatewayNodeInformer.HasSynced, + sitePeeringInformer.HasSynced, + assignmentInformer.HasSynced, + poolPeeringInformer.HasSynced, + }, + } + + // Start health server if enabled (readiness should not wait on site membership) + if cfg.HealthPort > 0 { + go startHealthServer(cfg.HealthPort, healthState) + } + + // Check if this node is a gateway node by checking the informer cache + isGatewayNode := isGatewayNodeFromCRDs(gatewayPoolInformer, pubKey) + if isGatewayNode { + klog.Info("Node is a gateway node (found in GatewayPool status)") + } + + // Wait for this node to appear in a SiteNodeSlice or GatewayPool + // This ensures the site controller has processed this node before we continue + mySiteName, err := waitForSiteMembership(ctx, sliceInformer, gatewayPoolInformer, pubKey) + if err != nil { + if err == context.Canceled { + return nil + } + + return err + } + + // Check if this node's site has manageCniPlugin enabled using the informer cache + manageCniPlugin := manageCNIForMembership(siteInformer, mySiteName, isGatewayNode) + + var nodePodCIDRs []string + if manageCniPlugin { + // Wait for podCIDRs and configure CNI + nodePodCIDRs, err = waitForPodCIDRsAndConfigure(ctx, clientset, cfg, &cniConfigured) + if err != nil { + if err == context.Canceled { + return nil + } + + return err + } + } else { + klog.Info("manageCniPlugin is false for this site - skipping CNI configuration") + // Still need to get the node's podCIDRs for WireGuard gateway IP calculation + node, err := clientset.CoreV1().Nodes().Get(ctx, cfg.NodeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get Node %s: %w", cfg.NodeName, err) + } + + nodePodCIDRs = node.Spec.PodCIDRs + // Mark CNI as "configured" since we're intentionally not managing it + cniConfigured = true + } + + // After CNI is configured (or skipped), watch Site CRD for WireGuard peers + // Pass the node's podCIDRs so routes can be configured with preferred source IPs + // Also pass the informers so they can be reused (already synced) + return watchSiteAndConfigureWireGuard(ctx, clientset, dynamicClient, cfg, pubKey, nodePodCIDRs, manageCniPlugin, healthState, netlinkCache, siteInformer, sliceInformer, gatewayPoolInformer, gatewayNodeInformer, sitePeeringInformer, assignmentInformer, poolPeeringInformer) +} diff --git a/cmd/unbounded-net-node/node_config_validation.go b/internal/net/nodeagent/node_config_validation.go similarity index 99% rename from cmd/unbounded-net-node/node_config_validation.go rename to internal/net/nodeagent/node_config_validation.go index eda654ded..0384ab7c8 100644 --- a/cmd/unbounded-net-node/node_config_validation.go +++ b/internal/net/nodeagent/node_config_validation.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "fmt" diff --git a/cmd/unbounded-net-node/node_config_validation_test.go b/internal/net/nodeagent/node_config_validation_test.go similarity index 99% rename from cmd/unbounded-net-node/node_config_validation_test.go rename to internal/net/nodeagent/node_config_validation_test.go index 061411d7e..298a7768d 100644 --- a/cmd/unbounded-net-node/node_config_validation_test.go +++ b/internal/net/nodeagent/node_config_validation_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "errors" diff --git a/cmd/unbounded-net-node/node_types.go b/internal/net/nodeagent/node_types.go similarity index 99% rename from cmd/unbounded-net-node/node_types.go rename to internal/net/nodeagent/node_types.go index d0d372713..f2bf9817a 100644 --- a/cmd/unbounded-net-node/node_types.go +++ b/internal/net/nodeagent/node_types.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "sync" diff --git a/cmd/unbounded-net-node/peer_healthcheck.go b/internal/net/nodeagent/peer_healthcheck.go similarity index 99% rename from cmd/unbounded-net-node/peer_healthcheck.go rename to internal/net/nodeagent/peer_healthcheck.go index 00bd26e9a..c3d86170b 100644 --- a/cmd/unbounded-net-node/peer_healthcheck.go +++ b/internal/net/nodeagent/peer_healthcheck.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "net" diff --git a/cmd/unbounded-net-node/reconciliation_helpers.go b/internal/net/nodeagent/reconciliation_helpers.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers.go rename to internal/net/nodeagent/reconciliation_helpers.go index f1ba45ef0..cab7a9c0f 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers.go +++ b/internal/net/nodeagent/reconciliation_helpers.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/reconciliation_helpers_bfd_test.go b/internal/net/nodeagent/reconciliation_helpers_bfd_test.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers_bfd_test.go rename to internal/net/nodeagent/reconciliation_helpers_bfd_test.go index 1e16ca18d..324daf465 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers_bfd_test.go +++ b/internal/net/nodeagent/reconciliation_helpers_bfd_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import "testing" diff --git a/cmd/unbounded-net-node/reconciliation_helpers_more_test.go b/internal/net/nodeagent/reconciliation_helpers_more_test.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers_more_test.go rename to internal/net/nodeagent/reconciliation_helpers_more_test.go index 46f722f24..d33a2e74c 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers_more_test.go +++ b/internal/net/nodeagent/reconciliation_helpers_more_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/reconciliation_helpers_test.go b/internal/net/nodeagent/reconciliation_helpers_test.go similarity index 99% rename from cmd/unbounded-net-node/reconciliation_helpers_test.go rename to internal/net/nodeagent/reconciliation_helpers_test.go index 725a85849..651f2c518 100644 --- a/cmd/unbounded-net-node/reconciliation_helpers_test.go +++ b/internal/net/nodeagent/reconciliation_helpers_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/route_annotations.go b/internal/net/nodeagent/route_annotations.go similarity index 99% rename from cmd/unbounded-net-node/route_annotations.go rename to internal/net/nodeagent/route_annotations.go index b47e9ffcf..eae63c617 100644 --- a/cmd/unbounded-net-node/route_annotations.go +++ b/internal/net/nodeagent/route_annotations.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "sort" diff --git a/cmd/unbounded-net-node/runtime_utils.go b/internal/net/nodeagent/runtime_utils.go similarity index 99% rename from cmd/unbounded-net-node/runtime_utils.go rename to internal/net/nodeagent/runtime_utils.go index f65692226..6e2b06503 100644 --- a/cmd/unbounded-net-node/runtime_utils.go +++ b/internal/net/nodeagent/runtime_utils.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "net" diff --git a/cmd/unbounded-net-node/runtime_utils_test.go b/internal/net/nodeagent/runtime_utils_test.go similarity index 99% rename from cmd/unbounded-net-node/runtime_utils_test.go rename to internal/net/nodeagent/runtime_utils_test.go index f1c5c4752..6f25fbc48 100644 --- a/cmd/unbounded-net-node/runtime_utils_test.go +++ b/internal/net/nodeagent/runtime_utils_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "slices" diff --git a/cmd/unbounded-net-node/site_watch_reconcile.go b/internal/net/nodeagent/site_watch_reconcile.go similarity index 98% rename from cmd/unbounded-net-node/site_watch_reconcile.go rename to internal/net/nodeagent/site_watch_reconcile.go index 5ba575087..ab5f42fe5 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile.go +++ b/internal/net/nodeagent/site_watch_reconcile.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" @@ -803,13 +803,24 @@ func isGatewayNodeFromCRDs(gatewayPoolInformer cache.SharedIndexInformer, myPubK return false } +func netbootMembershipFromCRDs( + sliceInformer, gatewayPoolInformer cache.SharedIndexInformer, + myPubKey string, +) (string, bool) { + if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + return siteName, true + } + + return "", isGatewayNodeFromCRDs(gatewayPoolInformer, myPubKey) +} + // waitForSiteMembership waits for this node to appear in a SiteNodeSlice or GatewayPool. // This ensures the site controller has processed this node before we continue. // Returns the site name once found, or empty string for gateway nodes without a site. func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInformer cache.SharedIndexInformer, myPubKey string) (string, error) { // Check immediately first - siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey) - if siteName != "" { + siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey) + if found { klog.Infof("Node found in site %q", siteName) return siteName, nil } @@ -822,7 +833,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform // Add event handlers to detect when we appear in a slice or pool sliceHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: @@ -830,7 +841,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform } }, UpdateFunc: func(oldObj, newObj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: @@ -841,7 +852,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform poolHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: @@ -849,7 +860,7 @@ func waitForSiteMembership(ctx context.Context, sliceInformer, gatewayPoolInform } }, UpdateFunc: func(oldObj, newObj interface{}) { - if siteName := findMySiteFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); siteName != "" { + if siteName, found := netbootMembershipFromCRDs(sliceInformer, gatewayPoolInformer, myPubKey); found { select { case foundCh <- siteName: default: @@ -921,6 +932,14 @@ func getManageCniPluginFromCRDs(siteInformer cache.SharedIndexInformer, siteName return true // Site not found, default to true } +func manageCNIForMembership(siteInformer cache.SharedIndexInformer, siteName string, isGatewayNode bool) bool { + if isGatewayNode && siteName == "" { + return false + } + + return getManageCniPluginFromCRDs(siteInformer, siteName) +} + var configureWireGuardFunc = configureWireGuard // updateWireGuardFromSlices reads Site and SiteNodeSlices from the informer caches and configures WireGuard diff --git a/cmd/unbounded-net-node/site_watch_reconcile_test.go b/internal/net/nodeagent/site_watch_reconcile_test.go similarity index 78% rename from cmd/unbounded-net-node/site_watch_reconcile_test.go rename to internal/net/nodeagent/site_watch_reconcile_test.go index 8c56511c7..9f48d3bad 100644 --- a/cmd/unbounded-net-node/site_watch_reconcile_test.go +++ b/internal/net/nodeagent/site_watch_reconcile_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" @@ -106,6 +106,31 @@ func TestWaitForSiteMembershipImmediate(t *testing.T) { } } +func TestWaitForSiteMembershipAcceptsSiteLessGateway(t *testing.T) { + sliceInformer := newTestInformer() + gatewayPoolInformer := newTestInformer() + + pool := &unboundednetv1alpha1.GatewayPool{ + ObjectMeta: metav1.ObjectMeta{Name: "external"}, + Spec: unboundednetv1alpha1.GatewayPoolSpec{NodeSelector: map[string]string{"external": "true"}}, + Status: unboundednetv1alpha1.GatewayPoolStatus{Nodes: []unboundednetv1alpha1.GatewayNodeInfo{ + {Name: "admin-gateway", WireGuardPublicKey: "pub-external"}, + }}, + } + if err := gatewayPoolInformer.GetStore().Add(toUnstructuredSiteWatch(t, pool)); err != nil { + t.Fatalf("add pool failed: %v", err) + } + + got, err := waitForSiteMembership(context.Background(), sliceInformer, gatewayPoolInformer, "pub-external") + if err != nil { + t.Fatalf("waitForSiteMembership returned error: %v", err) + } + + if got != "" { + t.Fatalf("site-less gateway membership = %q, want empty", got) + } +} + // TestGetManageCniPluginFromCRDs tests GetManageCniPluginFromCRDs. func TestGetManageCniPluginFromCRDs(t *testing.T) { siteInformer := newTestInformer() @@ -144,3 +169,15 @@ func TestGetManageCniPluginFromCRDs(t *testing.T) { t.Fatalf("expected missing site default to true") } } + +func TestSiteLessGatewayDisablesCNIManagement(t *testing.T) { + siteInformer := newTestInformer() + + if manageCNIForMembership(siteInformer, "", true) { + t.Fatal("site-less gateway must not wait for PodCIDRs or write CNI configuration") + } + + if !manageCNIForMembership(siteInformer, "", false) { + t.Fatal("ordinary node without resolved membership must retain the default CNI policy") + } +} diff --git a/cmd/unbounded-net-node/status_proto.go b/internal/net/nodeagent/status_proto.go similarity index 99% rename from cmd/unbounded-net-node/status_proto.go rename to internal/net/nodeagent/status_proto.go index 61dc259c0..9ba72b313 100644 --- a/cmd/unbounded-net-node/status_proto.go +++ b/internal/net/nodeagent/status_proto.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/status_proto_test.go b/internal/net/nodeagent/status_proto_test.go similarity index 99% rename from cmd/unbounded-net-node/status_proto_test.go rename to internal/net/nodeagent/status_proto_test.go index 4b4965f5e..01f256c5a 100644 --- a/cmd/unbounded-net-node/status_proto_test.go +++ b/internal/net/nodeagent/status_proto_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/status_server.go b/internal/net/nodeagent/status_server.go similarity index 99% rename from cmd/unbounded-net-node/status_server.go rename to internal/net/nodeagent/status_server.go index 10da70cf9..d75fc437c 100644 --- a/cmd/unbounded-net-node/status_server.go +++ b/internal/net/nodeagent/status_server.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "bytes" diff --git a/cmd/unbounded-net-node/status_server_http_test.go b/internal/net/nodeagent/status_server_http_test.go similarity index 99% rename from cmd/unbounded-net-node/status_server_http_test.go rename to internal/net/nodeagent/status_server_http_test.go index a19c87343..bcb211c67 100644 --- a/cmd/unbounded-net-node/status_server_http_test.go +++ b/internal/net/nodeagent/status_server_http_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "encoding/json" diff --git a/cmd/unbounded-net-node/status_server_test.go b/internal/net/nodeagent/status_server_test.go similarity index 99% rename from cmd/unbounded-net-node/status_server_test.go rename to internal/net/nodeagent/status_server_test.go index 6bf2810c3..2f7d4ea2d 100644 --- a/cmd/unbounded-net-node/status_server_test.go +++ b/internal/net/nodeagent/status_server_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/tunnel_config.go b/internal/net/nodeagent/tunnel_config.go similarity index 99% rename from cmd/unbounded-net-node/tunnel_config.go rename to internal/net/nodeagent/tunnel_config.go index 45c5692a7..b624ac8f1 100644 --- a/cmd/unbounded-net-node/tunnel_config.go +++ b/internal/net/nodeagent/tunnel_config.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/tunnel_interface_validation.go b/internal/net/nodeagent/tunnel_interface_validation.go similarity index 99% rename from cmd/unbounded-net-node/tunnel_interface_validation.go rename to internal/net/nodeagent/tunnel_interface_validation.go index 78396783d..fe76897c1 100644 --- a/cmd/unbounded-net-node/tunnel_interface_validation.go +++ b/internal/net/nodeagent/tunnel_interface_validation.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "fmt" diff --git a/cmd/unbounded-net-node/tunnel_interface_validation_test.go b/internal/net/nodeagent/tunnel_interface_validation_test.go similarity index 99% rename from cmd/unbounded-net-node/tunnel_interface_validation_test.go rename to internal/net/nodeagent/tunnel_interface_validation_test.go index ed37a1ed0..7f6248e1a 100644 --- a/cmd/unbounded-net-node/tunnel_interface_validation_test.go +++ b/internal/net/nodeagent/tunnel_interface_validation_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "strings" diff --git a/cmd/unbounded-net-node/wireguard_config.go b/internal/net/nodeagent/wireguard_config.go similarity index 99% rename from cmd/unbounded-net-node/wireguard_config.go rename to internal/net/nodeagent/wireguard_config.go index fb14e9d5a..d63bc5326 100644 --- a/cmd/unbounded-net-node/wireguard_config.go +++ b/internal/net/nodeagent/wireguard_config.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent import ( "context" diff --git a/cmd/unbounded-net-node/wireguard_config_test.go b/internal/net/nodeagent/wireguard_config_test.go similarity index 80% rename from cmd/unbounded-net-node/wireguard_config_test.go rename to internal/net/nodeagent/wireguard_config_test.go index 0fc827aec..1e1b08475 100644 --- a/cmd/unbounded-net-node/wireguard_config_test.go +++ b/internal/net/nodeagent/wireguard_config_test.go @@ -1,4 +1,4 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package main +package nodeagent diff --git a/internal/operator/bootstrap.go b/internal/operator/bootstrap.go index 0dcb71109..60cf8d439 100644 --- a/internal/operator/bootstrap.go +++ b/internal/operator/bootstrap.go @@ -53,6 +53,8 @@ var RequiredCRDNames = [...]string{ "machineconfigurations.unbounded-cloud.io", "machineoperationcredentials.unbounded-cloud.io", "machineconfigurationversions.unbounded-cloud.io", + "netbootendpoints.unbounded-cloud.io", + "netbootsessions.unbounded-cloud.io", "sitenodeslices.net.unbounded-cloud.io", "gatewaypools.net.unbounded-cloud.io", "gatewaypoolnodes.net.unbounded-cloud.io", diff --git a/internal/operator/bootstrap_test.go b/internal/operator/bootstrap_test.go index 30f3d1eee..15e160e6f 100644 --- a/internal/operator/bootstrap_test.go +++ b/internal/operator/bootstrap_test.go @@ -364,6 +364,8 @@ func TestRequiredCRDNames(t *testing.T) { "machineconfigurations.unbounded-cloud.io", "machineoperationcredentials.unbounded-cloud.io", "machineconfigurationversions.unbounded-cloud.io", + "netbootendpoints.unbounded-cloud.io", + "netbootsessions.unbounded-cloud.io", "sitenodeslices.net.unbounded-cloud.io", "gatewaypools.net.unbounded-cloud.io", "gatewaypoolnodes.net.unbounded-cloud.io", diff --git a/internal/operator/components/metalman/metalman.go b/internal/operator/components/metalman/metalman.go index 57e6f2a1a..7e3fe1f1b 100644 --- a/internal/operator/components/metalman/metalman.go +++ b/internal/operator/components/metalman/metalman.go @@ -1,25 +1,37 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Package metalman implements the per-Site metalman PXE controller component. +// Package metalman implements the per-Site Metalman control and serving plane. package metalman import ( "context" + "crypto/rand" + "crypto/sha256" + "fmt" + "sort" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" machinamanifests "github.com/Azure/unbounded/deploy/machina" "github.com/Azure/unbounded/internal/operator/component" ) -// Component reconciles the per-Site metalman PXE controller. +// Component reconciles the per-Site Metalman workloads. type Component struct{} // New returns the metalman per-Site component. @@ -40,39 +52,90 @@ func (Component) Enabled(site *unboundedv1alpha3.Site) bool { return unboundedv1alpha3.ComponentEnabled(&site.Spec.Components.Metalman.SiteComponentSpec) } -// Reconcile deploys the per-site metalman PXE controller and its RBAC. +// Reconcile deploys the per-site Metalman controller and server plane. func (Component) Reconcile(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) component.Result { if err := env.ApplyManifestFS(ctx, machinamanifests.Manifests, mutateSupportObject); err != nil { return component.Failed(err) } - if err := env.ApplyObject(ctx, deployment(site, env.Namespace, env.Config)); err != nil { + if err := ensureCapabilitySecret(ctx, env, site); err != nil { + return component.Failed(err) + } + + for _, obj := range []client.Object{ + controllerDeployment(site, env.Namespace, env.Config), + serverDeployment(site, env.Namespace, env.Config), + serverService(site, env.Namespace), + serverPodDisruptionBudget(site, env.Namespace), + } { + if err := env.ApplyObject(ctx, obj); err != nil { + return component.Failed(err) + } + } + + if err := reconcileEndpointEdges(ctx, env, site); err != nil { return component.Failed(err) } return component.Reconciled() } -// Cleanup removes the per-site metalman Deployment. The shared metalman RBAC is +// Cleanup removes the per-site Metalman workloads. The shared Metalman RBAC is // left in place; it is harmless when unreferenced and may still be used by other // sites. func (Component) Cleanup(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) error { - return env.DeleteIfExists(ctx, &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, - ObjectMeta: metav1.ObjectMeta{Name: DeploymentName(site.Name), Namespace: env.Namespace}, - }) + for _, obj := range []client.Object{ + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: DeploymentName(site.Name), Namespace: env.Namespace}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, ObjectMeta: metav1.ObjectMeta{Name: CapabilitySecretName(site.Name), Namespace: env.Namespace}}, + &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{APIVersion: "policy/v1", Kind: "PodDisruptionBudget"}, ObjectMeta: metav1.ObjectMeta{Name: ServerName(site.Name), Namespace: env.Namespace}}, + } { + if err := env.DeleteIfExists(ctx, obj); err != nil { + return err + } + } + + return nil } // SetupWatches recreates the per-site Deployment if it is deleted or drifts, via // its controller owner reference to the Site. The predicate drops status-only // updates so pod-count churn does not re-apply the Deployment. func (Component) SetupWatches(b *builder.Builder, env *component.Env) { + b.Watches(&unboundedv1alpha3.NetbootEndpoint{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []ctrl.Request { + endpoint, ok := obj.(*unboundedv1alpha3.NetbootEndpoint) + if !ok || endpoint.Spec.SiteRef == "" { + return nil + } + + return []ctrl.Request{{NamespacedName: client.ObjectKey{Name: endpoint.Spec.SiteRef}}} + })) + b.Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []ctrl.Request { + secret, ok := obj.(*corev1.Secret) + if !ok { + return nil + } + + return requestsForTLSSecret(ctx, env.Client, secret) + })) b.Owns(&appsv1.Deployment{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) + b.Owns(&corev1.Service{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) + b.Owns(&corev1.Secret{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) + b.Owns(&policyv1.PodDisruptionBudget{}, builder.WithPredicates(env.OwnedWorkloadPredicate())) } // DeploymentName is the per-site metalman Deployment name. func DeploymentName(site string) string { return "metalman-controller-" + site } +// ServerName is the per-site Metalman server Deployment and Service name. +func ServerName(site string) string { return "metalman-server-" + site } + +// CapabilitySecretName is the per-site capability signing Secret name. +func CapabilitySecretName(site string) string { return "metalman-capability-" + site } + +const capabilitySecretKey = "capability.key" + // SupportObjectNameSubstring identifies the metalman RBAC objects that ship in // the machina manifest set. It is exported so the machina component can skip // exactly the objects the metalman component owns and applies. @@ -95,25 +158,38 @@ func mutateSupportObject(obj *unstructured.Unstructured) error { return nil } -func deployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config) *appsv1.Deployment { +func controllerDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config) *appsv1.Deployment { + return roleDeployment(site, namespace, cfg, metalmanControllerRole, 1) +} + +func serverDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config) *appsv1.Deployment { + return roleDeployment(site, namespace, cfg, metalmanServerRole, 2) +} + +const ( + metalmanControllerRole = "controller" + metalmanServerRole = "server" + metalmanEdgeRole = "edge" + netbootEndpointLabel = "unbounded-cloud.io/netboot-endpoint" + edgeTLSChecksumAnnotation = "unbounded-cloud.io/tls-checksum" +) + +func roleDeployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Config, role string, replicas int32) *appsv1.Deployment { image := cfg.Image("metalman") + name := DeploymentName(site.Name) - labels := map[string]string{ - "app": "unbounded-pxe", - "app.kubernetes.io/name": "metalman-controller", - "app.kubernetes.io/component": "metalman", - unboundedv1alpha3.MachineSiteLabelKey: site.Name, + if role == metalmanServerRole { + name = ServerName(site.Name) } - args := []string{"serve-pxe", "--site=" + site.Name} - if site.Spec.Components.Metalman.DHCPAutoInterface != nil && *site.Spec.Components.Metalman.DHCPAutoInterface { - args = append(args, "--dhcp-auto-interface") + labels := map[string]string{ + "app": "unbounded-metalman", + "app.kubernetes.io/name": "metalman-" + role, + "app.kubernetes.io/component": role, + unboundedv1alpha3.MachineSiteLabelKey: site.Name, } - replicas := int32(1) - if site.Spec.Components.Metalman.Replicas != nil { - replicas = *site.Spec.Components.Metalman.Replicas - } + args := []string{role, "--site=" + site.Name, "--cache-dir=/var/cache/metalman"} env := []corev1.EnvVar{{ // Metalman resolves its leader-election lease namespace from @@ -135,11 +211,23 @@ func deployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Co env = append(env, corev1.EnvVar{Name: "METALMAN_APISERVER_URL", Value: cfg.APIServerEndpoint}) } - // metalman is hostNetwork and binds host ports (DHCP/TFTP/HTTP), so a surge - // pod cannot start while the old pod holds them on the same node. Terminate - // the old pod before creating the new one to avoid a rollout deadlock. - maxSurge := intstr.FromInt32(0) - maxUnavailable := intstr.FromInt32(1) + maxSurge := intstr.FromInt32(1) + maxUnavailable := intstr.FromInt32(0) + ports := []corev1.ContainerPort{{Name: "health", ContainerPort: 8081, Protocol: corev1.ProtocolTCP}} + + serviceAccountName := "metalman-controller" + if role == metalmanServerRole { + serviceAccountName = "metalman-server" + + ports = append(ports, corev1.ContainerPort{Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}) + } + + probe := func(path string) *corev1.Probe { + return &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: path, + Port: intstr.FromInt32(8081), + }}} + } return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, @@ -162,35 +250,572 @@ func deployment(site *unboundedv1alpha3.Site, namespace string, cfg component.Co Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ - HostNetwork: true, - ServiceAccountName: "metalman-controller", - // Match either the canonical or deprecated site label during - // the node-label deprecation window. Storage scopes its - // DaemonSet the same way. - Affinity: component.SiteNodeAffinity(site.Name), + ServiceAccountName: serviceAccountName, Containers: []corev1.Container{{ Name: "metalman", Image: image, ImagePullPolicy: corev1.PullAlways, Args: args, Env: env, - Ports: []corev1.ContainerPort{ - {Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}, - {Name: "health", ContainerPort: 8081, Protocol: corev1.ProtocolTCP}, - {Name: "dhcp", ContainerPort: 67, Protocol: corev1.ProtocolUDP}, - {Name: "tftp", ContainerPort: 69, Protocol: corev1.ProtocolUDP}, + Ports: ports, + LivenessProbe: probe("/healthz"), + ReadinessProbe: probe("/readyz"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, }, VolumeMounts: []corev1.VolumeMount{ {Name: "tmp", MountPath: "/tmp"}, {Name: "cache", MountPath: "/var/cache/metalman"}, + {Name: "capability-key", MountPath: "/var/run/secrets/metalman", ReadOnly: true}, }, }}, Volumes: []corev1.Volume{ {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "cache", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "capability-key", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: CapabilitySecretName(site.Name), + }}}, }, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ + MaxSkew: 1, + TopologyKey: corev1.LabelHostname, + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + }}, }, }, }, } } + +func ensureCapabilitySecret(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) error { + secret := &corev1.Secret{} + + key := client.ObjectKey{Namespace: env.Namespace, Name: CapabilitySecretName(site.Name)} + if err := env.Client.Get(ctx, key, secret); err == nil { + return nil + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("get capability Secret: %w", err) + } + + capabilityKey := make([]byte, 32) + if _, err := rand.Read(capabilityKey); err != nil { + return fmt.Errorf("generate capability key: %w", err) + } + + secret = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Data: map[string][]byte{capabilitySecretKey: capabilityKey}, + } + if err := env.Client.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create capability Secret: %w", err) + } + + return nil +} + +func serverService(site *unboundedv1alpha3.Site, namespace string) *corev1.Service { + labels := serverLabels(site.Name) + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: ServerName(site.Name), + Namespace: namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: corev1.ServiceSpec{ + Selector: labels, + Ports: []corev1.ServicePort{{ + Name: "http", + Port: 8880, + TargetPort: intstr.FromInt32(8880), + Protocol: corev1.ProtocolTCP, + }}, + }, + } +} + +func serverPodDisruptionBudget(site *unboundedv1alpha3.Site, namespace string) *policyv1.PodDisruptionBudget { + minAvailable := intstr.FromInt32(1) + + return &policyv1.PodDisruptionBudget{ + TypeMeta: metav1.TypeMeta{APIVersion: "policy/v1", Kind: "PodDisruptionBudget"}, + ObjectMeta: metav1.ObjectMeta{ + Name: ServerName(site.Name), + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &minAvailable, + Selector: &metav1.LabelSelector{MatchLabels: serverLabels(site.Name)}, + }, + } +} + +func serverLabels(site string) map[string]string { + return map[string]string{ + "app": "unbounded-metalman", + "app.kubernetes.io/name": "metalman-server", + "app.kubernetes.io/component": metalmanServerRole, + unboundedv1alpha3.MachineSiteLabelKey: site, + } +} + +func reconcileEndpointEdges(ctx context.Context, env *component.Env, site *unboundedv1alpha3.Site) error { + var endpoints unboundedv1alpha3.NetbootEndpointList + if err := env.Client.List(ctx, &endpoints); err != nil { + return fmt.Errorf("list NetbootEndpoints: %w", err) + } + + sort.Slice(endpoints.Items, func(i, j int) bool { return endpoints.Items[i].Name < endpoints.Items[j].Name }) + + desiredDeployments := map[string]struct{}{} + desiredServices := map[string]struct{}{} + desiredTLSSecrets := map[string]struct{}{} + + for i := range endpoints.Items { + endpoint := &endpoints.Items[i] + if endpoint.Spec.SiteRef != site.Name { + continue + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, env.Namespace, env.Config) + if err != nil { + return fmt.Errorf("build edge for NetbootEndpoint %s: %w", endpoint.Name, err) + } + + if deployment != nil { + if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { + secret, err := mirroredTLSSecret(ctx, env.Client, endpoint, site, env.Namespace) + if err != nil { + return fmt.Errorf("mirror TLS Secret for NetbootEndpoint %s: %w", endpoint.Name, err) + } + + if err := env.ApplyObject(ctx, secret); err != nil { + return err + } + + desiredTLSSecrets[secret.Name] = struct{}{} + deployment.Spec.Template.Annotations = map[string]string{ + edgeTLSChecksumAnnotation: tlsSecretChecksum(secret), + } + } + + if err := env.ApplyObject(ctx, deployment); err != nil { + return err + } + + liveDeployment := &appsv1.Deployment{} + if err := env.Client.Get(ctx, client.ObjectKeyFromObject(deployment), liveDeployment); err != nil { + return fmt.Errorf("get managed edge Deployment %s: %w", deployment.Name, err) + } + + if err := reconcileManagedEndpointStatus(ctx, env.Client, endpoint, liveDeployment); err != nil { + return fmt.Errorf("update NetbootEndpoint %s status: %w", endpoint.Name, err) + } + + desiredDeployments[deployment.Name] = struct{}{} + } + + if service != nil { + if err := env.ApplyObject(ctx, service); err != nil { + return err + } + + desiredServices[service.Name] = struct{}{} + } + } + + match := client.MatchingLabels{ + "app.kubernetes.io/component": metalmanEdgeRole, + unboundedv1alpha3.MachineSiteLabelKey: site.Name, + } + + var deployments appsv1.DeploymentList + if err := env.Client.List(ctx, &deployments, client.InNamespace(env.Namespace), match); err != nil { + return fmt.Errorf("list managed edge Deployments: %w", err) + } + + for i := range deployments.Items { + if _, ok := desiredDeployments[deployments.Items[i].Name]; !ok { + if err := env.DeleteIfExists(ctx, &deployments.Items[i]); err != nil { + return err + } + } + } + + var services corev1.ServiceList + if err := env.Client.List(ctx, &services, client.InNamespace(env.Namespace), match); err != nil { + return fmt.Errorf("list managed edge Services: %w", err) + } + + for i := range services.Items { + if _, ok := desiredServices[services.Items[i].Name]; !ok { + if err := env.DeleteIfExists(ctx, &services.Items[i]); err != nil { + return err + } + } + } + + var tlsSecrets corev1.SecretList + if err := env.Client.List(ctx, &tlsSecrets, client.InNamespace(env.Namespace), match); err != nil { + return fmt.Errorf("list managed edge TLS Secrets: %w", err) + } + + for i := range tlsSecrets.Items { + if _, ok := desiredTLSSecrets[tlsSecrets.Items[i].Name]; !ok { + if err := env.DeleteIfExists(ctx, &tlsSecrets.Items[i]); err != nil { + return err + } + } + } + + return nil +} + +func reconcileManagedEndpointStatus( + ctx context.Context, + kubeClient client.Client, + endpoint *unboundedv1alpha3.NetbootEndpoint, + deployment *appsv1.Deployment, +) error { + if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeExternalL2 { + return nil + } + + if deployment == nil { + return fmt.Errorf("managed endpoint has no Deployment") + } + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + current := &unboundedv1alpha3.NetbootEndpoint{} + if err := kubeClient.Get(ctx, client.ObjectKey{Name: endpoint.Name}, current); err != nil { + return err + } + + base := current.DeepCopy() + current.Status.ObservedGeneration = current.Generation + + condition := metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "EdgeUnavailable", + Message: "managed edge Deployment is not available", + ObservedGeneration: current.Generation, + } + if deployment.Status.ObservedGeneration >= deployment.Generation && deployment.Status.AvailableReplicas > 0 { + condition.Status = metav1.ConditionTrue + condition.Reason = "EdgeAvailable" + condition.Message = "managed edge Deployment is available" + } + + apimeta.SetStatusCondition(¤t.Status.Conditions, condition) + + return kubeClient.Status().Patch(ctx, current, client.MergeFrom(base)) + }) +} + +func tlsSecretChecksum(secret *corev1.Secret) string { + digest := sha256.New() + _, _ = digest.Write(secret.Data[corev1.TLSCertKey]) + _, _ = digest.Write([]byte{0}) + _, _ = digest.Write(secret.Data[corev1.TLSPrivateKeyKey]) + + return fmt.Sprintf("%x", digest.Sum(nil)) +} + +func mirroredTLSSecret( + ctx context.Context, + kubeClient client.Client, + endpoint *unboundedv1alpha3.NetbootEndpoint, + site *unboundedv1alpha3.Site, + namespace string, +) (*corev1.Secret, error) { + if endpoint.Spec.TLS.SecretRef == nil { + return nil, fmt.Errorf("TLS secretRef is required") + } + + ref := endpoint.Spec.TLS.SecretRef + + source := &corev1.Secret{} + if err := kubeClient.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, source); err != nil { + return nil, fmt.Errorf("get source Secret %s/%s: %w", ref.Namespace, ref.Name, err) + } + + certificate, certOK := source.Data[corev1.TLSCertKey] + + privateKey, keyOK := source.Data[corev1.TLSPrivateKeyKey] + if !certOK || !keyOK { + return nil, fmt.Errorf("source Secret %s/%s must contain %s and %s", ref.Namespace, ref.Name, corev1.TLSCertKey, corev1.TLSPrivateKeyKey) + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EdgeTLSSecretName(endpoint.Name), + Namespace: namespace, + Labels: endpointEdgeLabels(endpoint, site.Name), + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: append([]byte(nil), certificate...), + corev1.TLSPrivateKeyKey: append([]byte(nil), privateKey...), + }, + }, nil +} + +func requestsForTLSSecret(ctx context.Context, kubeClient client.Client, secret *corev1.Secret) []ctrl.Request { + var endpoints unboundedv1alpha3.NetbootEndpointList + if err := kubeClient.List(ctx, &endpoints); err != nil { + return nil + } + + sites := map[string]struct{}{} + + for i := range endpoints.Items { + ref := endpoints.Items[i].Spec.TLS.SecretRef + if endpoints.Items[i].Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret && ref != nil && + ref.Namespace == secret.Namespace && ref.Name == secret.Name { + sites[endpoints.Items[i].Spec.SiteRef] = struct{}{} + } + } + + names := make([]string, 0, len(sites)) + for site := range sites { + if site != "" { + names = append(names, site) + } + } + + sort.Strings(names) + + requests := make([]ctrl.Request, 0, len(names)) + for _, site := range names { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKey{Name: site}}) + } + + return requests +} + +func endpointEdgeObjects( + endpoint *unboundedv1alpha3.NetbootEndpoint, + site *unboundedv1alpha3.Site, + namespace string, + cfg component.Config, +) (*appsv1.Deployment, *corev1.Service, error) { + if endpoint.Spec.SiteRef != site.Name { + return nil, nil, fmt.Errorf("endpoint site %q does not match %q", endpoint.Spec.SiteRef, site.Name) + } + + if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeExternalL2 { + return nil, nil, nil + } + + labels := endpointEdgeLabels(endpoint, site.Name) + replicas := int32(2) + name := EdgeName(endpoint.Name) + backendURL := fmt.Sprintf("http://%s.%s.svc:8880", ServerName(site.Name), namespace) + args := []string{ + metalmanEdgeRole, + "--backend-url=" + backendURL, + "--endpoint=" + endpoint.Name, + } + ports := []corev1.ContainerPort{{Name: "http", ContainerPort: 8880, Protocol: corev1.ProtocolTCP}} + podSpec := corev1.PodSpec{ServiceAccountName: "metalman-edge"} + maxSurge := intstr.FromInt32(1) + maxUnavailable := intstr.FromInt32(0) + strategy := appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + } + + switch endpoint.Spec.Type { + case unboundedv1alpha3.NetbootEndpointTypeManagedL2: + if endpoint.Spec.ManagedL2 == nil { + return nil, nil, fmt.Errorf("managedL2 configuration is required") + } + + replicas = 1 + + args = append(args, + "--bind-address="+endpoint.Spec.ManagedL2.Address, + "--dhcp-enabled", + "--dhcp-interface="+endpoint.Spec.ManagedL2.Interface, + "--dhcp-server-ip="+endpoint.Spec.ManagedL2.Address, + "--tftp-enabled", + "--tftp-bind-address="+endpoint.Spec.ManagedL2.Address, + ) + ports = append(ports, + corev1.ContainerPort{Name: "dhcp", ContainerPort: 67, Protocol: corev1.ProtocolUDP}, + corev1.ContainerPort{Name: "tftp", ContainerPort: 69, Protocol: corev1.ProtocolUDP}, + ) + + affinity, err := requiredNodeAffinity(endpoint.Spec.ManagedL2.NodeSelector) + if err != nil { + return nil, nil, err + } + + podSpec.HostNetwork = true + podSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + podSpec.Affinity = affinity + strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} + case unboundedv1alpha3.NetbootEndpointTypeHTTP: + if endpoint.Spec.HTTP == nil { + return nil, nil, fmt.Errorf("http configuration is required") + } + default: + return nil, nil, fmt.Errorf("unsupported endpoint type %q", endpoint.Spec.Type) + } + + expirationSeconds := int64(3600) + + container := corev1.Container{ + Name: "metalman", + Image: cfg.Image("metalman"), + ImagePullPolicy: corev1.PullAlways, + Args: args, + Ports: ports, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("50m"), corev1.ResourceMemory: resource.MustParse("64Mi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + } + if endpoint.Spec.Type == unboundedv1alpha3.NetbootEndpointTypeManagedL2 { + container.VolumeMounts = []corev1.VolumeMount{{Name: "edge-token", MountPath: "/var/run/secrets/metalman", ReadOnly: true}} + podSpec.Volumes = []corev1.Volume{{Name: "edge-token", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "metalman-edge", + ExpirationSeconds: &expirationSeconds, + Path: "token", + }}}, + }}}} + } + + if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { + args = append(args, + "--tls-cert-file=/var/run/secrets/metalman-tls/tls.crt", + "--tls-key-file=/var/run/secrets/metalman-tls/tls.key", + ) + container.Args = args + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: "tls", MountPath: "/var/run/secrets/metalman-tls", ReadOnly: true, + }) + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: EdgeTLSSecretName(endpoint.Name)}}, + }) + } + + podSpec.Containers = []corev1.Container{container} + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: strategy, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: podSpec}, + }, + } + + if endpoint.Spec.Type != unboundedv1alpha3.NetbootEndpointTypeHTTP { + return deployment, nil, nil + } + + serviceType := endpoint.Spec.HTTP.ServiceType + if serviceType == "" { + serviceType = corev1.ServiceTypeClusterIP + } + + servicePort := corev1.ServicePort{Name: "http", Port: 8880, TargetPort: intstr.FromInt32(8880), Protocol: corev1.ProtocolTCP} + if endpoint.Spec.TLS.Mode == unboundedv1alpha3.NetbootEndpointTLSSecret { + servicePort.Name = "https" + servicePort.Port = 443 + } + + service := &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{component.SiteOwnerReference(site)}, + }, + Spec: corev1.ServiceSpec{ + Type: serviceType, + Selector: labels, + Ports: []corev1.ServicePort{servicePort}, + }, + } + + return deployment, service, nil +} + +// EdgeName returns the in-cluster workload name for an endpoint. +func EdgeName(endpoint string) string { return "metalman-edge-" + endpoint } + +// EdgeTLSSecretName returns the mirrored serving-certificate Secret name. +func EdgeTLSSecretName(endpoint string) string { return EdgeName(endpoint) + "-tls" } + +func endpointEdgeLabels(endpoint *unboundedv1alpha3.NetbootEndpoint, site string) map[string]string { + return map[string]string{ + "app": "unbounded-metalman", + "app.kubernetes.io/name": "metalman-edge", + "app.kubernetes.io/component": metalmanEdgeRole, + unboundedv1alpha3.MachineSiteLabelKey: site, + netbootEndpointLabel: endpoint.Name, + } +} + +func requiredNodeAffinity(selector metav1.LabelSelector) (*corev1.Affinity, error) { + requirements := make([]corev1.NodeSelectorRequirement, 0, len(selector.MatchLabels)+len(selector.MatchExpressions)) + + keys := make([]string, 0, len(selector.MatchLabels)) + for key := range selector.MatchLabels { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + requirements = append(requirements, corev1.NodeSelectorRequirement{Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{selector.MatchLabels[key]}}) + } + + for _, expression := range selector.MatchExpressions { + operator := corev1.NodeSelectorOperator(expression.Operator) + switch operator { + case corev1.NodeSelectorOpIn, corev1.NodeSelectorOpNotIn, corev1.NodeSelectorOpExists, corev1.NodeSelectorOpDoesNotExist: + default: + return nil, fmt.Errorf("unsupported node selector operator %q", expression.Operator) + } + + requirements = append(requirements, corev1.NodeSelectorRequirement{Key: expression.Key, Operator: operator, Values: expression.Values}) + } + + return &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: requirements}}, + }}}, nil +} diff --git a/internal/operator/components/metalman/metalman_test.go b/internal/operator/components/metalman/metalman_test.go index a7ace1aaa..072f943d1 100644 --- a/internal/operator/components/metalman/metalman_test.go +++ b/internal/operator/components/metalman/metalman_test.go @@ -4,10 +4,13 @@ package metalman import ( + "os" + "strings" "testing" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -23,7 +26,7 @@ func testScheme(t *testing.T) *runtime.Scheme { t.Helper() scheme := runtime.NewScheme() - for _, add := range []func(*runtime.Scheme) error{appsv1.AddToScheme, corev1.AddToScheme, unboundedv1alpha3.AddToScheme} { + for _, add := range []func(*runtime.Scheme) error{appsv1.AddToScheme, corev1.AddToScheme, policyv1.AddToScheme, unboundedv1alpha3.AddToScheme} { if err := add(scheme); err != nil { t.Fatalf("add to scheme: %v", err) } @@ -75,20 +78,180 @@ func TestMutateSupportObject(t *testing.T) { } } +func TestRBACSeparatesControllerAndServerIdentities(t *testing.T) { + data, err := os.ReadFile("../../../../deploy/machina/06-metalman-rbac.yaml.tmpl") + if err != nil { + t.Fatalf("read Metalman RBAC template: %v", err) + } + + manifest := string(data) + + for _, required := range []string{ + "name: metalman-controller", + "name: metalman-server", + "name: metalman-edge", + "resources: [\"tokenreviews\"]", + "resources: [\"serviceaccounts/token\"]", + } { + if !strings.Contains(manifest, required) { + t.Fatalf("RBAC template missing %q", required) + } + } + + serverNamespaceRole := manifest[strings.Index(manifest, "kind: Role\nmetadata:\n name: metalman-server"):] + + serverNamespaceRole = serverNamespaceRole[:strings.Index(serverNamespaceRole, "\n---")] + if !strings.Contains(serverNamespaceRole, "resources: [\"serviceaccounts/token\"]") { + t.Fatalf("metalman-server Role lacks bootstrap-token issuance:\n%s", serverNamespaceRole) + } + + serverClusterRole := manifest[strings.Index(manifest, "kind: ClusterRole\nmetadata:\n name: metalman-server"):] + + serverClusterRole = serverClusterRole[:strings.Index(serverClusterRole, "\n---")] + if !strings.Contains(serverClusterRole, "resources: [\"tokenreviews\"]") { + t.Fatalf("metalman-server ClusterRole lacks TokenReview permission:\n%s", serverClusterRole) + } + + controllerRole := manifest[strings.Index(manifest, "kind: ClusterRole\nmetadata:\n name: metalman-controller\nrules:"):] + + controllerRole = controllerRole[:strings.Index(controllerRole, "\n---")] + if strings.Contains(controllerRole, "resources: [\"tokenreviews\"]") { + t.Fatalf("metalman-controller retains server TokenReview permission:\n%s", controllerRole) + } + + kubeSystemBinding := manifest[strings.Index(manifest, "kind: RoleBinding\nmetadata:\n name: metalman-controller\n namespace: kube-system"):] + + kubeSystemBinding = kubeSystemBinding[:strings.Index(kubeSystemBinding, "\n---")] + if !strings.Contains(kubeSystemBinding, "name: metalman-server") { + t.Fatalf("kube-system metadata binding excludes metalman-server:\n%s", kubeSystemBinding) + } +} + +func TestOperatorRBACCanReconcileNetbootEndpoints(t *testing.T) { + data, err := os.ReadFile("../../../../deploy/unbounded-operator/02-rbac.yaml.tmpl") + if err != nil { + t.Fatalf("read operator RBAC template: %v", err) + } + + manifest := string(data) + for _, required := range []string{ + "resources: [\"netbootendpoints\"]", + "resources: [\"netbootendpoints/status\"]", + "resources: [\"poddisruptionbudgets\"]", + } { + if !strings.Contains(manifest, required) { + t.Fatalf("operator RBAC template missing %q", required) + } + } +} + +func TestReconcileManagedEndpointStatusFromDeploymentAvailability(t *testing.T) { + scheme := testScheme(t) + now := metav1.Now() + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-http", Generation: 3}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: "rack-a", + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + }, + Status: unboundedv1alpha3.NetbootEndpointStatus{Claim: &unboundedv1alpha3.NetbootEndpointClaim{ + HolderIdentity: "edge-pod-1", + RenewedAt: now, + }}, + } + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: EdgeName(endpoint.Name), Namespace: component.DefaultNamespace, Generation: 7}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 7, AvailableReplicas: 1}, + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(endpoint).WithObjects(endpoint, deployment).Build() + + if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, endpoint, deployment); err != nil { + t.Fatalf("reconcileManagedEndpointStatus: %v", err) + } + + updated := &unboundedv1alpha3.NetbootEndpoint{} + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { + t.Fatalf("get endpoint: %v", err) + } + + if updated.Status.ObservedGeneration != endpoint.Generation { + t.Fatalf("observed generation = %d, want %d", updated.Status.ObservedGeneration, endpoint.Generation) + } + + ready := findCondition(updated.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != "EdgeAvailable" { + t.Fatalf("Ready condition = %#v", ready) + } + + if updated.Status.Claim == nil || updated.Status.Claim.HolderIdentity != "edge-pod-1" || updated.Status.Claim.RenewedAt.Unix() != now.Unix() { + t.Fatalf("claim was not preserved: %#v", updated.Status.Claim) + } + + deployment.Status.AvailableReplicas = 0 + if err := kubeClient.Status().Update(t.Context(), deployment); err != nil { + t.Fatalf("update Deployment status: %v", err) + } + + if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, updated, deployment); err != nil { + t.Fatalf("reconcile unavailable endpoint: %v", err) + } + + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { + t.Fatalf("get unavailable endpoint: %v", err) + } + + ready = findCondition(updated.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "EdgeUnavailable" { + t.Fatalf("unavailable Ready condition = %#v", ready) + } +} + +func TestReconcileManagedEndpointStatusLeavesExternalClaimUntouched(t *testing.T) { + scheme := testScheme(t) + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "admin-laptop", Generation: 2}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{SiteRef: "rack-a", Type: unboundedv1alpha3.NetbootEndpointTypeExternalL2}, + Status: unboundedv1alpha3.NetbootEndpointStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "ExternalEdgeReady"}}, + }, + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(endpoint).WithObjects(endpoint).Build() + + if err := reconcileManagedEndpointStatus(t.Context(), kubeClient, endpoint, nil); err != nil { + t.Fatalf("reconcileManagedEndpointStatus: %v", err) + } + + updated := &unboundedv1alpha3.NetbootEndpoint{} + if err := kubeClient.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, updated); err != nil { + t.Fatal(err) + } + + if updated.Status.ObservedGeneration != 1 || updated.Status.Conditions[0].Reason != "ExternalEdgeReady" { + t.Fatalf("external endpoint status changed: %#v", updated.Status) + } +} + +func findCondition(conditions []metav1.Condition, conditionType string) *metav1.Condition { + for i := range conditions { + if conditions[i].Type == conditionType { + return &conditions[i] + } + } + + return nil +} + func TestDeployment(t *testing.T) { enabled := true - dhcpAuto := true - replicas := int32(3) site := &unboundedv1alpha3.Site{ ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}, Spec: unboundedv1alpha3.SiteSpec{Components: unboundedv1alpha3.SiteComponents{Metalman: &unboundedv1alpha3.MetalmanComponentSpec{ SiteComponentSpec: unboundedv1alpha3.SiteComponentSpec{Enabled: &enabled}, - DHCPAutoInterface: &dhcpAuto, - Replicas: &replicas, }}}, } - d := deployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3", APIServerEndpoint: "https://api.example:6443"}) + d := controllerDeployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3", APIServerEndpoint: "https://api.example:6443"}) if d.Name != "metalman-controller-rack-a" { t.Fatalf("name = %q", d.Name) } @@ -102,12 +265,12 @@ func TestDeployment(t *testing.T) { t.Fatalf("image = %q", container.Image) } - if got := container.Args; len(got) != 3 || got[0] != "serve-pxe" || got[1] != "--site=rack-a" || got[2] != "--dhcp-auto-interface" { + if got := container.Args; len(got) != 3 || got[0] != "controller" || got[1] != "--site=rack-a" || got[2] != "--cache-dir=/var/cache/metalman" { t.Fatalf("args = %#v", got) } - if d.Spec.Replicas == nil || *d.Spec.Replicas != 3 { - t.Fatalf("replicas = %v, want 3", d.Spec.Replicas) + if d.Spec.Replicas == nil || *d.Spec.Replicas != 1 { + t.Fatalf("replicas = %v, want 1", d.Spec.Replicas) } for _, path := range []string{"deployment", "selector", "pod"} { @@ -138,19 +301,492 @@ func TestDeployment(t *testing.T) { } assertSiteOwnerRef(t, d.OwnerReferences, "rack-a", "site-uid") - assertSiteAffinity(t, d.Spec.Template.Spec.Affinity, "rack-a") + assertOrdinaryPodNetworking(t, &d.Spec.Template.Spec) strategy := d.Spec.Strategy if strategy.Type != appsv1.RollingUpdateDeploymentStrategyType || strategy.RollingUpdate == nil { t.Fatalf("expected RollingUpdate strategy, got %+v", strategy) } - if got := strategy.RollingUpdate.MaxSurge; got == nil || got.IntValue() != 0 { - t.Fatalf("expected maxSurge=0, got %v", got) + if got := strategy.RollingUpdate.MaxSurge; got == nil || got.IntValue() != 1 { + t.Fatalf("expected maxSurge=1, got %v", got) + } + + if got := strategy.RollingUpdate.MaxUnavailable; got == nil || got.IntValue() != 0 { + t.Fatalf("expected maxUnavailable=0, got %v", got) + } +} + +func TestControllerAndServerWorkloadsAreSeparated(t *testing.T) { + site := &unboundedv1alpha3.Site{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}, + } + cfg := component.Config{ + ImageRegistry: "registry.example.com", + ImageTag: "v1.2.3", + APIServerEndpoint: "https://api.example:6443", + } + + controller := controllerDeployment(site, component.DefaultNamespace, cfg) + if controller.Spec.Replicas == nil || *controller.Spec.Replicas != 1 { + t.Fatalf("controller replicas = %v, want 1", controller.Spec.Replicas) + } + + assertOrdinaryPodNetworking(t, &controller.Spec.Template.Spec) + + controllerContainer := controller.Spec.Template.Spec.Containers[0] + if got := controllerContainer.Args; len(got) < 2 || got[0] != "controller" || got[1] != "--site=rack-a" { + t.Fatalf("controller args = %#v", got) + } + + if controller.Spec.Template.Spec.ServiceAccountName != "metalman-controller" { + t.Fatalf("controller service account = %q", controller.Spec.Template.Spec.ServiceAccountName) + } + + assertCapabilityKeyMount(t, &controller.Spec.Template.Spec, &controllerContainer, "rack-a") + + for _, port := range controllerContainer.Ports { + if port.Name == "http" || port.Name == "dhcp" || port.Name == "tftp" { + t.Fatalf("controller exposes data-plane port %#v", port) + } + } + + server := serverDeployment(site, component.DefaultNamespace, cfg) + if server.Spec.Replicas == nil || *server.Spec.Replicas != 2 { + t.Fatalf("server replicas = %v, want 2", server.Spec.Replicas) + } + + assertOrdinaryPodNetworking(t, &server.Spec.Template.Spec) + + serverContainer := server.Spec.Template.Spec.Containers[0] + if got := serverContainer.Args; len(got) < 3 || got[0] != "server" || got[1] != "--site=rack-a" || got[2] != "--cache-dir=/var/cache/metalman" { + t.Fatalf("server args = %#v", got) + } + + if server.Spec.Template.Spec.ServiceAccountName != "metalman-server" { + t.Fatalf("server service account = %q", server.Spec.Template.Spec.ServiceAccountName) + } + + assertCapabilityKeyMount(t, &server.Spec.Template.Spec, &serverContainer, "rack-a") + + if !hasContainerPort(serverContainer.Ports, "http", 8880) { + t.Fatalf("server ports = %#v, want HTTP 8880", serverContainer.Ports) + } + + assertWorkloadHealthAndResources(t, &serverContainer) + + if len(server.Spec.Template.Spec.TopologySpreadConstraints) != 1 { + t.Fatalf("server topology spread constraints = %#v", server.Spec.Template.Spec.TopologySpreadConstraints) + } + + spread := server.Spec.Template.Spec.TopologySpreadConstraints[0] + if spread.TopologyKey != corev1.LabelHostname || spread.MaxSkew != 1 || spread.WhenUnsatisfiable != corev1.ScheduleAnyway { + t.Fatalf("server topology spread = %#v", spread) + } + + if got := server.Spec.Strategy.RollingUpdate; got == nil || got.MaxUnavailable == nil || got.MaxUnavailable.IntValue() != 0 { + t.Fatalf("server maxUnavailable = %#v, want 0", got) + } + + service := serverService(site, component.DefaultNamespace) + if service.Name != ServerName("rack-a") { + t.Fatalf("service name = %q, want %q", service.Name, ServerName("rack-a")) + } + + if service.Spec.Selector["app.kubernetes.io/name"] != "metalman-server" || service.Spec.Selector[unboundedv1alpha3.MachineSiteLabelKey] != "rack-a" { + t.Fatalf("service selector = %#v", service.Spec.Selector) + } + + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != 8880 || service.Spec.Ports[0].TargetPort.IntValue() != 8880 { + t.Fatalf("service ports = %#v", service.Spec.Ports) + } + + pdb := serverPodDisruptionBudget(site, component.DefaultNamespace) + if pdb.Spec.MinAvailable == nil || pdb.Spec.MinAvailable.IntValue() != 1 { + t.Fatalf("PDB minAvailable = %#v, want 1", pdb.Spec.MinAvailable) + } + + if pdb.Spec.Selector == nil || pdb.Spec.Selector.MatchLabels["app.kubernetes.io/name"] != "metalman-server" { + t.Fatalf("PDB selector = %#v", pdb.Spec.Selector) + } +} + +func TestEndpointEdgeWorkloadMatrix(t *testing.T) { + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + cfg := component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"} + + t.Run("managed L2 is the only host-network edge", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "rack-a-lan"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeManagedL2, + ExternalURL: "http://192.0.2.10:8880", + ManagedL2: &unboundedv1alpha3.NetbootManagedL2Spec{ + NodeSelector: metav1.LabelSelector{MatchLabels: map[string]string{"provisioning-lan": "rack-a"}}, + Interface: "eno2", + Address: "192.0.2.10", + }, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + + if deployment == nil || service != nil { + t.Fatalf("managed L2 objects = deployment %v, service %v", deployment, service) + } + + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("managed L2 replicas = %v, want 1", deployment.Spec.Replicas) + } + + pod := &deployment.Spec.Template.Spec + if !pod.HostNetwork || pod.DNSPolicy != corev1.DNSClusterFirstWithHostNet { + t.Fatalf("managed L2 networking = hostNetwork %v, dnsPolicy %q", pod.HostNetwork, pod.DNSPolicy) + } + + if pod.Affinity == nil || pod.Affinity.NodeAffinity == nil || + pod.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + t.Fatalf("managed L2 edge lacks required placement: %#v", pod.Affinity) + } + + container := &pod.Containers[0] + for _, arg := range []string{ + "edge", + "--backend-url=http://metalman-server-rack-a." + component.DefaultNamespace + ".svc:8880", + "--endpoint=rack-a-lan", + "--dhcp-enabled", + "--dhcp-interface=eno2", + "--dhcp-server-ip=192.0.2.10", + "--tftp-enabled", + } { + if !containsString(container.Args, arg) { + t.Fatalf("managed L2 args %#v lack %q", container.Args, arg) + } + } + + if pod.ServiceAccountName != "metalman-edge" { + t.Fatalf("managed L2 service account = %q", pod.ServiceAccountName) + } + + assertProjectedEdgeToken(t, pod, container) + + if !hasContainerPort(container.Ports, "http", 8880) || + !hasContainerPort(container.Ports, "dhcp", 67) || + !hasContainerPort(container.Ports, "tftp", 69) { + t.Fatalf("managed L2 ports = %#v", container.Ports) + } + }) + + t.Run("HTTP edge uses ordinary replicated pods and a Service", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-http"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + ExternalURL: "https://boot.example.com", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSExternal, + }, + HTTP: &unboundedv1alpha3.NetbootHTTPEndpointSpec{ServiceType: corev1.ServiceTypeNodePort}, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + + if deployment == nil || service == nil { + t.Fatalf("HTTP objects = deployment %v, service %v", deployment, service) + } + + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 2 { + t.Fatalf("HTTP replicas = %v, want 2", deployment.Spec.Replicas) + } + + assertOrdinaryPodNetworking(t, &deployment.Spec.Template.Spec) + + args := deployment.Spec.Template.Spec.Containers[0].Args + if containsString(args, "--dhcp-enabled") || containsString(args, "--tftp-enabled") { + t.Fatalf("HTTP edge enables L2 protocols: %#v", args) + } + + if service.Spec.Type != corev1.ServiceTypeNodePort || service.Spec.Selector[netbootEndpointLabel] != endpoint.Name { + t.Fatalf("HTTP edge Service = %#v", service.Spec) + } + }) + + t.Run("Secret TLS mounts the mirrored certificate", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-https"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + ExternalURL: "https://boot.example.com", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSSecret, + SecretRef: &unboundedv1alpha3.NamespacedSecretReference{ + Namespace: "certificates", + Name: "boot-example-com", + }, + }, + HTTP: &unboundedv1alpha3.NetbootHTTPEndpointSpec{}, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + + container := &deployment.Spec.Template.Spec.Containers[0] + for _, arg := range []string{ + "--tls-cert-file=/var/run/secrets/metalman-tls/tls.crt", + "--tls-key-file=/var/run/secrets/metalman-tls/tls.key", + } { + if !containsString(container.Args, arg) { + t.Fatalf("Secret TLS args %#v lack %q", container.Args, arg) + } + } + + if len(deployment.Spec.Template.Spec.Volumes) != 1 || + deployment.Spec.Template.Spec.Volumes[0].Secret == nil || + deployment.Spec.Template.Spec.Volumes[0].Secret.SecretName != EdgeTLSSecretName(endpoint.Name) { + t.Fatalf("Secret TLS volumes = %#v", deployment.Spec.Template.Spec.Volumes) + } + + if len(container.VolumeMounts) != 1 || container.VolumeMounts[0].MountPath != "/var/run/secrets/metalman-tls" || !container.VolumeMounts[0].ReadOnly { + t.Fatalf("Secret TLS mounts = %#v", container.VolumeMounts) + } + + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Name != "https" || service.Spec.Ports[0].Port != 443 { + t.Fatalf("Secret TLS Service ports = %#v", service.Spec.Ports) + } + }) + + t.Run("external L2 has no in-cluster workload", func(t *testing.T) { + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "admin-laptop"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeExternalL2, + }, + } + + deployment, service, err := endpointEdgeObjects(endpoint, site, component.DefaultNamespace, cfg) + if err != nil { + t.Fatalf("endpointEdgeObjects: %v", err) + } + + if deployment != nil || service != nil { + t.Fatalf("external L2 objects = deployment %v, service %v", deployment, service) + } + }) +} + +func TestReconcileEndpointEdgesMirrorsRotatesAndRemovesTLSSecret(t *testing.T) { + scheme := testScheme(t) + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + source := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "boot-example-com", Namespace: "certificates"}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{corev1.TLSCertKey: []byte("certificate-v1"), corev1.TLSPrivateKeyKey: []byte("private-key-v1")}, + } + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-https"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeHTTP, + ExternalURL: "https://boot.example.com", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSSecret, + SecretRef: &unboundedv1alpha3.NamespacedSecretReference{Namespace: source.Namespace, Name: source.Name}, + }, + HTTP: &unboundedv1alpha3.NetbootHTTPEndpointSpec{}, + }, + } + env := &component.Env{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(endpoint).WithObjects(source, endpoint).Build(), + Scheme: scheme, Namespace: component.DefaultNamespace, + Config: component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}, + } + + assertMirror := func(wantCert string) string { + t.Helper() + + if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { + t.Fatalf("reconcileEndpointEdges: %v", err) + } + + mirror := &corev1.Secret{} + + key := client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeTLSSecretName(endpoint.Name)} + if err := env.Client.Get(t.Context(), key, mirror); err != nil { + t.Fatalf("get mirrored TLS Secret: %v", err) + } + + if got := string(mirror.Data[corev1.TLSCertKey]); got != wantCert { + t.Fatalf("mirrored certificate = %q, want %q", got, wantCert) + } + + if got := string(mirror.Data[corev1.TLSPrivateKeyKey]); got != "private-key-v1" { + t.Fatalf("mirrored private key = %q", got) + } + + deployment := &appsv1.Deployment{} + if err := env.Client.Get(t.Context(), client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeName(endpoint.Name)}, deployment); err != nil { + t.Fatalf("get TLS edge Deployment: %v", err) + } + + checksum := deployment.Spec.Template.Annotations[edgeTLSChecksumAnnotation] + if checksum == "" { + t.Fatal("TLS edge pod template lacks certificate checksum") + } + + return checksum + } + + firstChecksum := assertMirror("certificate-v1") + + source.Data[corev1.TLSCertKey] = []byte("certificate-v2") + if err := env.Client.Update(t.Context(), source); err != nil { + t.Fatalf("update source TLS Secret: %v", err) + } + + if secondChecksum := assertMirror("certificate-v2"); secondChecksum == firstChecksum { + t.Fatalf("TLS edge checksum did not change after certificate rotation: %q", secondChecksum) + } + + if err := env.Client.Get(t.Context(), client.ObjectKey{Name: endpoint.Name}, endpoint); err != nil { + t.Fatalf("refresh endpoint: %v", err) + } + + endpoint.Spec.TLS = unboundedv1alpha3.NetbootEndpointTLS{ + Trust: unboundedv1alpha3.NetbootEndpointTrustPublic, + Mode: unboundedv1alpha3.NetbootEndpointTLSExternal, + } + if err := env.Client.Update(t.Context(), endpoint); err != nil { + t.Fatalf("update endpoint TLS mode: %v", err) + } + + if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { + t.Fatalf("reconcile external TLS endpoint: %v", err) + } + + err := env.Client.Get(t.Context(), client.ObjectKey{Namespace: component.DefaultNamespace, Name: EdgeTLSSecretName(endpoint.Name)}, &corev1.Secret{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("mirrored TLS Secret still exists: %v", err) + } +} + +func TestTLSSecretChangeEnqueuesReferencingSite(t *testing.T) { + scheme := testScheme(t) + endpoint := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "public-https"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: "rack-a", + TLS: unboundedv1alpha3.NetbootEndpointTLS{ + Mode: unboundedv1alpha3.NetbootEndpointTLSSecret, + SecretRef: &unboundedv1alpha3.NamespacedSecretReference{Namespace: "certificates", Name: "boot-example-com"}, + }, + }, + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(endpoint).Build() + + requests := requestsForTLSSecret(t.Context(), kubeClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Namespace: "certificates", + Name: "boot-example-com", + }}) + if len(requests) != 1 || requests[0].Name != "rack-a" { + t.Fatalf("requests = %#v, want rack-a", requests) + } +} + +func TestReconcileEndpointEdgesDeletesStaleManagedWorkloads(t *testing.T) { + scheme := testScheme(t) + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + staleLabels := map[string]string{ + "app.kubernetes.io/component": metalmanEdgeRole, + unboundedv1alpha3.MachineSiteLabelKey: site.Name, + netbootEndpointLabel: "removed-endpoint", + } + staleDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ + Name: EdgeName("removed-endpoint"), + Namespace: component.DefaultNamespace, + Labels: staleLabels, + }} + staleService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: EdgeName("removed-endpoint"), + Namespace: component.DefaultNamespace, + Labels: staleLabels, + }} + external := &unboundedv1alpha3.NetbootEndpoint{ + ObjectMeta: metav1.ObjectMeta{Name: "external-endpoint"}, + Spec: unboundedv1alpha3.NetbootEndpointSpec{ + SiteRef: site.Name, + Type: unboundedv1alpha3.NetbootEndpointTypeExternalL2, + }, + } + env := &component.Env{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(staleDeployment, staleService, external).Build(), + Scheme: scheme, Namespace: component.DefaultNamespace, + Config: component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}, + } + + if err := reconcileEndpointEdges(t.Context(), env, site); err != nil { + t.Fatalf("reconcileEndpointEdges: %v", err) + } + + for _, object := range []client.Object{staleDeployment, staleService} { + err := env.Client.Get(t.Context(), client.ObjectKeyFromObject(object), object) + if !apierrors.IsNotFound(err) { + t.Fatalf("stale %T still exists: err=%v", object, err) + } + } +} + +func TestEnsureCapabilitySecretPreservesExistingKey(t *testing.T) { + scheme := testScheme(t) + env := &component.Env{ + Client: fake.NewClientBuilder().WithScheme(scheme).Build(), + Scheme: scheme, + Namespace: component.DefaultNamespace, + } + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a", UID: "site-uid"}} + + if err := ensureCapabilitySecret(t.Context(), env, site); err != nil { + t.Fatalf("first ensureCapabilitySecret: %v", err) + } + + key := client.ObjectKey{Namespace: component.DefaultNamespace, Name: CapabilitySecretName("rack-a")} + + first := &corev1.Secret{} + if err := env.Client.Get(t.Context(), key, first); err != nil { + t.Fatalf("get first capability Secret: %v", err) + } + + if len(first.Data[capabilitySecretKey]) != 32 { + t.Fatalf("capability key length = %d, want 32", len(first.Data[capabilitySecretKey])) + } + + if err := ensureCapabilitySecret(t.Context(), env, site); err != nil { + t.Fatalf("second ensureCapabilitySecret: %v", err) + } + + second := &corev1.Secret{} + if err := env.Client.Get(t.Context(), key, second); err != nil { + t.Fatalf("get second capability Secret: %v", err) } - if got := strategy.RollingUpdate.MaxUnavailable; got == nil || got.IntValue() != 1 { - t.Fatalf("expected maxUnavailable=1, got %v", got) + if string(second.Data[capabilitySecretKey]) != string(first.Data[capabilitySecretKey]) { + t.Fatal("capability key changed across reconciliation") } } @@ -163,7 +799,7 @@ func TestDeploymentRespectsNamespaceAndDefaults(t *testing.T) { }}}, } - d := deployment(site, "custom-ns", component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) + d := controllerDeployment(site, "custom-ns", component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) if d.Namespace != "custom-ns" { t.Fatalf("namespace = %q, want custom-ns", d.Namespace) } @@ -178,19 +814,13 @@ func TestDeploymentRespectsNamespaceAndDefaults(t *testing.T) { } func TestDeploymentAllowsZeroReplicas(t *testing.T) { - enabled := true - replicas := int32(0) - site := &unboundedv1alpha3.Site{ - ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}, - Spec: unboundedv1alpha3.SiteSpec{Components: unboundedv1alpha3.SiteComponents{Metalman: &unboundedv1alpha3.MetalmanComponentSpec{ - SiteComponentSpec: unboundedv1alpha3.SiteComponentSpec{Enabled: &enabled}, - Replicas: &replicas, - }}}, - } + // The split roles have fixed availability semantics; the former Site-level + // replica knob no longer scales the controller and data plane together. + site := &unboundedv1alpha3.Site{ObjectMeta: metav1.ObjectMeta{Name: "rack-a"}} - d := deployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) - if d.Spec.Replicas == nil || *d.Spec.Replicas != 0 { - t.Fatalf("replicas = %v, want 0", d.Spec.Replicas) + d := serverDeployment(site, component.DefaultNamespace, component.Config{ImageRegistry: "registry.example.com", ImageTag: "v1.2.3"}) + if d.Spec.Replicas == nil || *d.Spec.Replicas != 2 { + t.Fatalf("server replicas = %v, want 2", d.Spec.Replicas) } } @@ -220,6 +850,64 @@ func findEnv(env []corev1.EnvVar, name string) *corev1.EnvVar { return nil } +func assertOrdinaryPodNetworking(t *testing.T, podSpec *corev1.PodSpec) { + t.Helper() + + if podSpec.HostNetwork { + t.Fatal("pod unexpectedly uses host networking") + } + + if podSpec.Affinity != nil && podSpec.Affinity.NodeAffinity != nil && + podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + t.Fatalf("pod unexpectedly has required node affinity: %#v", podSpec.Affinity) + } +} + +func hasContainerPort(ports []corev1.ContainerPort, name string, port int32) bool { + for _, candidate := range ports { + if candidate.Name == name && candidate.ContainerPort == port { + return true + } + } + + return false +} + +func assertCapabilityKeyMount(t *testing.T, podSpec *corev1.PodSpec, container *corev1.Container, site string) { + t.Helper() + + for _, mount := range container.VolumeMounts { + if mount.Name == "capability-key" && mount.MountPath == "/var/run/secrets/metalman" && mount.ReadOnly { + for _, volume := range podSpec.Volumes { + if volume.Name == mount.Name && volume.Secret != nil && volume.Secret.SecretName == CapabilitySecretName(site) { + return + } + } + } + } + + t.Fatalf("missing read-only capability key mount: mounts=%#v volumes=%#v", container.VolumeMounts, podSpec.Volumes) +} + +func assertWorkloadHealthAndResources(t *testing.T, container *corev1.Container) { + t.Helper() + + for name, probe := range map[string]*corev1.Probe{"liveness": container.LivenessProbe, "readiness": container.ReadinessProbe} { + if probe == nil || probe.HTTPGet == nil || probe.HTTPGet.Port.IntValue() != 8081 { + t.Fatalf("%s probe = %#v, want HTTP probe on 8081", name, probe) + } + } + + if container.LivenessProbe.HTTPGet.Path != "/healthz" || container.ReadinessProbe.HTTPGet.Path != "/readyz" { + t.Fatalf("probe paths = %q, %q", container.LivenessProbe.HTTPGet.Path, container.ReadinessProbe.HTTPGet.Path) + } + + if container.Resources.Requests.Cpu().IsZero() || container.Resources.Requests.Memory().IsZero() || + container.Resources.Limits.Cpu().IsZero() || container.Resources.Limits.Memory().IsZero() { + t.Fatalf("resources are incomplete: %#v", container.Resources) + } +} + func assertSiteOwnerRef(t *testing.T, refs []metav1.OwnerReference, siteName, uid string) { t.Helper() @@ -243,40 +931,35 @@ func assertSiteOwnerRef(t *testing.T, refs []metav1.OwnerReference, siteName, ui } } -func assertSiteAffinity(t *testing.T, affinity *corev1.Affinity, siteName string) { - t.Helper() - - if affinity == nil || affinity.NodeAffinity == nil || affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { - t.Fatalf("missing node affinity: %#v", affinity) +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } } - terms := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms - if len(terms) != 2 { - t.Fatalf("node selector terms len = %d, want 2: %#v", len(terms), terms) - } + return false +} - want := map[string]bool{component.SiteLabelKey: false, component.DeprecatedSiteLabelKey: false} +func assertProjectedEdgeToken(t *testing.T, pod *corev1.PodSpec, container *corev1.Container) { + t.Helper() - for _, term := range terms { - if len(term.MatchExpressions) != 1 { - t.Fatalf("term must have one expression: %#v", term) + for _, mount := range container.VolumeMounts { + if mount.Name != "edge-token" || mount.MountPath != "/var/run/secrets/metalman" || !mount.ReadOnly { + continue } - expr := term.MatchExpressions[0] - if expr.Operator != corev1.NodeSelectorOpIn || len(expr.Values) != 1 || expr.Values[0] != siteName { - t.Fatalf("unexpected site affinity expression: %#v", expr) - } + for _, volume := range pod.Volumes { + if volume.Name != mount.Name || volume.Projected == nil || len(volume.Projected.Sources) != 1 { + continue + } - if _, ok := want[expr.Key]; !ok { - t.Fatalf("unexpected site affinity key %q", expr.Key) + token := volume.Projected.Sources[0].ServiceAccountToken + if token != nil && token.Audience == "metalman-edge" && token.Path == "token" { + return + } } - - want[expr.Key] = true } - for key, seen := range want { - if !seen { - t.Fatalf("site affinity missing key %q", key) - } - } + t.Fatalf("missing audience-bound projected edge token: mounts=%#v volumes=%#v", container.VolumeMounts, pod.Volumes) } diff --git a/internal/operator/migrate.go b/internal/operator/migrate.go index 402be2def..ffb10e6b4 100644 --- a/internal/operator/migrate.go +++ b/internal/operator/migrate.go @@ -542,18 +542,13 @@ func (r *LegacyReaper) detectComponents(ctx context.Context, siteName string) (m } } - metalman, dhcpAutoInterface, err := r.legacyMetalmanConfigForSite(ctx, siteName) + metalman, err := r.legacyMetalmanExistsForSite(ctx, siteName) if err != nil { return nil, err } if metalman { - config := map[string]any{"enabled": true} - if dhcpAutoInterface != nil { - config["dhcpAutoInterface"] = *dhcpAutoInterface - } - - components["metalman"] = config + components["metalman"] = map[string]any{"enabled": true} } return components, nil @@ -567,28 +562,13 @@ func (r *LegacyReaper) detectComponents(ctx context.Context, siteName string) (m // and the operator use) guards against the site label being carried under the // deprecated key on older clusters. func (r *LegacyReaper) legacyMetalmanExistsForSite(ctx context.Context, siteName string) (bool, error) { - found, _, err := r.legacyMetalmanConfigForSite(ctx, siteName) - - return found, err -} - -// legacyMetalmanConfigForSite finds the legacy per-site Metalman Deployment and -// preserves its --dhcp-auto-interface setting. Malformed or contradictory flag -// values block translation rather than silently changing DHCP behavior. -func (r *LegacyReaper) legacyMetalmanConfigForSite(ctx context.Context, siteName string) (bool, *bool, error) { reader := r.liveReader() - - var ( - dhcpAutoInterface *bool - effectiveValue *bool - ) - found := false for _, legacyNs := range r.LegacyNamespaces { var list appsv1.DeploymentList if err := reader.List(ctx, &list, client.InNamespace(legacyNs), client.MatchingLabels{"app": "unbounded-pxe"}); err != nil { - return false, nil, err + return false, err } for i := range list.Items { @@ -600,62 +580,6 @@ func (r *LegacyReaper) legacyMetalmanConfigForSite(ctx context.Context, siteName } found = true - - value, err := metalmanDHCPAutoInterface(d) - if err != nil { - return false, nil, err - } - - effective := false - if value != nil { - effective = *value - } - - if effectiveValue != nil && *effectiveValue != effective { - return false, nil, fmt.Errorf("legacy Metalman Deployments for Site %s have conflicting --dhcp-auto-interface values", siteName) - } - - matchedEffective := effective - effectiveValue = &matchedEffective - - if value != nil { - dhcpAutoInterface = value - } - } - } - - return found, dhcpAutoInterface, nil -} - -func metalmanDHCPAutoInterface(deploy *appsv1.Deployment) (*bool, error) { - var found *bool - - for _, container := range deploy.Spec.Template.Spec.Containers { - for _, arg := range container.Args { - var value bool - - switch { - case arg == "--dhcp-auto-interface": - value = true - case strings.HasPrefix(arg, "--dhcp-auto-interface="): - switch strings.TrimPrefix(arg, "--dhcp-auto-interface=") { - case "true": - value = true - case "false": - value = false - default: - return nil, fmt.Errorf("legacy Metalman Deployment %s/%s has invalid argument %q", deploy.Namespace, deploy.Name, arg) - } - default: - continue - } - - if found != nil && *found != value { - return nil, fmt.Errorf("legacy Metalman Deployment %s/%s has conflicting --dhcp-auto-interface arguments", deploy.Namespace, deploy.Name) - } - - matched := value - found = &matched } } diff --git a/internal/operator/migrate_gates_test.go b/internal/operator/migrate_gates_test.go index 9c1675eb4..c83f2a064 100644 --- a/internal/operator/migrate_gates_test.go +++ b/internal/operator/migrate_gates_test.go @@ -377,25 +377,25 @@ func TestLegacyMetalmanDetectionHardening(t *testing.T) { } }) - t.Run("conflicting matching Deployments fail closed", func(t *testing.T) { + t.Run("conflicting removed arguments do not block detection", func(t *testing.T) { byName := metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", "--dhcp-auto-interface") byLabel := metalmanDeploymentForSiteWithArgs(legacyNetNamespace, "edge", "--dhcp-auto-interface=false") byLabel.Name = "older-metalman-name" r := newReaper(t, byName, byLabel) - if _, _, err := r.legacyMetalmanConfigForSite(t.Context(), "edge"); err == nil { - t.Fatal("expected conflicting matching Metalman Deployments to fail closed") + if got, err := r.legacyMetalmanExistsForSite(t.Context(), "edge"); err != nil || !got { + t.Fatalf("legacyMetalmanExistsForSite = %t, err=%v; want true", got, err) } }) - t.Run("absent and enabled arguments conflict", func(t *testing.T) { + t.Run("absent and enabled removed arguments do not conflict", func(t *testing.T) { withoutFlag := metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", "serve-pxe") withFlag := metalmanDeploymentForSiteWithArgs(legacyNetNamespace, "edge", "--dhcp-auto-interface") withFlag.Name = "older-metalman-name" r := newReaper(t, withoutFlag, withFlag) - if _, _, err := r.legacyMetalmanConfigForSite(t.Context(), "edge"); err == nil { - t.Fatal("expected absent and enabled Metalman arguments to conflict") + if got, err := r.legacyMetalmanExistsForSite(t.Context(), "edge"); err != nil || !got { + t.Fatalf("legacyMetalmanExistsForSite = %t, err=%v; want true", got, err) } }) } diff --git a/internal/operator/migrate_test.go b/internal/operator/migrate_test.go index cd71d013d..92b82727b 100644 --- a/internal/operator/migrate_test.go +++ b/internal/operator/migrate_test.go @@ -230,15 +230,14 @@ func TestTranslateSitesCreatesMachinaSite(t *testing.T) { } } -func TestTranslateSitesPreservesMetalmanDHCPAutoInterface(t *testing.T) { +func TestTranslateSitesDropsLegacyMetalmanDHCPAutoInterface(t *testing.T) { tests := []struct { name string arg string - want bool }{ - {name: "bare", arg: "--dhcp-auto-interface", want: true}, - {name: "explicit true", arg: "--dhcp-auto-interface=true", want: true}, - {name: "explicit false", arg: "--dhcp-auto-interface=false", want: false}, + {name: "bare", arg: "--dhcp-auto-interface"}, + {name: "explicit true", arg: "--dhcp-auto-interface=true"}, + {name: "explicit false", arg: "--dhcp-auto-interface=false"}, } for _, tt := range tests { @@ -259,53 +258,35 @@ func TestTranslateSitesPreservesMetalmanDHCPAutoInterface(t *testing.T) { t.Fatalf("get translated Site: %v", err) } - value, found, err := unstructured.NestedBool(got.Object, "spec", "components", "metalman", "dhcpAutoInterface") - if err != nil || !found || value != tt.want { - t.Fatalf("dhcpAutoInterface = %t, found=%t, err=%v; want %t", value, found, err, tt.want) + _, found, err := unstructured.NestedBool(got.Object, "spec", "components", "metalman", "dhcpAutoInterface") + if err != nil || found { + t.Fatalf("dhcpAutoInterface found=%t, err=%v; want removed", found, err) } }) } } -func TestMetalmanDHCPAutoInterface(t *testing.T) { +func TestLegacyMetalmanDetectionIgnoresRemovedDHCPFlag(t *testing.T) { tests := []struct { - name string - args []string - want *bool - wantErr bool + name string + args []string }{ {name: "absent", args: []string{"serve-pxe"}}, - {name: "bare", args: []string{"--dhcp-auto-interface"}, want: boolPtr(true)}, - {name: "explicit true", args: []string{"--dhcp-auto-interface=true"}, want: boolPtr(true)}, - {name: "explicit false", args: []string{"--dhcp-auto-interface=false"}, want: boolPtr(false)}, - {name: "repeated same value", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=true"}, want: boolPtr(true)}, - {name: "invalid", args: []string{"--dhcp-auto-interface=yes"}, wantErr: true}, - {name: "conflicting", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=false"}, wantErr: true}, + {name: "bare", args: []string{"--dhcp-auto-interface"}}, + {name: "explicit true", args: []string{"--dhcp-auto-interface=true"}}, + {name: "explicit false", args: []string{"--dhcp-auto-interface=false"}}, + {name: "repeated same value", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=true"}}, + {name: "invalid", args: []string{"--dhcp-auto-interface=yes"}}, + {name: "conflicting", args: []string{"--dhcp-auto-interface", "--dhcp-auto-interface=false"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - deploy := metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", tt.args...) - - got, err := metalmanDHCPAutoInterface(deploy) - if (err != nil) != tt.wantErr { - t.Fatalf("metalmanDHCPAutoInterface error = %v, wantErr %t", err, tt.wantErr) - } - - if tt.wantErr { - return - } - - if got == nil || tt.want == nil { - if got != nil || tt.want != nil { - t.Fatalf("metalmanDHCPAutoInterface = %v, want %v", got, tt.want) - } - - return - } + r := newReaper(t, metalmanDeploymentForSiteWithArgs(legacyKubeNamespace, "edge", tt.args...)) - if *got != *tt.want { - t.Fatalf("metalmanDHCPAutoInterface = %t, want %t", *got, *tt.want) + got, err := r.legacyMetalmanExistsForSite(t.Context(), "edge") + if err != nil || !got { + t.Fatalf("legacyMetalmanExistsForSite = %t, err=%v; want true", got, err) } }) }