diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index c3cd57e34..f401cd45e 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -30,6 +30,8 @@ func Run() { newCmdDaemon(cmdCtx), newCmdReset(cmdCtx), newCmdVersion(), + newCmdRegenerateConfig(cmdCtx), + newCmdRegenerateNSpawnConfig(cmdCtx), newCmdRecordAgentUpgradeFailureSignal(), ) diff --git a/cmd/agent/internal/cmd/nspawn_config.go b/cmd/agent/internal/cmd/nspawn_config.go new file mode 100644 index 000000000..50e178647 --- /dev/null +++ b/cmd/agent/internal/cmd/nspawn_config.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + + "github.com/spf13/cobra" + + "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/phases" + "github.com/Azure/unbounded/pkg/agent/phases/rootfs" +) + +func newCmdRegenerateConfig(cmdCtx *CommandContext) *cobra.Command { + cmd := &cobra.Command{ + Use: "regenerate-config MACHINE_NAME", + Short: "Regenerate host-side configuration for a machine", + Hidden: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) + defer cancel() + + cmdCtx.Setup() + + return regenerateConfig(ctx, cmdCtx.Logger, args[0]) + }, + } + + return cmd +} + +func newCmdRegenerateNSpawnConfig(cmdCtx *CommandContext) *cobra.Command { + cmd := &cobra.Command{ + Use: "regenerate-nspawn-config MACHINE_NAME", + Short: "Regenerate host-side nspawn configuration for a machine", + Hidden: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) + defer cancel() + + cmdCtx.Setup() + + return regenerateNSpawnConfig(ctx, cmdCtx.Logger, args[0]) + }, + } + + return cmd +} + +func regenerateConfig(ctx context.Context, log *slog.Logger, machineName string) error { + return regenerateNSpawnConfig(ctx, log, machineName) +} + +func regenerateNSpawnConfig(ctx context.Context, log *slog.Logger, machineName string) error { + cfg, ok, err := loadAppliedConfigForMachine(log, machineName) + if err != nil { + return err + } + + if !ok { + log.Info("applied config not found, skipping nspawn config regeneration", "machine", machineName) + return nil + } + + rootFS, err := goalstates.ResolveNSpawnConfig(cfg, machineName) + if err != nil { + return fmt.Errorf("resolve nspawn config goal state: %w", err) + } + + if err := phases.ExecuteTask(ctx, log, rootfs.EnsureNSpawnConfig(log, rootFS)); err != nil { + return fmt.Errorf("regenerate nspawn config for %s: %w", machineName, err) + } + + // systemd loaded the nspawn service drop-in before starting this required + // oneshot unit. Reload the manager so the pending nspawn start observes the + // regenerated service properties, including path-specific DeviceAllow entries. + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "daemon-reload"); err != nil { + return fmt.Errorf("reload systemd after regenerating config for %s: %w", machineName, err) + } + + return nil +} + +func loadAppliedConfigForMachine(log *slog.Logger, machineName string) (*provision.AgentConfig, bool, error) { + if machineName != goalstates.NSpawnMachineKube1 && machineName != goalstates.NSpawnMachineKube2 { + return nil, false, fmt.Errorf("unsupported nspawn machine %q", machineName) + } + + return loadAppliedConfig(log, goalstates.AppliedConfigPath(machineName), goalstates.AppliedConfigChecksumPath(machineName)) +} + +func loadAppliedConfig(log *slog.Logger, path, checksumPath string) (*provision.AgentConfig, bool, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + + if err != nil { + return nil, false, fmt.Errorf("read applied config %s: %w", path, err) + } + + if err := goalstates.VerifyChecksum(data, checksumPath); err != nil { + return nil, false, fmt.Errorf("verify applied config checksum for %s: %w", path, err) + } + + if _, statErr := os.Stat(checksumPath); errors.Is(statErr, os.ErrNotExist) { + log.Warn( + "no checksum sidecar found, skipping integrity check", + "config_path", path, + "checksum_path", checksumPath, + ) + } + + var cfg provision.AgentConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, false, fmt.Errorf("decode applied config %s: %w", path, err) + } + + source, err := provision.ResolveMachineName(&cfg) + if err != nil { + return nil, false, fmt.Errorf("resolve applied config machine name %s: %w", path, err) + } + + if source != "config" { + log.Info("resolved unbounded MachineName", "name", cfg.MachineName, "source", source) + } + + if err := cfg.BackfillNodeName(); err != nil { + return nil, false, fmt.Errorf("backfill applied config node name %s: %w", path, err) + } + + return &cfg, true, nil +} diff --git a/cmd/agent/internal/cmd/nspawn_config_test.go b/cmd/agent/internal/cmd/nspawn_config_test.go new file mode 100644 index 000000000..3532537da --- /dev/null +++ b/cmd/agent/internal/cmd/nspawn_config_test.go @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" +) + +func TestLoadAppliedConfig(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + configPath := filepath.Join(dir, "applied-config.json") + checksumPath := configPath + ".sha256" + want := provision.AgentConfig{ + MachineName: "machine-1", + NodeName: "node-1", + } + + data, err := json.Marshal(&want) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, data, 0o600)) + require.NoError(t, os.WriteFile(checksumPath, []byte(goalstates.ComputeChecksum(data)+"\n"), 0o600)) + + got, ok, err := loadAppliedConfig(testLogger(), configPath, checksumPath) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, want.MachineName, got.MachineName) + require.Equal(t, want.NodeName, got.NodeName) +} + +func TestLoadAppliedConfigMissing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + got, ok, err := loadAppliedConfig( + testLogger(), + filepath.Join(dir, "missing.json"), + filepath.Join(dir, "missing.json.sha256"), + ) + + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, got) +} + +func TestLoadAppliedConfigChecksumMismatch(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + configPath := filepath.Join(dir, "applied-config.json") + checksumPath := configPath + ".sha256" + require.NoError(t, os.WriteFile(configPath, []byte(`{"MachineName":"machine-1"}`), 0o600)) + require.NoError(t, os.WriteFile(checksumPath, []byte(goalstates.ComputeChecksum([]byte("different"))), 0o600)) + + got, ok, err := loadAppliedConfig(testLogger(), configPath, checksumPath) + require.ErrorIs(t, err, goalstates.ErrChecksumMismatch) + require.False(t, ok) + require.Nil(t, got) +} diff --git a/docs/content/reference/agent/nspawn.md b/docs/content/reference/agent/nspawn.md index dccf6a427..f4027f119 100644 --- a/docs/content/reference/agent/nspawn.md +++ b/docs/content/reference/agent/nspawn.md @@ -152,16 +152,20 @@ The agent also auto-mounts host storage and InfiniBand hardware: access. Sources are not created or required to exist during config validation. -Device discovery runs once when the machine is provisioned. Disks or HCAs -hot-plugged after the machine has started are not picked up until the machine -is re-provisioned or soft-rebooted. +Device discovery runs when the machine is provisioned and is refreshed by a +host-side systemd hook before systemd starts the nspawn machine. Device mapping +changes that occur while the host is offline are picked up on the next host +boot before the machine starts. Disks or HCAs hot-plugged after the machine has +started are not picked up until the machine is restarted, re-provisioned, or +soft-rebooted. -The configuration is written to two files on the host before the machine boots: +The configuration is written to these files on the host before the machine boots: | File | Path | |---|---| | nspawn config | `/etc/systemd/nspawn/.nspawn` | | Service override | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | +| Config regeneration unit | `/etc/systemd/system/unbounded-agent-regenerate-config@.service` | ### Customization points @@ -284,6 +288,7 @@ The container operates in the host's network namespace (`VirtualEthernet=no`): | `/var/lib/machines/` | Container rootfs directory. | | `/etc/systemd/nspawn/.nspawn` | nspawn configuration file. | | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | Systemd service override. | +| `/etc/systemd/system/unbounded-agent-regenerate-config@.service` | Host-side oneshot unit that regenerates host-side configuration before machine start. | | `/run/host-nvidia//` | (Inside container) Read-only bind-mount of host NVIDIA library directories. | ## See Also diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index c467d274d..1054e6830 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -30,6 +30,8 @@ wait-for-node Wait for the node to appear and become Ready. validate-host-nspawn-distro Verify the nspawn machine distro matches the host. validate-node-config Verify configured node and kubelet settings. + validate-device-refresh-after-host-reboot + Verify discovered devices refresh on host boot. dump-persisted-agent-config Print persisted agent config files from the VM. validate-workload Deploy test pods on the agent node. validate-kube-proxy Verify kube-proxy is Running on all nodes. @@ -144,6 +146,8 @@ DAEMON_BINARY_CURRENT = "/usr/local/bin/unbounded-agent-current" DAEMON_BINARY_LAST_GOOD = "/usr/local/bin/unbounded-agent-last-good" BPFFS_SENTINEL = "unbounded-e2e-bpffs-sentinel" +DEVICE_REFRESH_PATH = "/dev/infiniband/unbounded-e2e-zero" +DEVICE_REFRESH_TMPFILES_PATH = "/etc/tmpfiles.d/unbounded-e2e-device.conf" # --------------------------------------------------------------------------- @@ -595,6 +599,7 @@ class NodeConfig: block_external_network: bool = False additional_host_mounts: tuple[dict[str, Any], ...] = () additional_host_devices: tuple[str, ...] = () + validate_device_refresh_after_host_reboot: bool = False local_dns: bool = False path: str = "" @@ -636,6 +641,9 @@ def load_node_config( block_external_network = cfg.get("blockExternalNetwork", False) additional_host_mounts = cfg.get("additionalHostMounts", []) additional_host_devices = cfg.get("additionalHostDevices", []) + validate_device_refresh_after_host_reboot = cfg.get( + "validateDeviceRefreshAfterHostReboot", False, + ) local_dns = cfg.get("localDNS", False) if not isinstance(name, str) or not name: @@ -672,6 +680,11 @@ def load_node_config( ) if not isinstance(local_dns, bool): die(f"node config {config_path} field 'localDNS' must be a boolean") + if not isinstance(validate_device_refresh_after_host_reboot, bool): + die( + f"node config {config_path} field " + f"'validateDeviceRefreshAfterHostReboot' must be a boolean" + ) if not isinstance(additional_host_devices, list) or not all( isinstance(d, str) and d for d in additional_host_devices ): @@ -691,6 +704,7 @@ def load_node_config( block_external_network=block_external_network, additional_host_mounts=tuple(dict(m) for m in additional_host_mounts), additional_host_devices=tuple(additional_host_devices), + validate_device_refresh_after_host_reboot=validate_device_refresh_after_host_reboot, local_dns=local_dns, path=str(config_path), ) @@ -813,6 +827,10 @@ def log_active_node_config(node_config: NodeConfig) -> None: log(f" additional host devices: {', '.join(node_config.additional_host_devices)}") else: log(f" additional host devices: ") + log( + " validate device refresh after host reboot: " + f"{node_config.validate_device_refresh_after_host_reboot}" + ) def _safe_name(value: str) -> str: @@ -2887,6 +2905,125 @@ def validate_additional_host_devices_config(node_config: NodeConfig) -> None: log("Additional host devices configuration validated") +def wait_for_vm_host_reboot(previous_boot_id: str, timeout_secs: int = 300) -> None: + """Wait for SSH to return with a different host boot ID.""" + + log(f"Waiting for VM host to reboot (timeout: {timeout_secs}s)...") + deadline = time.monotonic() + timeout_secs + while time.monotonic() < deadline: + result = subprocess.run( + [ + "ssh", *SSH_OPTS, SSH_TARGET, + "cat /proc/sys/kernel/random/boot_id", + ], + capture_output=True, + text=True, + check=False, + ) + current_boot_id = result.stdout.strip() if result.returncode == 0 else "" + if current_boot_id and current_boot_id != previous_boot_id: + log(f"VM host boot ID changed: {previous_boot_id} -> {current_boot_id}") + return + time.sleep(3) + + die("timed out waiting for VM host reboot") + + +def wait_for_nspawn_machine(machine: str, timeout_secs: int = 300) -> None: + """Wait for the named nspawn machine to become active after host reboot.""" + + log(f"Waiting for nspawn machine '{machine}' to become active...") + deadline = time.monotonic() + timeout_secs + while time.monotonic() < deadline: + result = ssh_capture_quiet(f"sudo machinectl show {machine}") + if result.returncode == 0: + log(f"nspawn machine '{machine}' is active") + return + time.sleep(3) + + die(f"timed out waiting for nspawn machine '{machine}' after host reboot") + + +def validate_device_refresh_after_host_reboot() -> None: + """Verify a device appearing during host boot is usable inside nspawn.""" + + machine = active_nspawn_machine() + nspawn_config_path = f"/etc/systemd/nspawn/{machine}.nspawn" + override_path = ( + f"/etc/systemd/system/systemd-nspawn@{machine}.service.d/override.conf" + ) + + log("Validating device discovery refresh after a VM host reboot...") + ssh_cmd( + f"sudo test ! -e {DEVICE_REFRESH_PATH} && " + f"! sudo grep -Fq {DEVICE_REFRESH_PATH} {nspawn_config_path} && " + f"! sudo grep -Fq {DEVICE_REFRESH_PATH} {override_path}" + ) + + # Create an alias of /dev/zero on the next boot. Placing it under + # /dev/infiniband makes it part of normal host device discovery without + # requiring RDMA hardware in the e2e VM. + ssh_cmd(f""" +sudo tee {DEVICE_REFRESH_TMPFILES_PATH} >/dev/null <<'EOF' +d /dev/infiniband 0755 root root - +c {DEVICE_REFRESH_PATH} 0666 root root - 1:5 +EOF +""") + + previous_boot_id = ssh_capture("cat /proc/sys/kernel/random/boot_id").strip() + if not previous_boot_id: + die("VM host boot ID was empty before reboot") + + # SSH normally exits with 255 when systemd tears down the connection. + subprocess.run( + ["ssh", *SSH_OPTS, SSH_TARGET, "sudo systemctl reboot"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + wait_for_vm_host_reboot(previous_boot_id) + wait_for_nspawn_machine(machine) + wait_for_node_ready(AGENT_MACHINE_NAME) + + ssh_cmd(f"sudo test -c {DEVICE_REFRESH_PATH}") + nspawn_config = ssh_capture(f"sudo cat {nspawn_config_path}") + expected_bind = f"Bind={DEVICE_REFRESH_PATH}" + if expected_bind not in nspawn_config: + die( + f"nspawn config {nspawn_config_path} missing refreshed directive " + f"{expected_bind!r}; full config:\n{nspawn_config}" + ) + + service_override = ssh_capture(f"sudo cat {override_path}") + expected_allow = f"DeviceAllow={DEVICE_REFRESH_PATH} rwm" + if expected_allow not in service_override: + die( + f"service override {override_path} missing refreshed directive " + f"{expected_allow!r}; full override:\n{service_override}" + ) + + loaded_device_allow = ssh_capture( + f"sudo systemctl show systemd-nspawn@{machine}.service " + "--property=DeviceAllow" + ) + if DEVICE_REFRESH_PATH not in loaded_device_allow: + die( + "systemd did not load the refreshed DeviceAllow setting; " + f"loaded property: {loaded_device_allow!r}" + ) + + # The manager property verifies the service-level permission was reloaded; + # reading the device verifies that the refreshed bind is usable in nspawn. + machine_shell( + machine, + f"test -c {DEVICE_REFRESH_PATH} && " + f"dd if={DEVICE_REFRESH_PATH} of=/dev/null bs=1 count=1 status=none", + ) + + log("Device discovery refresh after VM host reboot validated") + + def _run_scenario_command(command: str, node_config: NodeConfig, env: dict[str, str]) -> None: args = [sys.executable, str(Path(__file__))] if VERBOSE: @@ -2925,6 +3062,11 @@ def _validate_node_config_scenario(node_config: NodeConfig, index: int, agent_ur ): _run_scenario_command(command, node_config, env) + if node_config.validate_device_refresh_after_host_reboot: + _run_scenario_command( + "validate-device-refresh-after-host-reboot", node_config, env, + ) + if node_config.local_dns: _run_scenario_command("validate-node-reboot-operation", node_config, env) _run_scenario_command("validate-node-config", node_config, env) @@ -4315,6 +4457,9 @@ def command(_node_config: NodeConfig) -> None: "wait-for-node-registered": _without_node_config(wait_for_node_registered), "validate-host-nspawn-distro": _without_node_config(validate_host_nspawn_distro), "validate-node-config": validate_node_config, + "validate-device-refresh-after-host-reboot": _without_node_config( + validate_device_refresh_after_host_reboot, + ), "validate-kube-proxy": _without_node_config(validate_kube_proxy), "validate-workload": _without_node_config(validate_workload), "install-machine-crd": _without_node_config(install_machine_crd), diff --git a/hack/agent/e2e-kind/node-configs/README.md b/hack/agent/e2e-kind/node-configs/README.md index 716e9afb3..a58611966 100644 --- a/hack/agent/e2e-kind/node-configs/README.md +++ b/hack/agent/e2e-kind/node-configs/README.md @@ -15,6 +15,8 @@ JSON file can be passed to `e2e.py` with `--node-config`. | `blockExternalNetwork` | boolean | Optional. When true, the e2e installs required host packages, then blocks VM egress outside local e2e networks before running bootstrap. This is intended for offline bootstrap validation. The offline artifact bundle includes kube-system images needed for node readiness plus the e2e workload image. | | `localDNS` | boolean | Enables the nspawn-local CoreDNS cache and validates service health, resolver wiring, metrics, listener addresses, and NOTRACK rules. The scenario must also provide `nodeIP` so metrics bind to the Node InternalIP. | | `additionalHostMounts` | array of objects | Optional extra host bind-mounts for the nspawn machine. Each entry has a required string `source` (clean absolute host path), optional string `target` (defaults to `source`), and optional bool `readOnly`. The e2e passes each entry as `--additional-host-mount` to `manual-bootstrap` and validates the resulting `Bind=` / `BindReadOnly=` directives in the nspawn config. | +| `additionalHostDevices` | array of strings | Optional host device paths or systemd device-group specifiers passed through `--additional-host-device`. The e2e validates generated `Bind=` and `DeviceAllow=` directives for device paths. | +| `validateDeviceRefreshAfterHostReboot` | boolean | Creates a synthetic device during the next VM host boot and verifies nspawn regeneration makes the device accessible inside the machine. | The `validate-node-configs` parent process prepares OCI refs once, then passes the local refs to each child `e2e.py` invocation with diff --git a/hack/agent/e2e-kind/node-configs/additional-host-mounts.json b/hack/agent/e2e-kind/node-configs/additional-host-mounts.json index 5390a8a59..f70e8fb33 100644 --- a/hack/agent/e2e-kind/node-configs/additional-host-mounts.json +++ b/hack/agent/e2e-kind/node-configs/additional-host-mounts.json @@ -1,5 +1,6 @@ { "name": "additional-host-mounts", + "validateDeviceRefreshAfterHostReboot": true, "nodeLabels": { "e2e.unbounded-cloud.io/config": "additional-host-mounts" }, diff --git a/pkg/agent/goalstates/constants.go b/pkg/agent/goalstates/constants.go index 80e0ffb0c..ca316af18 100644 --- a/pkg/agent/goalstates/constants.go +++ b/pkg/agent/goalstates/constants.go @@ -74,6 +74,12 @@ func BPFFSMountPath(machineName string) string { return fmt.Sprintf("%s/%s", BPFFSMountDir, machineName) } +// ConfigRegenerationUnit returns the systemd unit that regenerates host-side +// configuration before the named machine starts. +func ConfigRegenerationUnit(machineName string) string { + return fmt.Sprintf("unbounded-agent-regenerate-config@%s.service", machineName) +} + // AppliedConfigPath returns the path to the applied config file for the // given nspawn machine name, e.g. /etc/unbounded/agent/kube1-applied-config.json. func AppliedConfigPath(machineName string) string { diff --git a/pkg/agent/goalstates/resolve.go b/pkg/agent/goalstates/resolve.go index 3a8606b17..6c22a5f46 100644 --- a/pkg/agent/goalstates/resolve.go +++ b/pkg/agent/goalstates/resolve.go @@ -33,17 +33,49 @@ type MachineGoalState struct { NodeStart *NodeStart } +// ResolveNSpawnConfig probes only the host state needed to render the +// systemd-nspawn configuration for a machine. Unlike ResolveMachine, it does +// not resolve network-dependent node services such as LocalDNS. +func ResolveNSpawnConfig(cfg *config.AgentConfig, machineName string) (*RootFS, error) { + if err := config.ValidateAdditionalHostDevices(cfg.AdditionalHostDevices); err != nil { + return nil, err + } + + additionalHostMounts, err := resolveAdditionalHostMounts(cfg.AdditionalHostMounts) + if err != nil { + return nil, err + } + + nvidia, err := ResolveNvidiaHost(runtime.GOARCH) + if err != nil { + return nil, fmt.Errorf("resolve nvidia host: %w", err) + } + + return &RootFS{ + MachineDir: filepath.Join("/var/lib/machines", machineName), + NSpawnConfigFile: filepath.Join( + SystemdNSpawnDir, + machineName+".nspawn", + ), + ServiceOverrideFile: filepath.Join( + SystemdSystemDir, + fmt.Sprintf("systemd-nspawn@%s.service.d", machineName), + "override.conf", + ), + Nvidia: nvidia, + AMD: ResolveAMDHost(), + HostDevices: DiscoverHostDevices(cfg.AdditionalHostDevices), + AdditionalHostMounts: additionalHostMounts, + }, nil +} + // ResolveMachine probes the host (kernel version, hostname, GPU hardware) and // resolves the complete goal state for the named nspawn machine from an agent // config and caller-provided download overrides. func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName string, downloads *DownloadOverrides) (*MachineGoalState, error) { sandboxImage := cfg.CRI.Containerd.SandboxImage - if err := config.ValidateAdditionalHostDevices(cfg.AdditionalHostDevices); err != nil { - return nil, err - } - - additionalHostMounts, err := resolveAdditionalHostMounts(cfg.AdditionalHostMounts) + nspawnConfig, err := ResolveNSpawnConfig(cfg, machineName) if err != nil { return nil, err } @@ -58,12 +90,8 @@ func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName strin return nil, fmt.Errorf("get host hostname: %w", err) } - nvidia, err := ResolveNvidiaHost(runtime.GOARCH) - if err != nil { - return nil, fmt.Errorf("resolve nvidia host: %w", err) - } - - amd := ResolveAMDHost() + nvidia := nspawnConfig.Nvidia + amd := nspawnConfig.AMD ociImage := ResolveOCIImage(log, cfg.OCIImage, len(nvidia.GPUDevicePaths) > 0) @@ -98,16 +126,9 @@ func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName strin } rootFS := &RootFS{ - MachineDir: filepath.Join("/var/lib/machines", machineName), - NSpawnConfigFile: filepath.Join( - SystemdNSpawnDir, - machineName+".nspawn", - ), - ServiceOverrideFile: filepath.Join( - SystemdSystemDir, - fmt.Sprintf("systemd-nspawn@%s.service.d", machineName), - "override.conf", - ), + MachineDir: nspawnConfig.MachineDir, + NSpawnConfigFile: nspawnConfig.NSpawnConfigFile, + ServiceOverrideFile: nspawnConfig.ServiceOverrideFile, HostArch: runtime.GOARCH, HostKernel: kernel, Hostname: hostname, @@ -120,8 +141,8 @@ func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName strin OCIImage: ociImage, Nvidia: nvidia, AMD: amd, - HostDevices: DiscoverHostDevices(cfg.AdditionalHostDevices), - AdditionalHostMounts: additionalHostMounts, + HostDevices: nspawnConfig.HostDevices, + AdditionalHostMounts: nspawnConfig.AdditionalHostMounts, } nodeStart := &NodeStart{ diff --git a/pkg/agent/goalstates/resolve_test.go b/pkg/agent/goalstates/resolve_test.go index d5e7f4649..f208876bf 100644 --- a/pkg/agent/goalstates/resolve_test.go +++ b/pkg/agent/goalstates/resolve_test.go @@ -20,6 +20,19 @@ func discardLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } +func TestResolveNSpawnConfigDoesNotResolveLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &config.AgentConfig{ + LocalDNS: &config.AgentLocalDNSConfig{Enabled: true}, + } + + got, err := ResolveNSpawnConfig(cfg, NSpawnMachineKube1) + require.NoError(t, err) + require.Equal(t, "/var/lib/machines/kube1", got.MachineDir) + require.Equal(t, "/etc/systemd/nspawn/kube1.nspawn", got.NSpawnConfigFile) +} + func TestResolveOCIImage_ConfigImageTakesPrecedence(t *testing.T) { // Even when env vars and GPU are present, configImage wins. t.Setenv("AGENT_OCI_IMAGE", "env-image:latest") diff --git a/pkg/agent/phases/reset/nspawn.go b/pkg/agent/phases/reset/nspawn.go index 1795ef34c..df6b24b2a 100644 --- a/pkg/agent/phases/reset/nspawn.go +++ b/pkg/agent/phases/reset/nspawn.go @@ -31,11 +31,13 @@ func (t *removeNSpawnConfig) Name() string { return "remove-nspawn-config" } func (t *removeNSpawnConfig) Do(_ context.Context) error { nspawnFile := fmt.Sprintf("%s/%s.nspawn", goalstates.SystemdNSpawnDir, t.machineName) overrideDir := fmt.Sprintf("%s/systemd-nspawn@%s.service.d", goalstates.SystemdSystemDir, t.machineName) + configRegenerationUnit := fmt.Sprintf("%s/%s", goalstates.SystemdSystemDir, goalstates.ConfigRegenerationUnit(t.machineName)) - t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir) + t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir, "config_regeneration_unit", configRegenerationUnit) removeFileIfExists(t.log, nspawnFile) removeAllIfExists(t.log, overrideDir) + removeFileIfExists(t.log, configRegenerationUnit) return nil } @@ -79,7 +81,8 @@ func (t *removeBPFFSMount) Do(ctx context.Context) error { // CleanupMachine returns a composite task that removes all artifacts of an // nspawn machine: its nspawn configuration and rootfs. func CleanupMachine(log *slog.Logger, machineName string) phases.Task { - return phases.Serial(log, + return phases.Serial( + log, RemoveNSpawnConfig(log, machineName), RemoveMachine(log, machineName), RemoveBPFFSMount(log, machineName), diff --git a/pkg/agent/phases/rootfs/assets/config-regeneration.service b/pkg/agent/phases/rootfs/assets/config-regeneration.service new file mode 100644 index 000000000..871b0ee03 --- /dev/null +++ b/pkg/agent/phases/rootfs/assets/config-regeneration.service @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[Unit] +Description=Regenerate configuration for {{.MachineName}} +Wants=systemd-udev-settle.service +After=systemd-udev-settle.service + +[Service] +Type=oneshot +ExecStart={{.AgentBinaryPath}} regenerate-config {{.MachineName}} diff --git a/pkg/agent/phases/rootfs/assets/service-override.conf b/pkg/agent/phases/rootfs/assets/service-override.conf index 4ae633223..aae54c388 100644 --- a/pkg/agent/phases/rootfs/assets/service-override.conf +++ b/pkg/agent/phases/rootfs/assets/service-override.conf @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires={{.ConfigRegenerationUnit}} +After={{.ConfigRegenerationUnit}} StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/nspawn.go b/pkg/agent/phases/rootfs/nspawn.go index 06c6d06ee..dc547e191 100644 --- a/pkg/agent/phases/rootfs/nspawn.go +++ b/pkg/agent/phases/rootfs/nspawn.go @@ -21,11 +21,11 @@ import ( "github.com/Azure/unbounded/pkg/agent/phases/rootfs/oci" ) -//go:embed assets/nspawn.conf assets/service-override.conf +//go:embed assets/nspawn.conf assets/config-regeneration.service assets/service-override.conf var nspawnAssets embed.FS var nspawnTemplates = template.Must( - template.New("nspawn").ParseFS(nspawnAssets, "assets/nspawn.conf", "assets/service-override.conf"), + template.New("nspawn").ParseFS(nspawnAssets, "assets/nspawn.conf", "assets/config-regeneration.service", "assets/service-override.conf"), ) type ensureNSpawnWorkspace struct { @@ -66,12 +66,29 @@ func EnsureNSpawnWorkspace(log *slog.Logger, goalState *goalstates.RootFS) phase func (e *ensureNSpawnWorkspace) Name() string { return "ensure-nspawn-workspace" } +type ensureNSpawnConfig struct { + log *slog.Logger + goalState *goalstates.RootFS +} + +// EnsureNSpawnConfig returns a task that only writes the host-side +// systemd-nspawn configuration files for a machine. +func EnsureNSpawnConfig(log *slog.Logger, goalState *goalstates.RootFS) phases.Task { + return &ensureNSpawnConfig{log: log, goalState: goalState} +} + +func (e *ensureNSpawnConfig) Name() string { return "ensure-nspawn-config" } + +func (e *ensureNSpawnConfig) Do(_ context.Context) error { + return writeNSpawnConfigs(e.log, e.goalState) +} + func (e *ensureNSpawnWorkspace) Do(ctx context.Context) error { if err := e.bootstrapWorkspace(ctx); err != nil { return fmt.Errorf("bootstrap machine directory %s: %w", e.goalState.MachineDir, err) } - if err := e.writeNSpawnConfigs(); err != nil { + if err := writeNSpawnConfigs(e.log, e.goalState); err != nil { return err } @@ -103,23 +120,25 @@ type nspawnTemplateData struct { NvidiaBinDir string AMDGPUDevicePaths []string AMDSysFSPaths []string + ConfigRegenerationUnit string + AgentBinaryPath string } // TODO: migrate AdditionalHostMounts, HostDevicePaths/HostDeviceGroupSpecifiers, // and AMDGPUDevicePaths to structured bind/device-allow targets in a follow-up PR. -// writeNSpawnConfigs renders the nspawn and service-override templates with -// device and GPU data (when present) and writes them to their configured paths. -func (e *ensureNSpawnWorkspace) writeNSpawnConfigs() error { +// writeNSpawnConfigs renders the nspawn-related templates with device and GPU +// data (when present) and writes them to their configured paths. +func writeNSpawnConfigs(log *slog.Logger, goalState *goalstates.RootFS) error { // MachineName is the basename of MachineDir (e.g. "kube1" from // "/var/lib/machines/kube1"); nspawn always names the machine after that // directory. - machineName := filepath.Base(e.goalState.MachineDir) - hostDevicePaths := e.goalState.HostDevices.Paths() - hostDeviceGroupSpecifiers := e.goalState.HostDevices.DeviceGroupSpecifiers() - amdGPUDevicePaths := pathsExcluding(e.goalState.AMD.GPUDevicePaths, e.goalState.Nvidia.GPUDevicePaths) + machineName := filepath.Base(goalState.MachineDir) + hostDevicePaths := goalState.HostDevices.Paths() + hostDeviceGroupSpecifiers := goalState.HostDevices.DeviceGroupSpecifiers() + amdGPUDevicePaths := pathsExcluding(goalState.AMD.GPUDevicePaths, goalState.Nvidia.GPUDevicePaths) - archiveDir := filepath.Join(e.goalState.MachineDir, strings.TrimPrefix(goalstates.ContainerImageArchiveDir, "/")) + archiveDir := filepath.Join(goalState.MachineDir, strings.TrimPrefix(goalstates.ContainerImageArchiveDir, "/")) if err := os.MkdirAll(archiveDir, 0o755); err != nil { return fmt.Errorf("create container image archive mount point: %w", err) } @@ -131,37 +150,39 @@ func (e *ensureNSpawnWorkspace) writeNSpawnConfigs() error { ContainerImageArchiveHostDir: goalstates.ContainerImageArchiveHostDir, HostDevicePaths: hostDevicePaths, HostDeviceGroupSpecifiers: hostDeviceGroupSpecifiers, - AdditionalHostMounts: e.goalState.AdditionalHostMounts, - NvidiaDeviceTargets: nvidiaNSpawnDeviceTargets(e.goalState.Nvidia.GPUDevicePaths), - NvidiaLibDirMounts: e.goalState.Nvidia.LibDirMounts, - NvidiaI386LibDirMounts: e.goalState.Nvidia.I386LibDirMounts, - NvidiaBinDir: nvidiaHostBinDir(e.goalState.Nvidia), + AdditionalHostMounts: goalState.AdditionalHostMounts, + NvidiaDeviceTargets: nvidiaNSpawnDeviceTargets(goalState.Nvidia.GPUDevicePaths), + NvidiaLibDirMounts: goalState.Nvidia.LibDirMounts, + NvidiaI386LibDirMounts: goalState.Nvidia.I386LibDirMounts, + NvidiaBinDir: nvidiaHostBinDir(goalState.Nvidia), AMDGPUDevicePaths: amdGPUDevicePaths, - AMDSysFSPaths: e.goalState.AMD.SysFSPaths, + AMDSysFSPaths: goalState.AMD.SysFSPaths, + ConfigRegenerationUnit: goalstates.ConfigRegenerationUnit(machineName), + AgentBinaryPath: goalstates.DaemonBinaryPath, } if len(hostDevicePaths) > 0 { - e.log.Info("host devices detected, configuring nspawn bind-mounts", + log.Info("host devices detected, configuring nspawn bind-mounts", "total", len(hostDevicePaths), - "kvm", len(e.goalState.HostDevices.KVM), - "network", len(e.goalState.HostDevices.Network), - "block", len(e.goalState.HostDevices.Block), - "infiniband", len(e.goalState.HostDevices.Infiniband), - "additional", len(e.goalState.HostDevices.Additional)) + "kvm", len(goalState.HostDevices.KVM), + "network", len(goalState.HostDevices.Network), + "block", len(goalState.HostDevices.Block), + "infiniband", len(goalState.HostDevices.Infiniband), + "additional", len(goalState.HostDevices.Additional)) } - if len(e.goalState.AdditionalHostMounts) > 0 { - e.log.Info("additional host mounts configured", - "count", len(e.goalState.AdditionalHostMounts)) + if len(goalState.AdditionalHostMounts) > 0 { + log.Info("additional host mounts configured", + "count", len(goalState.AdditionalHostMounts)) } - if len(e.goalState.Nvidia.GPUDevicePaths) > 0 { - e.log.Info("GPU devices detected, configuring nspawn bind-mounts", - "count", len(e.goalState.Nvidia.GPUDevicePaths)) + if len(goalState.Nvidia.GPUDevicePaths) > 0 { + log.Info("GPU devices detected, configuring nspawn bind-mounts", + "count", len(goalState.Nvidia.GPUDevicePaths)) } if len(amdGPUDevicePaths) > 0 { - e.log.Info("AMD GPU devices detected, configuring nspawn bind-mounts", + log.Info("AMD GPU devices detected, configuring nspawn bind-mounts", "count", len(amdGPUDevicePaths)) } @@ -171,8 +192,8 @@ func (e *ensureNSpawnWorkspace) writeNSpawnConfigs() error { return fmt.Errorf("render nspawn config template: %w", err) } - if err := utilio.WriteFile(e.goalState.NSpawnConfigFile, nspawnBuf.Bytes(), 0o644); err != nil { - return fmt.Errorf("write nspawn config %s: %w", e.goalState.NSpawnConfigFile, err) + if err := utilio.WriteFile(goalState.NSpawnConfigFile, nspawnBuf.Bytes(), 0o644); err != nil { + return fmt.Errorf("write nspawn config %s: %w", goalState.NSpawnConfigFile, err) } // Render and write the systemd service override drop-in. @@ -181,8 +202,19 @@ func (e *ensureNSpawnWorkspace) writeNSpawnConfigs() error { return fmt.Errorf("render service override template: %w", err) } - if err := utilio.WriteFile(e.goalState.ServiceOverrideFile, overrideBuf.Bytes(), 0o644); err != nil { - return fmt.Errorf("write service override %s: %w", e.goalState.ServiceOverrideFile, err) + if err := utilio.WriteFile(goalState.ServiceOverrideFile, overrideBuf.Bytes(), 0o644); err != nil { + return fmt.Errorf("write service override %s: %w", goalState.ServiceOverrideFile, err) + } + + unitFile := filepath.Join(goalstates.SystemdSystemDir, templateData.ConfigRegenerationUnit) + + unitBuf := &bytes.Buffer{} + if err := nspawnTemplates.ExecuteTemplate(unitBuf, "config-regeneration.service", templateData); err != nil { + return fmt.Errorf("render config regeneration unit template: %w", err) + } + + if err := utilio.WriteFile(unitFile, unitBuf.Bytes(), 0o644); err != nil { + return fmt.Errorf("write config regeneration unit %s: %w", unitFile, err) } return nil diff --git a/pkg/agent/phases/rootfs/nspawn_render_test.go b/pkg/agent/phases/rootfs/nspawn_render_test.go index 8dddbda5f..b10fd0788 100644 --- a/pkg/agent/phases/rootfs/nspawn_render_test.go +++ b/pkg/agent/phases/rootfs/nspawn_render_test.go @@ -21,29 +21,19 @@ import ( func TestServiceOverride_RenderedSnapshot(t *testing.T) { t.Parallel() - requireRenderedSnapshot(t, "service-override-kube1.conf.golden", "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - }) + requireRenderedSnapshot(t, "service-override-kube1.conf.golden", "service-override.conf", defaultNSpawnTemplateData("kube1")) } func TestServiceOverride_MachineNameSnapshot(t *testing.T) { t.Parallel() - requireRenderedSnapshot(t, "service-override-kube2.conf.golden", "service-override.conf", nspawnTemplateData{ - MachineName: "kube2", - BPFFSMountPath: goalstates.BPFFSMountPath("kube2"), - }) + requireRenderedSnapshot(t, "service-override-kube2.conf.golden", "service-override.conf", defaultNSpawnTemplateData("kube2")) } func TestNSpawnConfig_RenderedSnapshot(t *testing.T) { t.Parallel() - requireRenderedSnapshot(t, "nspawn.conf.golden", "nspawn.conf", nspawnTemplateData{ - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - ContainerImageArchiveDir: goalstates.ContainerImageArchiveDir, - ContainerImageArchiveHostDir: goalstates.ContainerImageArchiveHostDir, - }) + requireRenderedSnapshot(t, "nspawn.conf.golden", "nspawn.conf", defaultNSpawnTemplateData("kube1")) } func TestNSpawnRenderedScenarios(t *testing.T) { @@ -118,11 +108,10 @@ func TestServiceOverride_HostDevicesDeviceAllow(t *testing.T) { t.Parallel() var buf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - HostDevicePaths: []string{"/dev/kvm"}, - })) + + data := defaultNSpawnTemplateData("kube1") + data.HostDevicePaths = []string{"/dev/kvm"} + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", data)) out := buf.String() @@ -164,19 +153,13 @@ func TestServiceOverride_MultipleHostDevices(t *testing.T) { devices := []string{"/dev/kvm", "/dev/net/tun", "/dev/vhost-net", "/dev/sda", "/dev/infiniband/uverbs0"} var nspawnBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", nspawnTemplateData{ - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - ContainerImageArchiveDir: goalstates.ContainerImageArchiveDir, - ContainerImageArchiveHostDir: goalstates.ContainerImageArchiveHostDir, - HostDevicePaths: devices, - })) + + data := defaultNSpawnTemplateData("kube1") + data.HostDevicePaths = devices + require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", data)) var overrideBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - HostDevicePaths: devices, - })) + require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", data)) // Every host device must get both a bind mount in the .nspawn config and a // matching cgroup DeviceAllow in the service drop-in; otherwise the node is @@ -193,20 +176,14 @@ func TestServiceOverride_AMDGPUDevices(t *testing.T) { devices := []string{"/dev/dri/card0", "/dev/dri/renderD128", "/dev/kfd"} var nspawnBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", nspawnTemplateData{ - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - ContainerImageArchiveDir: goalstates.ContainerImageArchiveDir, - ContainerImageArchiveHostDir: goalstates.ContainerImageArchiveHostDir, - AMDGPUDevicePaths: devices, - AMDSysFSPaths: []string{"/sys/module/amdgpu", "/sys/class/kfd"}, - })) + + data := defaultNSpawnTemplateData("kube1") + data.AMDGPUDevicePaths = devices + data.AMDSysFSPaths = []string{"/sys/module/amdgpu", "/sys/class/kfd"} + require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", data)) var overrideBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - AMDGPUDevicePaths: devices, - })) + require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", data)) for _, dev := range devices { require.Contains(t, nspawnBuf.String(), "Bind="+dev) @@ -308,10 +285,7 @@ func TestServiceOverride_BaseDeviceAllow(t *testing.T) { t.Parallel() var buf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - })) + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", defaultNSpawnTemplateData("kube1"))) out := buf.String() @@ -324,15 +298,46 @@ func TestServiceOverride_BaseDeviceAllow(t *testing.T) { require.Equal(t, 2, strings.Count(out, "DeviceAllow=")) } -func nspawnRenderScenarioData() nspawnTemplateData { +func TestServiceOverride_ConfigRegenerationDependency(t *testing.T) { + t.Parallel() + + out := requireRenderedSnapshot(t, "service-override-kube1.conf.golden", "service-override.conf", defaultNSpawnTemplateData("kube1")) + + require.Contains(t, out, "Requires=unbounded-agent-regenerate-config@kube1.service") + require.Contains(t, out, "After=unbounded-agent-regenerate-config@kube1.service") + require.Less(t, strings.Index(out, "[Unit]"), strings.Index(out, "Requires=unbounded-agent-regenerate-config@kube1.service")) + require.Less(t, strings.Index(out, "After=unbounded-agent-regenerate-config@kube1.service"), strings.Index(out, "[Service]")) +} + +func TestConfigRegenerationUnit(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "config-regeneration.service", defaultNSpawnTemplateData("kube1"))) + + out := buf.String() + require.Contains(t, out, "Description=Regenerate configuration for kube1") + require.Contains(t, out, "Wants=systemd-udev-settle.service") + require.Contains(t, out, "After=systemd-udev-settle.service") + require.Contains(t, out, "Type=oneshot") + require.Contains(t, out, "ExecStart=/usr/local/bin/unbounded-agent regenerate-config kube1") +} + +func defaultNSpawnTemplateData(machineName string) nspawnTemplateData { return nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), + MachineName: machineName, + BPFFSMountPath: goalstates.BPFFSMountPath(machineName), ContainerImageArchiveDir: goalstates.ContainerImageArchiveDir, ContainerImageArchiveHostDir: goalstates.ContainerImageArchiveHostDir, + ConfigRegenerationUnit: goalstates.ConfigRegenerationUnit(machineName), + AgentBinaryPath: goalstates.DaemonBinaryPath, } } +func nspawnRenderScenarioData() nspawnTemplateData { + return defaultNSpawnTemplateData("kube1") +} + func requireRenderedGolden(t *testing.T, name string, data nspawnTemplateData) { t.Helper() diff --git a/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden b/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden index 907d0e5f7..04653b353 100644 --- a/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires=unbounded-agent-regenerate-config@kube1.service +After=unbounded-agent-regenerate-config@kube1.service StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden b/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden index 994310799..281789b65 100644 --- a/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires=unbounded-agent-regenerate-config@kube1.service +After=unbounded-agent-regenerate-config@kube1.service StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden b/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden index 317671f89..e46f2679c 100644 --- a/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires=unbounded-agent-regenerate-config@kube1.service +After=unbounded-agent-regenerate-config@kube1.service StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden index 907d0e5f7..04653b353 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires=unbounded-agent-regenerate-config@kube1.service +After=unbounded-agent-regenerate-config@kube1.service StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden index 6489a76b9..905cbafc3 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires=unbounded-agent-regenerate-config@kube2.service +After=unbounded-agent-regenerate-config@kube2.service StartLimitIntervalSec=0 [Service]