From 1873c882903584522113400383ccc534ef9c33ed Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:05:35 +0000 Subject: [PATCH 1/4] Initial plan From c0eb92a1b4d60edcb611402da0d0f41a9c2428af Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:13:30 +0000 Subject: [PATCH 2/4] Refresh nspawn config before machine start --- cmd/agent/internal/cmd/cmd.go | 1 + cmd/agent/internal/cmd/nspawn_config.go | 111 ++++++++++++++++++ docs/content/reference/agent/nspawn.md | 13 +- pkg/agent/goalstates/constants.go | 6 + pkg/agent/phases/reset/nspawn.go | 4 +- .../assets/nspawn-config-refresh.service | 9 ++ .../rootfs/assets/service-override.conf | 2 + pkg/agent/phases/rootfs/nspawn.go | 83 +++++++++---- pkg/agent/phases/rootfs/nspawn_render_test.go | 88 +++++++------- .../service-override-kube1.conf.golden | 2 + .../service-override-kube2.conf.golden | 2 + 11 files changed, 250 insertions(+), 71 deletions(-) create mode 100644 cmd/agent/internal/cmd/nspawn_config.go create mode 100644 pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index c3cd57e34..7acc00046 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -30,6 +30,7 @@ func Run() { newCmdDaemon(cmdCtx), newCmdReset(cmdCtx), newCmdVersion(), + newCmdRegenerateNSpawnConfig(cmdCtx), newCmdRecordAgentUpgradeFailureSignal(), ) diff --git a/cmd/agent/internal/cmd/nspawn_config.go b/cmd/agent/internal/cmd/nspawn_config.go new file mode 100644 index 000000000..7329189d3 --- /dev/null +++ b/cmd/agent/internal/cmd/nspawn_config.go @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + + "github.com/spf13/cobra" + + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/phases" + "github.com/Azure/unbounded/pkg/agent/phases/rootfs" +) + +func newCmdRegenerateNSpawnConfig(cmdCtx *CommandContext) *cobra.Command { + cmd := &cobra.Command{ + Use: "regenerate-nspawn-config MACHINE_NAME", + Short: "Regenerate host-side nspawn configuration for a machine", + Hidden: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) + defer cancel() + + cmdCtx.Setup() + + return regenerateNSpawnConfig(ctx, cmdCtx.Logger, args[0]) + }, + } + + return cmd +} + +func regenerateNSpawnConfig(ctx context.Context, log *slog.Logger, machineName string) error { + cfg, ok, err := loadAppliedConfigForMachine(log, machineName) + if err != nil { + return err + } + + if !ok { + log.Info("applied config not found, skipping nspawn config regeneration", "machine", machineName) + return nil + } + + gs, err := goalstates.ResolveMachine(log, cfg, machineName, nil) + if err != nil { + return fmt.Errorf("resolve machine goal state: %w", err) + } + + if err := phases.ExecuteTask(ctx, log, rootfs.EnsureNSpawnConfig(log, gs.RootFS)); err != nil { + return fmt.Errorf("regenerate nspawn config for %s: %w", machineName, err) + } + + return nil +} + +func loadAppliedConfigForMachine(log *slog.Logger, machineName string) (*provision.AgentConfig, bool, error) { + if machineName != goalstates.NSpawnMachineKube1 && machineName != goalstates.NSpawnMachineKube2 { + return nil, false, fmt.Errorf("unsupported nspawn machine %q", machineName) + } + + path := goalstates.AppliedConfigPath(machineName) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + + if err != nil { + return nil, false, fmt.Errorf("read applied config %s: %w", path, err) + } + + checksumPath := goalstates.AppliedConfigChecksumPath(machineName) + if err := goalstates.VerifyChecksum(data, checksumPath); err != nil { + return nil, false, fmt.Errorf("verify applied config checksum for %s: %w", machineName, err) + } + + if _, statErr := os.Stat(checksumPath); errors.Is(statErr, os.ErrNotExist) { + log.Warn("no checksum sidecar found, skipping integrity check", + "config_path", path, + "checksum_path", checksumPath, + ) + } + + var cfg provision.AgentConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, false, fmt.Errorf("decode applied config %s: %w", path, err) + } + + source, err := provision.ResolveMachineName(&cfg) + if err != nil { + return nil, false, fmt.Errorf("resolve applied config machine name %s: %w", path, err) + } + + if source != "config" { + log.Info("resolved unbounded MachineName", "name", cfg.MachineName, "source", source) + } + + if err := cfg.BackfillNodeName(); err != nil { + return nil, false, fmt.Errorf("backfill applied config node name %s: %w", path, err) + } + + return &cfg, true, nil +} diff --git a/docs/content/reference/agent/nspawn.md b/docs/content/reference/agent/nspawn.md index 573f4d80b..f8f0c77fc 100644 --- a/docs/content/reference/agent/nspawn.md +++ b/docs/content/reference/agent/nspawn.md @@ -129,16 +129,20 @@ The agent also auto-mounts host storage and InfiniBand hardware: list non-standard host device nodes under `/dev` (for example `/dev/uinput`) that should be exposed to workloads inside the machine. -Device discovery runs once when the machine is provisioned. Disks or HCAs -hot-plugged after the machine has started are not picked up until the machine -is re-provisioned or soft-rebooted. +Device discovery runs when the machine is provisioned and is refreshed by a +host-side systemd hook before systemd starts the nspawn machine. Device mapping +changes that occur while the host is offline are picked up on the next host +boot before the machine starts. Disks or HCAs hot-plugged after the machine has +started are not picked up until the machine is restarted, re-provisioned, or +soft-rebooted. -The configuration is written to two files on the host before the machine boots: +The configuration is written to these files on the host before the machine boots: | File | Path | |---|---| | nspawn config | `/etc/systemd/nspawn/.nspawn` | | Service override | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | +| Config refresh unit | `/etc/systemd/system/unbounded-agent-nspawn-config@.service` | ### Customization points @@ -257,6 +261,7 @@ The container operates in the host's network namespace (`VirtualEthernet=no`): | `/var/lib/machines/` | Container rootfs directory. | | `/etc/systemd/nspawn/.nspawn` | nspawn configuration file. | | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | Systemd service override. | +| `/etc/systemd/system/unbounded-agent-nspawn-config@.service` | Host-side oneshot unit that refreshes nspawn configuration before machine start. | | `/run/host-nvidia//` | (Inside container) Read-only bind-mount of host NVIDIA library directories. | ## See Also diff --git a/pkg/agent/goalstates/constants.go b/pkg/agent/goalstates/constants.go index 9d1977d95..1931d17a7 100644 --- a/pkg/agent/goalstates/constants.go +++ b/pkg/agent/goalstates/constants.go @@ -74,6 +74,12 @@ func BPFFSMountPath(machineName string) string { return fmt.Sprintf("%s/%s", BPFFSMountDir, machineName) } +// NSpawnConfigRefreshUnit returns the systemd unit that refreshes host-side +// nspawn configuration before the named machine starts. +func NSpawnConfigRefreshUnit(machineName string) string { + return fmt.Sprintf("unbounded-agent-nspawn-config@%s.service", machineName) +} + // AppliedConfigPath returns the path to the applied config file for the // given nspawn machine name, e.g. /etc/unbounded/agent/kube1-applied-config.json. func AppliedConfigPath(machineName string) string { diff --git a/pkg/agent/phases/reset/nspawn.go b/pkg/agent/phases/reset/nspawn.go index 1795ef34c..3c18476ab 100644 --- a/pkg/agent/phases/reset/nspawn.go +++ b/pkg/agent/phases/reset/nspawn.go @@ -31,11 +31,13 @@ func (t *removeNSpawnConfig) Name() string { return "remove-nspawn-config" } func (t *removeNSpawnConfig) Do(_ context.Context) error { nspawnFile := fmt.Sprintf("%s/%s.nspawn", goalstates.SystemdNSpawnDir, t.machineName) overrideDir := fmt.Sprintf("%s/systemd-nspawn@%s.service.d", goalstates.SystemdSystemDir, t.machineName) + configRefreshUnit := fmt.Sprintf("%s/%s", goalstates.SystemdSystemDir, goalstates.NSpawnConfigRefreshUnit(t.machineName)) - t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir) + t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir, "config_refresh_unit", configRefreshUnit) removeFileIfExists(t.log, nspawnFile) removeAllIfExists(t.log, overrideDir) + removeFileIfExists(t.log, configRefreshUnit) return nil } diff --git a/pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service b/pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service new file mode 100644 index 000000000..0e59f5c67 --- /dev/null +++ b/pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[Unit] +Description=Regenerate nspawn configuration for {{.MachineName}} + +[Service] +Type=oneshot +ExecStart={{.AgentBinaryPath}} regenerate-nspawn-config {{.MachineName}} diff --git a/pkg/agent/phases/rootfs/assets/service-override.conf b/pkg/agent/phases/rootfs/assets/service-override.conf index bf4079a46..65f4e7b46 100644 --- a/pkg/agent/phases/rootfs/assets/service-override.conf +++ b/pkg/agent/phases/rootfs/assets/service-override.conf @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires={{.ConfigRefreshUnit}} +After={{.ConfigRefreshUnit}} StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/nspawn.go b/pkg/agent/phases/rootfs/nspawn.go index af004993c..eb5fb7bbc 100644 --- a/pkg/agent/phases/rootfs/nspawn.go +++ b/pkg/agent/phases/rootfs/nspawn.go @@ -18,11 +18,11 @@ import ( "github.com/Azure/unbounded/pkg/agent/phases/rootfs/oci" ) -//go:embed assets/nspawn.conf assets/service-override.conf +//go:embed assets/nspawn.conf assets/nspawn-config-refresh.service assets/service-override.conf var nspawnAssets embed.FS var nspawnTemplates = template.Must( - template.New("nspawn").ParseFS(nspawnAssets, "assets/nspawn.conf", "assets/service-override.conf"), + template.New("nspawn").ParseFS(nspawnAssets, "assets/nspawn.conf", "assets/nspawn-config-refresh.service", "assets/service-override.conf"), ) type ensureNSpawnWorkspace struct { @@ -40,12 +40,29 @@ func EnsureNSpawnWorkspace(log *slog.Logger, goalState *goalstates.RootFS) phase func (e *ensureNSpawnWorkspace) Name() string { return "ensure-nspawn-workspace" } +type ensureNSpawnConfig struct { + log *slog.Logger + goalState *goalstates.RootFS +} + +// EnsureNSpawnConfig returns a task that only writes the host-side +// systemd-nspawn configuration files for a machine. +func EnsureNSpawnConfig(log *slog.Logger, goalState *goalstates.RootFS) phases.Task { + return &ensureNSpawnConfig{log: log, goalState: goalState} +} + +func (e *ensureNSpawnConfig) Name() string { return "ensure-nspawn-config" } + +func (e *ensureNSpawnConfig) Do(_ context.Context) error { + return writeNSpawnConfigs(e.log, e.goalState) +} + func (e *ensureNSpawnWorkspace) Do(ctx context.Context) error { if err := e.bootstrapWorkspace(ctx); err != nil { return fmt.Errorf("bootstrap machine directory %s: %w", e.goalState.MachineDir, err) } - if err := e.writeNSpawnConfigs(); err != nil { + if err := writeNSpawnConfigs(e.log, e.goalState); err != nil { return err } @@ -70,44 +87,48 @@ type nspawnTemplateData struct { NvidiaLibDirMounts []goalstates.NvidiaLibDirMount AMDGPUDevicePaths []string AMDSysFSPaths []string + ConfigRefreshUnit string + AgentBinaryPath string } -// writeNSpawnConfigs renders the nspawn and service-override templates with -// device and GPU data (when present) and writes them to their configured paths. -func (e *ensureNSpawnWorkspace) writeNSpawnConfigs() error { +// writeNSpawnConfigs renders the nspawn-related templates with device and GPU +// data (when present) and writes them to their configured paths. +func writeNSpawnConfigs(log *slog.Logger, goalState *goalstates.RootFS) error { // MachineName is the basename of MachineDir (e.g. "kube1" from // "/var/lib/machines/kube1"); nspawn always names the machine after that // directory. - machineName := filepath.Base(e.goalState.MachineDir) - hostDevicePaths := e.goalState.HostDevices.Paths() - amdGPUDevicePaths := pathsExcluding(e.goalState.AMD.GPUDevicePaths, e.goalState.Nvidia.GPUDevicePaths) + machineName := filepath.Base(goalState.MachineDir) + hostDevicePaths := goalState.HostDevices.Paths() + amdGPUDevicePaths := pathsExcluding(goalState.AMD.GPUDevicePaths, goalState.Nvidia.GPUDevicePaths) templateData := nspawnTemplateData{ MachineName: machineName, BPFFSMountPath: goalstates.BPFFSMountPath(machineName), HostDevicePaths: hostDevicePaths, - NvidiaGPUDevicePaths: e.goalState.Nvidia.GPUDevicePaths, - NvidiaLibDirMounts: e.goalState.Nvidia.LibDirMounts, + NvidiaGPUDevicePaths: goalState.Nvidia.GPUDevicePaths, + NvidiaLibDirMounts: goalState.Nvidia.LibDirMounts, AMDGPUDevicePaths: amdGPUDevicePaths, - AMDSysFSPaths: e.goalState.AMD.SysFSPaths, + AMDSysFSPaths: goalState.AMD.SysFSPaths, + ConfigRefreshUnit: goalstates.NSpawnConfigRefreshUnit(machineName), + AgentBinaryPath: goalstates.DaemonBinaryPath, } if len(hostDevicePaths) > 0 { - e.log.Info("host devices detected, configuring nspawn bind-mounts", + log.Info("host devices detected, configuring nspawn bind-mounts", "total", len(hostDevicePaths), - "kvm", len(e.goalState.HostDevices.KVM), - "network", len(e.goalState.HostDevices.Network), - "block", len(e.goalState.HostDevices.Block), - "infiniband", len(e.goalState.HostDevices.Infiniband), - "additional", len(e.goalState.HostDevices.Additional)) + "kvm", len(goalState.HostDevices.KVM), + "network", len(goalState.HostDevices.Network), + "block", len(goalState.HostDevices.Block), + "infiniband", len(goalState.HostDevices.Infiniband), + "additional", len(goalState.HostDevices.Additional)) } - if len(e.goalState.Nvidia.GPUDevicePaths) > 0 { - e.log.Info("GPU devices detected, configuring nspawn bind-mounts", - "count", len(e.goalState.Nvidia.GPUDevicePaths)) + if len(goalState.Nvidia.GPUDevicePaths) > 0 { + log.Info("GPU devices detected, configuring nspawn bind-mounts", + "count", len(goalState.Nvidia.GPUDevicePaths)) } if len(amdGPUDevicePaths) > 0 { - e.log.Info("AMD GPU devices detected, configuring nspawn bind-mounts", + log.Info("AMD GPU devices detected, configuring nspawn bind-mounts", "count", len(amdGPUDevicePaths)) } @@ -117,8 +138,8 @@ func (e *ensureNSpawnWorkspace) writeNSpawnConfigs() error { return fmt.Errorf("render nspawn config template: %w", err) } - if err := utilio.WriteFile(e.goalState.NSpawnConfigFile, nspawnBuf.Bytes(), 0o644); err != nil { - return fmt.Errorf("write nspawn config %s: %w", e.goalState.NSpawnConfigFile, err) + if err := utilio.WriteFile(goalState.NSpawnConfigFile, nspawnBuf.Bytes(), 0o644); err != nil { + return fmt.Errorf("write nspawn config %s: %w", goalState.NSpawnConfigFile, err) } // Render and write the systemd service override drop-in. @@ -127,8 +148,18 @@ func (e *ensureNSpawnWorkspace) writeNSpawnConfigs() error { return fmt.Errorf("render service override template: %w", err) } - if err := utilio.WriteFile(e.goalState.ServiceOverrideFile, overrideBuf.Bytes(), 0o644); err != nil { - return fmt.Errorf("write service override %s: %w", e.goalState.ServiceOverrideFile, err) + if err := utilio.WriteFile(goalState.ServiceOverrideFile, overrideBuf.Bytes(), 0o644); err != nil { + return fmt.Errorf("write service override %s: %w", goalState.ServiceOverrideFile, err) + } + + unitFile := filepath.Join(goalstates.SystemdSystemDir, templateData.ConfigRefreshUnit) + unitBuf := &bytes.Buffer{} + if err := nspawnTemplates.ExecuteTemplate(unitBuf, "nspawn-config-refresh.service", templateData); err != nil { + return fmt.Errorf("render nspawn config refresh unit template: %w", err) + } + + if err := utilio.WriteFile(unitFile, unitBuf.Bytes(), 0o644); err != nil { + return fmt.Errorf("write nspawn config refresh unit %s: %w", unitFile, err) } return nil diff --git a/pkg/agent/phases/rootfs/nspawn_render_test.go b/pkg/agent/phases/rootfs/nspawn_render_test.go index 07cfc49a7..72485c3dd 100644 --- a/pkg/agent/phases/rootfs/nspawn_render_test.go +++ b/pkg/agent/phases/rootfs/nspawn_render_test.go @@ -17,38 +17,28 @@ import ( func TestServiceOverride_RenderedSnapshot(t *testing.T) { t.Parallel() - requireRenderedSnapshot(t, "service-override-kube1.conf.golden", "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - }) + requireRenderedSnapshot(t, "service-override-kube1.conf.golden", "service-override.conf", defaultNSpawnTemplateData("kube1")) } func TestServiceOverride_MachineNameSnapshot(t *testing.T) { t.Parallel() - requireRenderedSnapshot(t, "service-override-kube2.conf.golden", "service-override.conf", nspawnTemplateData{ - MachineName: "kube2", - BPFFSMountPath: goalstates.BPFFSMountPath("kube2"), - }) + requireRenderedSnapshot(t, "service-override-kube2.conf.golden", "service-override.conf", defaultNSpawnTemplateData("kube2")) } func TestNSpawnConfig_RenderedSnapshot(t *testing.T) { t.Parallel() - requireRenderedSnapshot(t, "nspawn.conf.golden", "nspawn.conf", nspawnTemplateData{ - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - }) + requireRenderedSnapshot(t, "nspawn.conf.golden", "nspawn.conf", defaultNSpawnTemplateData("kube1")) } func TestServiceOverride_HostDevicesDeviceAllow(t *testing.T) { t.Parallel() var buf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - HostDevicePaths: []string{"/dev/kvm"}, - })) + data := defaultNSpawnTemplateData("kube1") + data.HostDevicePaths = []string{"/dev/kvm"} + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", data)) out := buf.String() @@ -67,17 +57,12 @@ func TestServiceOverride_MultipleHostDevices(t *testing.T) { devices := []string{"/dev/kvm", "/dev/net/tun", "/dev/vhost-net", "/dev/sda", "/dev/infiniband/uverbs0"} var nspawnBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", nspawnTemplateData{ - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - HostDevicePaths: devices, - })) + data := defaultNSpawnTemplateData("kube1") + data.HostDevicePaths = devices + require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", data)) var overrideBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - HostDevicePaths: devices, - })) + require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", data)) // Every host device must get both a bind mount in the .nspawn config and a // matching cgroup DeviceAllow in the service drop-in; otherwise the node is @@ -94,18 +79,13 @@ func TestServiceOverride_AMDGPUDevices(t *testing.T) { devices := []string{"/dev/dri/card0", "/dev/dri/renderD128", "/dev/kfd"} var nspawnBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", nspawnTemplateData{ - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - AMDGPUDevicePaths: devices, - AMDSysFSPaths: []string{"/sys/module/amdgpu", "/sys/class/kfd"}, - })) + data := defaultNSpawnTemplateData("kube1") + data.AMDGPUDevicePaths = devices + data.AMDSysFSPaths = []string{"/sys/module/amdgpu", "/sys/class/kfd"} + require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", data)) var overrideBuf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - AMDGPUDevicePaths: devices, - })) + require.NoError(t, nspawnTemplates.ExecuteTemplate(&overrideBuf, "service-override.conf", data)) for _, dev := range devices { require.Contains(t, nspawnBuf.String(), "Bind="+dev) @@ -133,17 +113,45 @@ func TestServiceOverride_NoHostDevicesNoDeviceAllow(t *testing.T) { t.Parallel() var buf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", nspawnTemplateData{ - MachineName: "kube1", - BPFFSMountPath: goalstates.BPFFSMountPath("kube1"), - // No HostDevicePaths and no GPU devices. - })) + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", defaultNSpawnTemplateData("kube1"))) // With no devices the drop-in must not contain any DeviceAllow lines, // which is what keeps the existing golden snapshots unchanged. require.NotContains(t, buf.String(), "DeviceAllow=") } +func TestServiceOverride_ConfigRefreshDependency(t *testing.T) { + t.Parallel() + + out := requireRenderedSnapshot(t, "service-override-kube1.conf.golden", "service-override.conf", defaultNSpawnTemplateData("kube1")) + + require.Contains(t, out, "Requires=unbounded-agent-nspawn-config@kube1.service") + require.Contains(t, out, "After=unbounded-agent-nspawn-config@kube1.service") + require.Less(t, strings.Index(out, "[Unit]"), strings.Index(out, "Requires=unbounded-agent-nspawn-config@kube1.service")) + require.Less(t, strings.Index(out, "After=unbounded-agent-nspawn-config@kube1.service"), strings.Index(out, "[Service]")) +} + +func TestNSpawnConfigRefreshUnit(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "nspawn-config-refresh.service", defaultNSpawnTemplateData("kube1"))) + + out := buf.String() + require.Contains(t, out, "Description=Regenerate nspawn configuration for kube1") + require.Contains(t, out, "Type=oneshot") + require.Contains(t, out, "ExecStart=/usr/local/bin/unbounded-agent regenerate-nspawn-config kube1") +} + +func defaultNSpawnTemplateData(machineName string) nspawnTemplateData { + return nspawnTemplateData{ + MachineName: machineName, + BPFFSMountPath: goalstates.BPFFSMountPath(machineName), + ConfigRefreshUnit: goalstates.NSpawnConfigRefreshUnit(machineName), + AgentBinaryPath: goalstates.DaemonBinaryPath, + } +} + func requireRenderedSnapshot(t *testing.T, goldenFile, templateName string, data nspawnTemplateData) string { t.Helper() diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden index db147d346..46de5c62d 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires=unbounded-agent-nspawn-config@kube1.service +After=unbounded-agent-nspawn-config@kube1.service StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden index 2e445ebc1..d238e7ede 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden @@ -32,6 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] +Requires=unbounded-agent-nspawn-config@kube2.service +After=unbounded-agent-nspawn-config@kube2.service StartLimitIntervalSec=0 [Service] From 71158beb45980941f0638c9985b5ec73ca9911da Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:11:24 +0000 Subject: [PATCH 3/4] Rename nspawn refresh unit to config regeneration --- cmd/agent/internal/cmd/cmd.go | 1 + cmd/agent/internal/cmd/nspawn_config.go | 27 +++++++++- docs/content/reference/agent/nspawn.md | 4 +- pkg/agent/goalstates/constants.go | 8 +-- pkg/agent/phases/reset/nspawn.go | 9 ++-- .../rootfs/assets/config-regeneration.service | 9 ++++ .../assets/nspawn-config-refresh.service | 9 ---- .../rootfs/assets/service-override.conf | 4 +- pkg/agent/phases/rootfs/nspawn.go | 49 ++++++++++--------- pkg/agent/phases/rootfs/nspawn_render_test.go | 29 ++++++----- .../service-override-kube1.conf.golden | 4 +- .../service-override-kube2.conf.golden | 4 +- 12 files changed, 94 insertions(+), 63 deletions(-) create mode 100644 pkg/agent/phases/rootfs/assets/config-regeneration.service delete mode 100644 pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index 7acc00046..f401cd45e 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -30,6 +30,7 @@ func Run() { newCmdDaemon(cmdCtx), newCmdReset(cmdCtx), newCmdVersion(), + newCmdRegenerateConfig(cmdCtx), newCmdRegenerateNSpawnConfig(cmdCtx), newCmdRecordAgentUpgradeFailureSignal(), ) diff --git a/cmd/agent/internal/cmd/nspawn_config.go b/cmd/agent/internal/cmd/nspawn_config.go index 7329189d3..6553de34b 100644 --- a/cmd/agent/internal/cmd/nspawn_config.go +++ b/cmd/agent/internal/cmd/nspawn_config.go @@ -20,6 +20,25 @@ import ( "github.com/Azure/unbounded/pkg/agent/phases/rootfs" ) +func newCmdRegenerateConfig(cmdCtx *CommandContext) *cobra.Command { + cmd := &cobra.Command{ + Use: "regenerate-config MACHINE_NAME", + Short: "Regenerate host-side configuration for a machine", + Hidden: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) + defer cancel() + + cmdCtx.Setup() + + return regenerateConfig(ctx, cmdCtx.Logger, args[0]) + }, + } + + return cmd +} + func newCmdRegenerateNSpawnConfig(cmdCtx *CommandContext) *cobra.Command { cmd := &cobra.Command{ Use: "regenerate-nspawn-config MACHINE_NAME", @@ -39,6 +58,10 @@ func newCmdRegenerateNSpawnConfig(cmdCtx *CommandContext) *cobra.Command { return cmd } +func regenerateConfig(ctx context.Context, log *slog.Logger, machineName string) error { + return regenerateNSpawnConfig(ctx, log, machineName) +} + func regenerateNSpawnConfig(ctx context.Context, log *slog.Logger, machineName string) error { cfg, ok, err := loadAppliedConfigForMachine(log, machineName) if err != nil { @@ -68,6 +91,7 @@ func loadAppliedConfigForMachine(log *slog.Logger, machineName string) (*provisi } path := goalstates.AppliedConfigPath(machineName) + data, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return nil, false, nil @@ -83,7 +107,8 @@ func loadAppliedConfigForMachine(log *slog.Logger, machineName string) (*provisi } if _, statErr := os.Stat(checksumPath); errors.Is(statErr, os.ErrNotExist) { - log.Warn("no checksum sidecar found, skipping integrity check", + log.Warn( + "no checksum sidecar found, skipping integrity check", "config_path", path, "checksum_path", checksumPath, ) diff --git a/docs/content/reference/agent/nspawn.md b/docs/content/reference/agent/nspawn.md index f8f0c77fc..893018ffc 100644 --- a/docs/content/reference/agent/nspawn.md +++ b/docs/content/reference/agent/nspawn.md @@ -142,7 +142,7 @@ The configuration is written to these files on the host before the machine boots |---|---| | nspawn config | `/etc/systemd/nspawn/.nspawn` | | Service override | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | -| Config refresh unit | `/etc/systemd/system/unbounded-agent-nspawn-config@.service` | +| Config regeneration unit | `/etc/systemd/system/unbounded-agent-regenerate-config@.service` | ### Customization points @@ -261,7 +261,7 @@ The container operates in the host's network namespace (`VirtualEthernet=no`): | `/var/lib/machines/` | Container rootfs directory. | | `/etc/systemd/nspawn/.nspawn` | nspawn configuration file. | | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | Systemd service override. | -| `/etc/systemd/system/unbounded-agent-nspawn-config@.service` | Host-side oneshot unit that refreshes nspawn configuration before machine start. | +| `/etc/systemd/system/unbounded-agent-regenerate-config@.service` | Host-side oneshot unit that regenerates host-side configuration before machine start. | | `/run/host-nvidia//` | (Inside container) Read-only bind-mount of host NVIDIA library directories. | ## See Also diff --git a/pkg/agent/goalstates/constants.go b/pkg/agent/goalstates/constants.go index 1931d17a7..dce9002f0 100644 --- a/pkg/agent/goalstates/constants.go +++ b/pkg/agent/goalstates/constants.go @@ -74,10 +74,10 @@ func BPFFSMountPath(machineName string) string { return fmt.Sprintf("%s/%s", BPFFSMountDir, machineName) } -// NSpawnConfigRefreshUnit returns the systemd unit that refreshes host-side -// nspawn configuration before the named machine starts. -func NSpawnConfigRefreshUnit(machineName string) string { - return fmt.Sprintf("unbounded-agent-nspawn-config@%s.service", machineName) +// ConfigRegenerationUnit returns the systemd unit that regenerates host-side +// configuration before the named machine starts. +func ConfigRegenerationUnit(machineName string) string { + return fmt.Sprintf("unbounded-agent-regenerate-config@%s.service", machineName) } // AppliedConfigPath returns the path to the applied config file for the diff --git a/pkg/agent/phases/reset/nspawn.go b/pkg/agent/phases/reset/nspawn.go index 3c18476ab..df6b24b2a 100644 --- a/pkg/agent/phases/reset/nspawn.go +++ b/pkg/agent/phases/reset/nspawn.go @@ -31,13 +31,13 @@ func (t *removeNSpawnConfig) Name() string { return "remove-nspawn-config" } func (t *removeNSpawnConfig) Do(_ context.Context) error { nspawnFile := fmt.Sprintf("%s/%s.nspawn", goalstates.SystemdNSpawnDir, t.machineName) overrideDir := fmt.Sprintf("%s/systemd-nspawn@%s.service.d", goalstates.SystemdSystemDir, t.machineName) - configRefreshUnit := fmt.Sprintf("%s/%s", goalstates.SystemdSystemDir, goalstates.NSpawnConfigRefreshUnit(t.machineName)) + configRegenerationUnit := fmt.Sprintf("%s/%s", goalstates.SystemdSystemDir, goalstates.ConfigRegenerationUnit(t.machineName)) - t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir, "config_refresh_unit", configRefreshUnit) + t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir, "config_regeneration_unit", configRegenerationUnit) removeFileIfExists(t.log, nspawnFile) removeAllIfExists(t.log, overrideDir) - removeFileIfExists(t.log, configRefreshUnit) + removeFileIfExists(t.log, configRegenerationUnit) return nil } @@ -81,7 +81,8 @@ func (t *removeBPFFSMount) Do(ctx context.Context) error { // CleanupMachine returns a composite task that removes all artifacts of an // nspawn machine: its nspawn configuration and rootfs. func CleanupMachine(log *slog.Logger, machineName string) phases.Task { - return phases.Serial(log, + return phases.Serial( + log, RemoveNSpawnConfig(log, machineName), RemoveMachine(log, machineName), RemoveBPFFSMount(log, machineName), diff --git a/pkg/agent/phases/rootfs/assets/config-regeneration.service b/pkg/agent/phases/rootfs/assets/config-regeneration.service new file mode 100644 index 000000000..4a981599d --- /dev/null +++ b/pkg/agent/phases/rootfs/assets/config-regeneration.service @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[Unit] +Description=Regenerate configuration for {{.MachineName}} + +[Service] +Type=oneshot +ExecStart={{.AgentBinaryPath}} regenerate-config {{.MachineName}} diff --git a/pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service b/pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service deleted file mode 100644 index 0e59f5c67..000000000 --- a/pkg/agent/phases/rootfs/assets/nspawn-config-refresh.service +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -[Unit] -Description=Regenerate nspawn configuration for {{.MachineName}} - -[Service] -Type=oneshot -ExecStart={{.AgentBinaryPath}} regenerate-nspawn-config {{.MachineName}} diff --git a/pkg/agent/phases/rootfs/assets/service-override.conf b/pkg/agent/phases/rootfs/assets/service-override.conf index 65f4e7b46..f9e920735 100644 --- a/pkg/agent/phases/rootfs/assets/service-override.conf +++ b/pkg/agent/phases/rootfs/assets/service-override.conf @@ -32,8 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] -Requires={{.ConfigRefreshUnit}} -After={{.ConfigRefreshUnit}} +Requires={{.ConfigRegenerationUnit}} +After={{.ConfigRegenerationUnit}} StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/nspawn.go b/pkg/agent/phases/rootfs/nspawn.go index eb5fb7bbc..fec0f1b66 100644 --- a/pkg/agent/phases/rootfs/nspawn.go +++ b/pkg/agent/phases/rootfs/nspawn.go @@ -18,11 +18,11 @@ import ( "github.com/Azure/unbounded/pkg/agent/phases/rootfs/oci" ) -//go:embed assets/nspawn.conf assets/nspawn-config-refresh.service assets/service-override.conf +//go:embed assets/nspawn.conf assets/config-regeneration.service assets/service-override.conf var nspawnAssets embed.FS var nspawnTemplates = template.Must( - template.New("nspawn").ParseFS(nspawnAssets, "assets/nspawn.conf", "assets/nspawn-config-refresh.service", "assets/service-override.conf"), + template.New("nspawn").ParseFS(nspawnAssets, "assets/nspawn.conf", "assets/config-regeneration.service", "assets/service-override.conf"), ) type ensureNSpawnWorkspace struct { @@ -80,15 +80,15 @@ func (e *ensureNSpawnWorkspace) bootstrapWorkspace(ctx context.Context) error { type nspawnTemplateData struct { // MachineName is the nspawn machine name (e.g. "kube1"). Used by the // service drop-in for the ExecStartPre `machinectl terminate` cleanup. - MachineName string - BPFFSMountPath string - HostDevicePaths []string - NvidiaGPUDevicePaths []string - NvidiaLibDirMounts []goalstates.NvidiaLibDirMount - AMDGPUDevicePaths []string - AMDSysFSPaths []string - ConfigRefreshUnit string - AgentBinaryPath string + MachineName string + BPFFSMountPath string + HostDevicePaths []string + NvidiaGPUDevicePaths []string + NvidiaLibDirMounts []goalstates.NvidiaLibDirMount + AMDGPUDevicePaths []string + AMDSysFSPaths []string + ConfigRegenerationUnit string + AgentBinaryPath string } // writeNSpawnConfigs renders the nspawn-related templates with device and GPU @@ -101,15 +101,15 @@ func writeNSpawnConfigs(log *slog.Logger, goalState *goalstates.RootFS) error { hostDevicePaths := goalState.HostDevices.Paths() amdGPUDevicePaths := pathsExcluding(goalState.AMD.GPUDevicePaths, goalState.Nvidia.GPUDevicePaths) templateData := nspawnTemplateData{ - MachineName: machineName, - BPFFSMountPath: goalstates.BPFFSMountPath(machineName), - HostDevicePaths: hostDevicePaths, - NvidiaGPUDevicePaths: goalState.Nvidia.GPUDevicePaths, - NvidiaLibDirMounts: goalState.Nvidia.LibDirMounts, - AMDGPUDevicePaths: amdGPUDevicePaths, - AMDSysFSPaths: goalState.AMD.SysFSPaths, - ConfigRefreshUnit: goalstates.NSpawnConfigRefreshUnit(machineName), - AgentBinaryPath: goalstates.DaemonBinaryPath, + MachineName: machineName, + BPFFSMountPath: goalstates.BPFFSMountPath(machineName), + HostDevicePaths: hostDevicePaths, + NvidiaGPUDevicePaths: goalState.Nvidia.GPUDevicePaths, + NvidiaLibDirMounts: goalState.Nvidia.LibDirMounts, + AMDGPUDevicePaths: amdGPUDevicePaths, + AMDSysFSPaths: goalState.AMD.SysFSPaths, + ConfigRegenerationUnit: goalstates.ConfigRegenerationUnit(machineName), + AgentBinaryPath: goalstates.DaemonBinaryPath, } if len(hostDevicePaths) > 0 { @@ -152,14 +152,15 @@ func writeNSpawnConfigs(log *slog.Logger, goalState *goalstates.RootFS) error { return fmt.Errorf("write service override %s: %w", goalState.ServiceOverrideFile, err) } - unitFile := filepath.Join(goalstates.SystemdSystemDir, templateData.ConfigRefreshUnit) + unitFile := filepath.Join(goalstates.SystemdSystemDir, templateData.ConfigRegenerationUnit) + unitBuf := &bytes.Buffer{} - if err := nspawnTemplates.ExecuteTemplate(unitBuf, "nspawn-config-refresh.service", templateData); err != nil { - return fmt.Errorf("render nspawn config refresh unit template: %w", err) + if err := nspawnTemplates.ExecuteTemplate(unitBuf, "config-regeneration.service", templateData); err != nil { + return fmt.Errorf("render config regeneration unit template: %w", err) } if err := utilio.WriteFile(unitFile, unitBuf.Bytes(), 0o644); err != nil { - return fmt.Errorf("write nspawn config refresh unit %s: %w", unitFile, err) + return fmt.Errorf("write config regeneration unit %s: %w", unitFile, err) } return nil diff --git a/pkg/agent/phases/rootfs/nspawn_render_test.go b/pkg/agent/phases/rootfs/nspawn_render_test.go index 72485c3dd..5bf8f7500 100644 --- a/pkg/agent/phases/rootfs/nspawn_render_test.go +++ b/pkg/agent/phases/rootfs/nspawn_render_test.go @@ -36,6 +36,7 @@ func TestServiceOverride_HostDevicesDeviceAllow(t *testing.T) { t.Parallel() var buf bytes.Buffer + data := defaultNSpawnTemplateData("kube1") data.HostDevicePaths = []string{"/dev/kvm"} require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", data)) @@ -57,6 +58,7 @@ func TestServiceOverride_MultipleHostDevices(t *testing.T) { devices := []string{"/dev/kvm", "/dev/net/tun", "/dev/vhost-net", "/dev/sda", "/dev/infiniband/uverbs0"} var nspawnBuf bytes.Buffer + data := defaultNSpawnTemplateData("kube1") data.HostDevicePaths = devices require.NoError(t, nspawnTemplates.ExecuteTemplate(&nspawnBuf, "nspawn.conf", data)) @@ -79,6 +81,7 @@ func TestServiceOverride_AMDGPUDevices(t *testing.T) { devices := []string{"/dev/dri/card0", "/dev/dri/renderD128", "/dev/kfd"} var nspawnBuf bytes.Buffer + data := defaultNSpawnTemplateData("kube1") data.AMDGPUDevicePaths = devices data.AMDSysFSPaths = []string{"/sys/module/amdgpu", "/sys/class/kfd"} @@ -120,35 +123,35 @@ func TestServiceOverride_NoHostDevicesNoDeviceAllow(t *testing.T) { require.NotContains(t, buf.String(), "DeviceAllow=") } -func TestServiceOverride_ConfigRefreshDependency(t *testing.T) { +func TestServiceOverride_ConfigRegenerationDependency(t *testing.T) { t.Parallel() out := requireRenderedSnapshot(t, "service-override-kube1.conf.golden", "service-override.conf", defaultNSpawnTemplateData("kube1")) - require.Contains(t, out, "Requires=unbounded-agent-nspawn-config@kube1.service") - require.Contains(t, out, "After=unbounded-agent-nspawn-config@kube1.service") - require.Less(t, strings.Index(out, "[Unit]"), strings.Index(out, "Requires=unbounded-agent-nspawn-config@kube1.service")) - require.Less(t, strings.Index(out, "After=unbounded-agent-nspawn-config@kube1.service"), strings.Index(out, "[Service]")) + require.Contains(t, out, "Requires=unbounded-agent-regenerate-config@kube1.service") + require.Contains(t, out, "After=unbounded-agent-regenerate-config@kube1.service") + require.Less(t, strings.Index(out, "[Unit]"), strings.Index(out, "Requires=unbounded-agent-regenerate-config@kube1.service")) + require.Less(t, strings.Index(out, "After=unbounded-agent-regenerate-config@kube1.service"), strings.Index(out, "[Service]")) } -func TestNSpawnConfigRefreshUnit(t *testing.T) { +func TestConfigRegenerationUnit(t *testing.T) { t.Parallel() var buf bytes.Buffer - require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "nspawn-config-refresh.service", defaultNSpawnTemplateData("kube1"))) + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "config-regeneration.service", defaultNSpawnTemplateData("kube1"))) out := buf.String() - require.Contains(t, out, "Description=Regenerate nspawn configuration for kube1") + require.Contains(t, out, "Description=Regenerate configuration for kube1") require.Contains(t, out, "Type=oneshot") - require.Contains(t, out, "ExecStart=/usr/local/bin/unbounded-agent regenerate-nspawn-config kube1") + require.Contains(t, out, "ExecStart=/usr/local/bin/unbounded-agent regenerate-config kube1") } func defaultNSpawnTemplateData(machineName string) nspawnTemplateData { return nspawnTemplateData{ - MachineName: machineName, - BPFFSMountPath: goalstates.BPFFSMountPath(machineName), - ConfigRefreshUnit: goalstates.NSpawnConfigRefreshUnit(machineName), - AgentBinaryPath: goalstates.DaemonBinaryPath, + MachineName: machineName, + BPFFSMountPath: goalstates.BPFFSMountPath(machineName), + ConfigRegenerationUnit: goalstates.ConfigRegenerationUnit(machineName), + AgentBinaryPath: goalstates.DaemonBinaryPath, } } diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden index 46de5c62d..4999808f5 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden @@ -32,8 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] -Requires=unbounded-agent-nspawn-config@kube1.service -After=unbounded-agent-nspawn-config@kube1.service +Requires=unbounded-agent-regenerate-config@kube1.service +After=unbounded-agent-regenerate-config@kube1.service StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden index d238e7ede..3767ee335 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden @@ -32,8 +32,8 @@ # maps, but each nspawn machine should get its own pinned-object namespace. [Unit] -Requires=unbounded-agent-nspawn-config@kube2.service -After=unbounded-agent-nspawn-config@kube2.service +Requires=unbounded-agent-regenerate-config@kube2.service +After=unbounded-agent-regenerate-config@kube2.service StartLimitIntervalSec=0 [Service] From 416a1b905194e60c12b1babb53d4a4eb39e22ca3 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Mon, 10 Aug 2026 05:44:42 +0000 Subject: [PATCH 4/4] agent: harden nspawn config regeneration --- cmd/agent/internal/cmd/nspawn_config.go | 21 ++- cmd/agent/internal/cmd/nspawn_config_test.go | 69 +++++++++ hack/agent/e2e-kind/e2e.py | 145 ++++++++++++++++++ hack/agent/e2e-kind/node-configs/README.md | 2 + .../node-configs/additional-host-mounts.json | 1 + pkg/agent/goalstates/resolve.go | 67 +++++--- pkg/agent/goalstates/resolve_test.go | 13 ++ .../rootfs/assets/config-regeneration.service | 2 + pkg/agent/phases/rootfs/nspawn_render_test.go | 2 + 9 files changed, 293 insertions(+), 29 deletions(-) create mode 100644 cmd/agent/internal/cmd/nspawn_config_test.go diff --git a/cmd/agent/internal/cmd/nspawn_config.go b/cmd/agent/internal/cmd/nspawn_config.go index 6553de34b..50e178647 100644 --- a/cmd/agent/internal/cmd/nspawn_config.go +++ b/cmd/agent/internal/cmd/nspawn_config.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" + "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/phases" @@ -73,15 +74,22 @@ func regenerateNSpawnConfig(ctx context.Context, log *slog.Logger, machineName s return nil } - gs, err := goalstates.ResolveMachine(log, cfg, machineName, nil) + rootFS, err := goalstates.ResolveNSpawnConfig(cfg, machineName) if err != nil { - return fmt.Errorf("resolve machine goal state: %w", err) + return fmt.Errorf("resolve nspawn config goal state: %w", err) } - if err := phases.ExecuteTask(ctx, log, rootfs.EnsureNSpawnConfig(log, gs.RootFS)); err != nil { + if err := phases.ExecuteTask(ctx, log, rootfs.EnsureNSpawnConfig(log, rootFS)); err != nil { return fmt.Errorf("regenerate nspawn config for %s: %w", machineName, err) } + // systemd loaded the nspawn service drop-in before starting this required + // oneshot unit. Reload the manager so the pending nspawn start observes the + // regenerated service properties, including path-specific DeviceAllow entries. + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "daemon-reload"); err != nil { + return fmt.Errorf("reload systemd after regenerating config for %s: %w", machineName, err) + } + return nil } @@ -90,8 +98,10 @@ func loadAppliedConfigForMachine(log *slog.Logger, machineName string) (*provisi return nil, false, fmt.Errorf("unsupported nspawn machine %q", machineName) } - path := goalstates.AppliedConfigPath(machineName) + return loadAppliedConfig(log, goalstates.AppliedConfigPath(machineName), goalstates.AppliedConfigChecksumPath(machineName)) +} +func loadAppliedConfig(log *slog.Logger, path, checksumPath string) (*provision.AgentConfig, bool, error) { data, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return nil, false, nil @@ -101,9 +111,8 @@ func loadAppliedConfigForMachine(log *slog.Logger, machineName string) (*provisi return nil, false, fmt.Errorf("read applied config %s: %w", path, err) } - checksumPath := goalstates.AppliedConfigChecksumPath(machineName) if err := goalstates.VerifyChecksum(data, checksumPath); err != nil { - return nil, false, fmt.Errorf("verify applied config checksum for %s: %w", machineName, err) + return nil, false, fmt.Errorf("verify applied config checksum for %s: %w", path, err) } if _, statErr := os.Stat(checksumPath); errors.Is(statErr, os.ErrNotExist) { diff --git a/cmd/agent/internal/cmd/nspawn_config_test.go b/cmd/agent/internal/cmd/nspawn_config_test.go new file mode 100644 index 000000000..3532537da --- /dev/null +++ b/cmd/agent/internal/cmd/nspawn_config_test.go @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" +) + +func TestLoadAppliedConfig(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + configPath := filepath.Join(dir, "applied-config.json") + checksumPath := configPath + ".sha256" + want := provision.AgentConfig{ + MachineName: "machine-1", + NodeName: "node-1", + } + + data, err := json.Marshal(&want) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, data, 0o600)) + require.NoError(t, os.WriteFile(checksumPath, []byte(goalstates.ComputeChecksum(data)+"\n"), 0o600)) + + got, ok, err := loadAppliedConfig(testLogger(), configPath, checksumPath) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, want.MachineName, got.MachineName) + require.Equal(t, want.NodeName, got.NodeName) +} + +func TestLoadAppliedConfigMissing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + got, ok, err := loadAppliedConfig( + testLogger(), + filepath.Join(dir, "missing.json"), + filepath.Join(dir, "missing.json.sha256"), + ) + + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, got) +} + +func TestLoadAppliedConfigChecksumMismatch(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + configPath := filepath.Join(dir, "applied-config.json") + checksumPath := configPath + ".sha256" + require.NoError(t, os.WriteFile(configPath, []byte(`{"MachineName":"machine-1"}`), 0o600)) + require.NoError(t, os.WriteFile(checksumPath, []byte(goalstates.ComputeChecksum([]byte("different"))), 0o600)) + + got, ok, err := loadAppliedConfig(testLogger(), configPath, checksumPath) + require.ErrorIs(t, err, goalstates.ErrChecksumMismatch) + require.False(t, ok) + require.Nil(t, got) +} diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index c467d274d..1054e6830 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -30,6 +30,8 @@ wait-for-node Wait for the node to appear and become Ready. validate-host-nspawn-distro Verify the nspawn machine distro matches the host. validate-node-config Verify configured node and kubelet settings. + validate-device-refresh-after-host-reboot + Verify discovered devices refresh on host boot. dump-persisted-agent-config Print persisted agent config files from the VM. validate-workload Deploy test pods on the agent node. validate-kube-proxy Verify kube-proxy is Running on all nodes. @@ -144,6 +146,8 @@ DAEMON_BINARY_CURRENT = "/usr/local/bin/unbounded-agent-current" DAEMON_BINARY_LAST_GOOD = "/usr/local/bin/unbounded-agent-last-good" BPFFS_SENTINEL = "unbounded-e2e-bpffs-sentinel" +DEVICE_REFRESH_PATH = "/dev/infiniband/unbounded-e2e-zero" +DEVICE_REFRESH_TMPFILES_PATH = "/etc/tmpfiles.d/unbounded-e2e-device.conf" # --------------------------------------------------------------------------- @@ -595,6 +599,7 @@ class NodeConfig: block_external_network: bool = False additional_host_mounts: tuple[dict[str, Any], ...] = () additional_host_devices: tuple[str, ...] = () + validate_device_refresh_after_host_reboot: bool = False local_dns: bool = False path: str = "" @@ -636,6 +641,9 @@ def load_node_config( block_external_network = cfg.get("blockExternalNetwork", False) additional_host_mounts = cfg.get("additionalHostMounts", []) additional_host_devices = cfg.get("additionalHostDevices", []) + validate_device_refresh_after_host_reboot = cfg.get( + "validateDeviceRefreshAfterHostReboot", False, + ) local_dns = cfg.get("localDNS", False) if not isinstance(name, str) or not name: @@ -672,6 +680,11 @@ def load_node_config( ) if not isinstance(local_dns, bool): die(f"node config {config_path} field 'localDNS' must be a boolean") + if not isinstance(validate_device_refresh_after_host_reboot, bool): + die( + f"node config {config_path} field " + f"'validateDeviceRefreshAfterHostReboot' must be a boolean" + ) if not isinstance(additional_host_devices, list) or not all( isinstance(d, str) and d for d in additional_host_devices ): @@ -691,6 +704,7 @@ def load_node_config( block_external_network=block_external_network, additional_host_mounts=tuple(dict(m) for m in additional_host_mounts), additional_host_devices=tuple(additional_host_devices), + validate_device_refresh_after_host_reboot=validate_device_refresh_after_host_reboot, local_dns=local_dns, path=str(config_path), ) @@ -813,6 +827,10 @@ def log_active_node_config(node_config: NodeConfig) -> None: log(f" additional host devices: {', '.join(node_config.additional_host_devices)}") else: log(f" additional host devices: ") + log( + " validate device refresh after host reboot: " + f"{node_config.validate_device_refresh_after_host_reboot}" + ) def _safe_name(value: str) -> str: @@ -2887,6 +2905,125 @@ def validate_additional_host_devices_config(node_config: NodeConfig) -> None: log("Additional host devices configuration validated") +def wait_for_vm_host_reboot(previous_boot_id: str, timeout_secs: int = 300) -> None: + """Wait for SSH to return with a different host boot ID.""" + + log(f"Waiting for VM host to reboot (timeout: {timeout_secs}s)...") + deadline = time.monotonic() + timeout_secs + while time.monotonic() < deadline: + result = subprocess.run( + [ + "ssh", *SSH_OPTS, SSH_TARGET, + "cat /proc/sys/kernel/random/boot_id", + ], + capture_output=True, + text=True, + check=False, + ) + current_boot_id = result.stdout.strip() if result.returncode == 0 else "" + if current_boot_id and current_boot_id != previous_boot_id: + log(f"VM host boot ID changed: {previous_boot_id} -> {current_boot_id}") + return + time.sleep(3) + + die("timed out waiting for VM host reboot") + + +def wait_for_nspawn_machine(machine: str, timeout_secs: int = 300) -> None: + """Wait for the named nspawn machine to become active after host reboot.""" + + log(f"Waiting for nspawn machine '{machine}' to become active...") + deadline = time.monotonic() + timeout_secs + while time.monotonic() < deadline: + result = ssh_capture_quiet(f"sudo machinectl show {machine}") + if result.returncode == 0: + log(f"nspawn machine '{machine}' is active") + return + time.sleep(3) + + die(f"timed out waiting for nspawn machine '{machine}' after host reboot") + + +def validate_device_refresh_after_host_reboot() -> None: + """Verify a device appearing during host boot is usable inside nspawn.""" + + machine = active_nspawn_machine() + nspawn_config_path = f"/etc/systemd/nspawn/{machine}.nspawn" + override_path = ( + f"/etc/systemd/system/systemd-nspawn@{machine}.service.d/override.conf" + ) + + log("Validating device discovery refresh after a VM host reboot...") + ssh_cmd( + f"sudo test ! -e {DEVICE_REFRESH_PATH} && " + f"! sudo grep -Fq {DEVICE_REFRESH_PATH} {nspawn_config_path} && " + f"! sudo grep -Fq {DEVICE_REFRESH_PATH} {override_path}" + ) + + # Create an alias of /dev/zero on the next boot. Placing it under + # /dev/infiniband makes it part of normal host device discovery without + # requiring RDMA hardware in the e2e VM. + ssh_cmd(f""" +sudo tee {DEVICE_REFRESH_TMPFILES_PATH} >/dev/null <<'EOF' +d /dev/infiniband 0755 root root - +c {DEVICE_REFRESH_PATH} 0666 root root - 1:5 +EOF +""") + + previous_boot_id = ssh_capture("cat /proc/sys/kernel/random/boot_id").strip() + if not previous_boot_id: + die("VM host boot ID was empty before reboot") + + # SSH normally exits with 255 when systemd tears down the connection. + subprocess.run( + ["ssh", *SSH_OPTS, SSH_TARGET, "sudo systemctl reboot"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + wait_for_vm_host_reboot(previous_boot_id) + wait_for_nspawn_machine(machine) + wait_for_node_ready(AGENT_MACHINE_NAME) + + ssh_cmd(f"sudo test -c {DEVICE_REFRESH_PATH}") + nspawn_config = ssh_capture(f"sudo cat {nspawn_config_path}") + expected_bind = f"Bind={DEVICE_REFRESH_PATH}" + if expected_bind not in nspawn_config: + die( + f"nspawn config {nspawn_config_path} missing refreshed directive " + f"{expected_bind!r}; full config:\n{nspawn_config}" + ) + + service_override = ssh_capture(f"sudo cat {override_path}") + expected_allow = f"DeviceAllow={DEVICE_REFRESH_PATH} rwm" + if expected_allow not in service_override: + die( + f"service override {override_path} missing refreshed directive " + f"{expected_allow!r}; full override:\n{service_override}" + ) + + loaded_device_allow = ssh_capture( + f"sudo systemctl show systemd-nspawn@{machine}.service " + "--property=DeviceAllow" + ) + if DEVICE_REFRESH_PATH not in loaded_device_allow: + die( + "systemd did not load the refreshed DeviceAllow setting; " + f"loaded property: {loaded_device_allow!r}" + ) + + # The manager property verifies the service-level permission was reloaded; + # reading the device verifies that the refreshed bind is usable in nspawn. + machine_shell( + machine, + f"test -c {DEVICE_REFRESH_PATH} && " + f"dd if={DEVICE_REFRESH_PATH} of=/dev/null bs=1 count=1 status=none", + ) + + log("Device discovery refresh after VM host reboot validated") + + def _run_scenario_command(command: str, node_config: NodeConfig, env: dict[str, str]) -> None: args = [sys.executable, str(Path(__file__))] if VERBOSE: @@ -2925,6 +3062,11 @@ def _validate_node_config_scenario(node_config: NodeConfig, index: int, agent_ur ): _run_scenario_command(command, node_config, env) + if node_config.validate_device_refresh_after_host_reboot: + _run_scenario_command( + "validate-device-refresh-after-host-reboot", node_config, env, + ) + if node_config.local_dns: _run_scenario_command("validate-node-reboot-operation", node_config, env) _run_scenario_command("validate-node-config", node_config, env) @@ -4315,6 +4457,9 @@ def command(_node_config: NodeConfig) -> None: "wait-for-node-registered": _without_node_config(wait_for_node_registered), "validate-host-nspawn-distro": _without_node_config(validate_host_nspawn_distro), "validate-node-config": validate_node_config, + "validate-device-refresh-after-host-reboot": _without_node_config( + validate_device_refresh_after_host_reboot, + ), "validate-kube-proxy": _without_node_config(validate_kube_proxy), "validate-workload": _without_node_config(validate_workload), "install-machine-crd": _without_node_config(install_machine_crd), diff --git a/hack/agent/e2e-kind/node-configs/README.md b/hack/agent/e2e-kind/node-configs/README.md index 716e9afb3..a58611966 100644 --- a/hack/agent/e2e-kind/node-configs/README.md +++ b/hack/agent/e2e-kind/node-configs/README.md @@ -15,6 +15,8 @@ JSON file can be passed to `e2e.py` with `--node-config`. | `blockExternalNetwork` | boolean | Optional. When true, the e2e installs required host packages, then blocks VM egress outside local e2e networks before running bootstrap. This is intended for offline bootstrap validation. The offline artifact bundle includes kube-system images needed for node readiness plus the e2e workload image. | | `localDNS` | boolean | Enables the nspawn-local CoreDNS cache and validates service health, resolver wiring, metrics, listener addresses, and NOTRACK rules. The scenario must also provide `nodeIP` so metrics bind to the Node InternalIP. | | `additionalHostMounts` | array of objects | Optional extra host bind-mounts for the nspawn machine. Each entry has a required string `source` (clean absolute host path), optional string `target` (defaults to `source`), and optional bool `readOnly`. The e2e passes each entry as `--additional-host-mount` to `manual-bootstrap` and validates the resulting `Bind=` / `BindReadOnly=` directives in the nspawn config. | +| `additionalHostDevices` | array of strings | Optional host device paths or systemd device-group specifiers passed through `--additional-host-device`. The e2e validates generated `Bind=` and `DeviceAllow=` directives for device paths. | +| `validateDeviceRefreshAfterHostReboot` | boolean | Creates a synthetic device during the next VM host boot and verifies nspawn regeneration makes the device accessible inside the machine. | The `validate-node-configs` parent process prepares OCI refs once, then passes the local refs to each child `e2e.py` invocation with diff --git a/hack/agent/e2e-kind/node-configs/additional-host-mounts.json b/hack/agent/e2e-kind/node-configs/additional-host-mounts.json index 5390a8a59..f70e8fb33 100644 --- a/hack/agent/e2e-kind/node-configs/additional-host-mounts.json +++ b/hack/agent/e2e-kind/node-configs/additional-host-mounts.json @@ -1,5 +1,6 @@ { "name": "additional-host-mounts", + "validateDeviceRefreshAfterHostReboot": true, "nodeLabels": { "e2e.unbounded-cloud.io/config": "additional-host-mounts" }, diff --git a/pkg/agent/goalstates/resolve.go b/pkg/agent/goalstates/resolve.go index 3a8606b17..6c22a5f46 100644 --- a/pkg/agent/goalstates/resolve.go +++ b/pkg/agent/goalstates/resolve.go @@ -33,17 +33,49 @@ type MachineGoalState struct { NodeStart *NodeStart } +// ResolveNSpawnConfig probes only the host state needed to render the +// systemd-nspawn configuration for a machine. Unlike ResolveMachine, it does +// not resolve network-dependent node services such as LocalDNS. +func ResolveNSpawnConfig(cfg *config.AgentConfig, machineName string) (*RootFS, error) { + if err := config.ValidateAdditionalHostDevices(cfg.AdditionalHostDevices); err != nil { + return nil, err + } + + additionalHostMounts, err := resolveAdditionalHostMounts(cfg.AdditionalHostMounts) + if err != nil { + return nil, err + } + + nvidia, err := ResolveNvidiaHost(runtime.GOARCH) + if err != nil { + return nil, fmt.Errorf("resolve nvidia host: %w", err) + } + + return &RootFS{ + MachineDir: filepath.Join("/var/lib/machines", machineName), + NSpawnConfigFile: filepath.Join( + SystemdNSpawnDir, + machineName+".nspawn", + ), + ServiceOverrideFile: filepath.Join( + SystemdSystemDir, + fmt.Sprintf("systemd-nspawn@%s.service.d", machineName), + "override.conf", + ), + Nvidia: nvidia, + AMD: ResolveAMDHost(), + HostDevices: DiscoverHostDevices(cfg.AdditionalHostDevices), + AdditionalHostMounts: additionalHostMounts, + }, nil +} + // ResolveMachine probes the host (kernel version, hostname, GPU hardware) and // resolves the complete goal state for the named nspawn machine from an agent // config and caller-provided download overrides. func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName string, downloads *DownloadOverrides) (*MachineGoalState, error) { sandboxImage := cfg.CRI.Containerd.SandboxImage - if err := config.ValidateAdditionalHostDevices(cfg.AdditionalHostDevices); err != nil { - return nil, err - } - - additionalHostMounts, err := resolveAdditionalHostMounts(cfg.AdditionalHostMounts) + nspawnConfig, err := ResolveNSpawnConfig(cfg, machineName) if err != nil { return nil, err } @@ -58,12 +90,8 @@ func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName strin return nil, fmt.Errorf("get host hostname: %w", err) } - nvidia, err := ResolveNvidiaHost(runtime.GOARCH) - if err != nil { - return nil, fmt.Errorf("resolve nvidia host: %w", err) - } - - amd := ResolveAMDHost() + nvidia := nspawnConfig.Nvidia + amd := nspawnConfig.AMD ociImage := ResolveOCIImage(log, cfg.OCIImage, len(nvidia.GPUDevicePaths) > 0) @@ -98,16 +126,9 @@ func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName strin } rootFS := &RootFS{ - MachineDir: filepath.Join("/var/lib/machines", machineName), - NSpawnConfigFile: filepath.Join( - SystemdNSpawnDir, - machineName+".nspawn", - ), - ServiceOverrideFile: filepath.Join( - SystemdSystemDir, - fmt.Sprintf("systemd-nspawn@%s.service.d", machineName), - "override.conf", - ), + MachineDir: nspawnConfig.MachineDir, + NSpawnConfigFile: nspawnConfig.NSpawnConfigFile, + ServiceOverrideFile: nspawnConfig.ServiceOverrideFile, HostArch: runtime.GOARCH, HostKernel: kernel, Hostname: hostname, @@ -120,8 +141,8 @@ func ResolveMachine(log *slog.Logger, cfg *config.AgentConfig, machineName strin OCIImage: ociImage, Nvidia: nvidia, AMD: amd, - HostDevices: DiscoverHostDevices(cfg.AdditionalHostDevices), - AdditionalHostMounts: additionalHostMounts, + HostDevices: nspawnConfig.HostDevices, + AdditionalHostMounts: nspawnConfig.AdditionalHostMounts, } nodeStart := &NodeStart{ diff --git a/pkg/agent/goalstates/resolve_test.go b/pkg/agent/goalstates/resolve_test.go index d5e7f4649..f208876bf 100644 --- a/pkg/agent/goalstates/resolve_test.go +++ b/pkg/agent/goalstates/resolve_test.go @@ -20,6 +20,19 @@ func discardLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } +func TestResolveNSpawnConfigDoesNotResolveLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &config.AgentConfig{ + LocalDNS: &config.AgentLocalDNSConfig{Enabled: true}, + } + + got, err := ResolveNSpawnConfig(cfg, NSpawnMachineKube1) + require.NoError(t, err) + require.Equal(t, "/var/lib/machines/kube1", got.MachineDir) + require.Equal(t, "/etc/systemd/nspawn/kube1.nspawn", got.NSpawnConfigFile) +} + func TestResolveOCIImage_ConfigImageTakesPrecedence(t *testing.T) { // Even when env vars and GPU are present, configImage wins. t.Setenv("AGENT_OCI_IMAGE", "env-image:latest") diff --git a/pkg/agent/phases/rootfs/assets/config-regeneration.service b/pkg/agent/phases/rootfs/assets/config-regeneration.service index 4a981599d..871b0ee03 100644 --- a/pkg/agent/phases/rootfs/assets/config-regeneration.service +++ b/pkg/agent/phases/rootfs/assets/config-regeneration.service @@ -3,6 +3,8 @@ [Unit] Description=Regenerate configuration for {{.MachineName}} +Wants=systemd-udev-settle.service +After=systemd-udev-settle.service [Service] Type=oneshot diff --git a/pkg/agent/phases/rootfs/nspawn_render_test.go b/pkg/agent/phases/rootfs/nspawn_render_test.go index 5fb89535e..5219e51e0 100644 --- a/pkg/agent/phases/rootfs/nspawn_render_test.go +++ b/pkg/agent/phases/rootfs/nspawn_render_test.go @@ -311,6 +311,8 @@ func TestConfigRegenerationUnit(t *testing.T) { out := buf.String() require.Contains(t, out, "Description=Regenerate configuration for kube1") + require.Contains(t, out, "Wants=systemd-udev-settle.service") + require.Contains(t, out, "After=systemd-udev-settle.service") require.Contains(t, out, "Type=oneshot") require.Contains(t, out, "ExecStart=/usr/local/bin/unbounded-agent regenerate-config kube1") }