feat(guestd): replace the Health-only stub with the V2b in-guest supervisor (RIG-2493)

go/internal/guestd/boot_test.go

@@ -14,6 +14,7 @@ import (
 	"context"
 	"errors"
 	"slices"
+	"syscall"
 	"testing"
 )
 
@@ -59,8 +60,8 @@ func (f *fakeMount) Mount() error {
 // terminal step, exactly as production does. served holds the service serve was
 // handed, or nil if serve was never reached. cmdline is the kernel command line
 // the readCmdline step returns; cmdlineErr, when non-nil, makes that step fail.
-func newSteps(rec *recorder, apiErr, netErr, mountErr, cmdlineErr error, cmdline string) (bootSteps, **healthService, chan struct{}) {
-	served := new(*healthService)
+func newSteps(rec *recorder, apiErr, netErr, mountErr, cmdlineErr error, cmdline string) (bootSteps, **supervisor, chan struct{}) {
+	served := new(*supervisor)
 	reached := make(chan struct{})
 	steps := bootSteps{
 		mountAPIFilesystems: func() error {
@@ -73,13 +74,17 @@ func newSteps(rec *recorder, apiErr, netErr, mountErr, cmdlineErr error, cmdline
 		},
 		net:       &fakeNet{rec: rec, err: netErr},
 		workspace: &fakeMount{rec: rec, err: mountErr},
-		serve: func(ctx context.Context, _ uint32, svc *healthService) error {
+		serve: func(ctx context.Context, _ uint32, svc *supervisor) error {
 			rec.mark("serve")
 			*served = svc
 			close(reached)
 			<-ctx.Done()
 			return ctx.Err()
 		},
+		powerOff: func() error {
+			rec.mark("poweroff")
+			return nil
+		},
 	}
 	return steps, served, reached
 }
@@ -279,3 +284,90 @@ func TestBootFailsClosedOnBadCmdline(t *testing.T) {
 		t.Fatal("a health service was constructed despite a bad cmdline")
 	}
 }
+
+// TestBootPowersOffOnRPCStop drives the full run() poweroff gate: a serve that
+// simulates an RPC Stop (flags rpcStop + cancels serving via initiateStop) then
+// drains clean must end in reboot(RB_POWER_OFF), since a bare PID-1 exit panics
+// the kernel (§(d)). Exercises the integration TestRPCStopCancels... asserts in
+// isolation.
+func TestBootPowersOffOnRPCStop(t *testing.T) {
+	rec := &recorder{}
+	steps := bootSteps{
+		mountAPIFilesystems: func() error { rec.mark("api"); return nil },
+		readCmdline:         func() ([]byte, error) { rec.mark("cmdline"); return []byte("compass.vsock_port=1024"), nil },
+		net:                 &fakeNet{rec: rec},
+		workspace:           &fakeMount{rec: rec},
+		serve: func(_ context.Context, _ uint32, svc *supervisor) error {
+			rec.mark("serve")
+			svc.initiateStop(syscall.SIGTERM) // RPC Stop: sets rpcStop, cancels serving
+			return nil                        // clean drain
+		},
+		powerOff: func() error { rec.mark("poweroff"); return nil },
+	}
+	if err := run(t.Context(), config{}, steps); err != nil {
+		t.Fatalf("run after a clean RPC stop = %v, want nil", err)
+	}
+	if !rec.ran("poweroff") {
+		t.Fatalf("power-off did not run after an RPC stop; order was %v", rec.steps)
+	}
+}
+
+// TestBootPowersOffOnRPCStopDespiteDrainError is the reliability contract: an
+// RPC Stop whose graceful drain overran (serve returns a non-nil error, e.g. a
+// child that ignored SIGTERM held Shutdown past its deadline) must STILL power
+// off, so the VMM observes a real guest shutdown within the host's timeout
+// instead of burning it to a hard kill. The poweroff is gated on rpcStop alone,
+// not on a clean serveErr.
+func TestBootPowersOffOnRPCStopDespiteDrainError(t *testing.T) {
+	rec := &recorder{}
+	drainErr := errors.New("shutdown deadline exceeded")
+	steps := bootSteps{
+		mountAPIFilesystems: func() error { rec.mark("api"); return nil },
+		readCmdline:         func() ([]byte, error) { rec.mark("cmdline"); return []byte("compass.vsock_port=1024"), nil },
+		net:                 &fakeNet{rec: rec},
+		workspace:           &fakeMount{rec: rec},
+		serve: func(_ context.Context, _ uint32, svc *supervisor) error {
+			rec.mark("serve")
+			svc.initiateStop(syscall.SIGTERM)
+			return drainErr // drain overran
+		},
+		powerOff: func() error { rec.mark("poweroff"); return nil },
+	}
+	// run returns powerOff()'s result (nil), NOT the drain error — the guest
+	// powered off, so main never falls through to a bare PID-1 exit.
+	if err := run(t.Context(), config{}, steps); err != nil {
+		t.Fatalf("run after an RPC stop with a drain error = %v, want nil (powered off)", err)
+	}
+	if !rec.ran("poweroff") {
+		t.Fatalf("power-off was skipped on a drain error during an RPC stop; order was %v", rec.steps)
+	}
+}
+
+// TestBootDoesNotPowerOffOnSignalCancel is the negative gate: a Unix-signal
+// shutdown (ctx cancelled, no RPC Stop) must NOT power off — rpcStop is false,
+// so run returns the serve error and lets main exit (the V2a path), and the
+// host observes the dial failure.
+func TestBootDoesNotPowerOffOnSignalCancel(t *testing.T) {
+	rec := &recorder{}
+	steps := bootSteps{
+		mountAPIFilesystems: func() error { rec.mark("api"); return nil },
+		readCmdline:         func() ([]byte, error) { rec.mark("cmdline"); return []byte("compass.vsock_port=1024"), nil },
+		net:                 &fakeNet{rec: rec},
+		workspace:           &fakeMount{rec: rec},
+		serve: func(ctx context.Context, _ uint32, _ *supervisor) error {
+			rec.mark("serve")
+			<-ctx.Done()
+			return ctx.Err()
+		},
+		powerOff: func() error { rec.mark("poweroff"); return nil },
+	}
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel() // simulate a Unix-signal shutdown: serve returns on ctx, rpcStop false
+	err := run(ctx, config{}, steps)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("run after a signal cancel = %v, want context.Canceled", err)
+	}
+	if rec.ran("poweroff") {
+		t.Fatalf("power-off ran on a plain signal cancel (rpcStop false); order was %v", rec.steps)
+	}
+}

go/internal/guestd/cmdline.go

@@ -3,6 +3,7 @@
 package guestd
 
 import (
+	"encoding/hex"
 	"fmt"
 	"strconv"
 	"strings"
@@ -58,3 +59,42 @@ func parseVsockPort(procCmdline string) (uint32, error) {
 	}
 	return uint32(n), nil
 }
+
+// bootNonceKey is the kernel-cmdline parameter carrying the per-session boot
+// nonce (§(e)) — a random hex value the host generates per session, passes on
+// the cmdline beside compass.vsock_port, and expects guestd to echo in
+// HealthResponse.boot_nonce. It binds the guest answering the handshake to THIS
+// BootConfig (a liveness/identity check against a stale VMM on a recycled
+// socket), not an authentication secret. It is OPTIONAL: a V2a-style cmdline
+// carries no nonce, so an absent key echoes an empty nonce and Health still
+// answers. A present-but-malformed value is a boot-config bug and fail-closes.
+const bootNonceKey = "compass.boot_nonce"
+
+// parseBootNonce extracts compass.boot_nonce=<hex> from a /proc/cmdline string,
+// following the same last-occurrence-wins tokenisation as parseVsockPort. A
+// missing key returns (nil, nil) — the nonce is optional hardening, not a
+// fail-closed boot parameter. A present key with an empty or non-hex value is a
+// malformed boot config and returns an error.
+func parseBootNonce(procCmdline string) ([]byte, error) {
+	raw := ""
+	found := false
+	for tok := range strings.FieldsSeq(procCmdline) {
+		key, val, ok := strings.Cut(tok, "=")
+		if !ok || key != bootNonceKey {
+			continue
+		}
+		raw = val
+		found = true
+	}
+	if !found {
+		return nil, nil
+	}
+	if raw == "" {
+		return nil, fmt.Errorf("kernel cmdline %s has an empty value", bootNonceKey)
+	}
+	nonce, err := hex.DecodeString(raw)
+	if err != nil {
+		return nil, fmt.Errorf("kernel cmdline %s=%q is not valid hex: %w", bootNonceKey, raw, err)
+	}
+	return nonce, nil
+}

go/internal/guestd/cmdline_test.go

@@ -7,7 +7,10 @@ package guestd
 // the handshake without a valid non-zero vsock port, so every malformed cmdline
 // is an error and only a well-formed compass.vsock_port=<n> yields a port.
 
-import "testing"
+import (
+	"bytes"
+	"testing"
+)
 
 func TestParseVsockPort(t *testing.T) {
 	tests := []struct {
@@ -107,3 +110,69 @@ func TestParseVsockPort(t *testing.T) {
 		})
 	}
 }
+
+// TestParseBootNonce defends the boot-nonce contract (§(e)): the nonce is
+// OPTIONAL hardening, so an absent key is (nil, nil) and Health still answers;
+// a present key must be valid hex; an empty or non-hex value is a malformed
+// boot config and fail-closes.
+func TestParseBootNonce(t *testing.T) {
+	tests := []struct {
+		name    string
+		cmdline string
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name:    "absent key echoes empty nonce",
+			cmdline: "console=ttyS0 compass.vsock_port=1024",
+			want:    nil,
+		},
+		{
+			name:    "valid hex nonce",
+			cmdline: "compass.vsock_port=1024 compass.boot_nonce=deadbeef",
+			want:    []byte{0xde, 0xad, 0xbe, 0xef},
+		},
+		{
+			name:    "trailing newline as /proc/cmdline yields",
+			cmdline: "compass.boot_nonce=00ff\n",
+			want:    []byte{0x00, 0xff},
+		},
+		{
+			name:    "last occurrence wins",
+			cmdline: "compass.boot_nonce=aa compass.boot_nonce=bb",
+			want:    []byte{0xbb},
+		},
+		{
+			name:    "empty value is an error",
+			cmdline: "compass.boot_nonce=",
+			wantErr: true,
+		},
+		{
+			name:    "non-hex value is an error",
+			cmdline: "compass.boot_nonce=zzzz",
+			wantErr: true,
+		},
+		{
+			name:    "odd-length hex is an error",
+			cmdline: "compass.boot_nonce=abc",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseBootNonce(tt.cmdline)
+			if tt.wantErr {
+				if err == nil {
+					t.Fatalf("parseBootNonce(%q) = %x, nil; want error", tt.cmdline, got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parseBootNonce(%q) unexpected error: %v", tt.cmdline, err)
+			}
+			if !bytes.Equal(got, tt.want) {
+				t.Fatalf("parseBootNonce(%q) = %x, want %x", tt.cmdline, got, tt.want)
+			}
+		})
+	}
+}

go/internal/guestd/guestd.go

@@ -52,6 +52,10 @@ type workspaceMounter interface {
 // cmdline inside run(), after /proc is mounted (§(d) step 1).
 type config struct {
 	guestdVersion string
+	// log is the guestd logger, threaded to run() so its diagnostics match the
+	// handler Run configures rather than the package global. Optional: a nil log
+	// (boot tests) falls back to slog.Default().
+	log *slog.Logger
 }
 
 // bootSteps are the injectable seams the orchestrator (run) depends on. The
@@ -67,10 +71,16 @@ type bootSteps struct {
 	readCmdline func() ([]byte, error)
 	net         netProvisioner
 	workspace   workspaceMounter
-	// serve receives the fully-provisioned Health service and serves it until
-	// ctx is cancelled. It is the LAST step: reaching it is the proof that net
-	// and mount both succeeded.
-	serve func(ctx context.Context, port uint32, svc *healthService) error
+	// serve receives the fully-provisioned supervisor and serves it until ctx
+	// is cancelled. It is the LAST step: reaching it is the proof that net and
+	// mount both succeeded.
+	serve func(ctx context.Context, port uint32, svc *supervisor) error
+	// powerOff performs the PID-1-legal reboot(RB_POWER_OFF) that ends an
+	// RPC-driven Stop (§(d)): a bare PID-1 exit panics the kernel, which
+	// cloud-hypervisor never observes as a VMM exit, so guestd must power the
+	// guest off explicitly. Injectable so the hermetic boot tests assert the
+	// trigger without actually rebooting the test host.
+	powerOff func() error
 }
 
 // Run is the production entry point: it wires the real Linux boot steps and
@@ -83,19 +93,28 @@ func Run(ctx context.Context, log *slog.Logger) error {
 		net:                 &linuxNetProvisioner{iface: defaultNetIface, log: log},
 		workspace:           &virtioFSMounter{tag: workspaceTag, target: workspaceTarget},
 		serve:               serveVsock,
+		powerOff:            powerOff,
 	}
-	return run(ctx, config{guestdVersion: Version}, steps)
+	return run(ctx, config{guestdVersion: Version, log: log}, steps)
 }
 
 // run executes the fail-closed boot sequence in the exact order §(d) fixes:
-// (1) API filesystems, (2) read the vsock port from the now-readable kernel
-// cmdline, (3) networking, (4) virtio-fs workspace, (5) serve the vsock Health
-// handshake, (6) idle inside serve until ctx is cancelled. Any step error
-// aborts the sequence before the next one runs, so a failing provisioner never
-// reaches the mount and a failing mount never reaches the server — Health is
-// served only after net and mount both succeed. The cmdline read is inside the
-// sequence, after the API mount, because /proc is not readable before it.
+// (1) API filesystems, (2) read the vsock port + optional boot nonce from the
+// now-readable kernel cmdline, (3) networking, (4) virtio-fs workspace,
+// (5) serve the vsock GuestControl surface, (6) idle inside serve until ctx is
+// cancelled — by a Unix signal (V2a path) or an RPC Stop (§(d)). Any step error
+// aborts the sequence before the next one runs, so Health is served only after
+// net and mount both succeed. When serve returns because an RPC Stop cancelled
+// it, run ends in reboot(RB_POWER_OFF): a bare PID-1 exit panics the kernel,
+// which the VMM never observes as a guest exit.
 func run(ctx context.Context, cfg config, steps bootSteps) error {
+	// log is the guestd logger threaded from Run; a nil (test-constructed config)
+	// falls back to the default handler, which writes to os.Stderr (ttyS0) — the
+	// same sink the production logger uses, so a boot test needs no logger.
+	log := cfg.log
+	if log == nil {
+		log = slog.Default()
+	}
 	if err := steps.mountAPIFilesystems(); err != nil {
 		return fmt.Errorf("mounting API filesystems: %w", err)
 	}
@@ -109,7 +128,10 @@ func run(ctx context.Context, cfg config, steps bootSteps) error {
 	if err != nil {
 		return err
 	}
-
+	bootNonce, err := parseBootNonce(string(cmdline))
+	if err != nil {
+		return err
+	}
 	if err := steps.net.Provision(ctx); err != nil {
 		return fmt.Errorf("provisioning network: %w", err)
 	}
@@ -118,12 +140,41 @@ func run(ctx context.Context, cfg config, steps bootSteps) error {
 	}
 
 	// Both bringup steps passed, so the served state is unconditionally true —
-	// a successful handshake is the proof of that. If the sequence ever grew a
-	// step that could serve degraded state, these would reflect it.
-	svc := &healthService{
+	// a successful handshake is the proof of that. The supervisor starts in
+	// stateReady (Health answers, exec refused until Provision opens the gate).
+	serveCtx, stopServing := context.WithCancel(ctx)
+	defer stopServing()
+	svc := &supervisor{
 		version:          cfg.guestdVersion,
 		netProvisioned:   true,
 		workspaceMounted: true,
+		bootNonce:        bootNonce,
+		newCredential:    linuxCredential,
+		stopServing:      stopServing,
+		state:            stateReady,
+		execs:            make(map[string]*childExec),
+	}
+	serveErr := steps.serve(serveCtx, port, svc)
+
+	// An RPC-driven Stop (Signal("", ...)) cancels serveCtx from inside the
+	// supervisor; ctx (the process signal context) is still live. In that case
+	// the guest is going down UNCONDITIONALLY, so guestd must power the guest
+	// off explicitly (§(d)) — a PID-1 exit would panic the kernel. The
+	// power-off is gated on rpcStop alone, NOT on a clean serveErr: a graceful
+	// drain that overran its deadline (e.g. a child that ignored SIGTERM)
+	// returns a non-nil Shutdown error, but the guest must STILL power off so
+	// the VMM observes a real shutdown within the host's timeout rather than
+	// burning the full timeout to a hard kill. A serve fault or a Unix-signal
+	// cancel (rpcStop false) returns as before, letting main exit and the host
+	// observe the dial failure.
+	svc.mu.Lock()
+	rpcStop := svc.rpcStop
+	svc.mu.Unlock()
+	if rpcStop {
+		if serveErr != nil {
+			log.Error("guest serve drain returned an error on RPC stop; powering off anyway", "error", serveErr)
+		}
+		return steps.powerOff()
 	}
-	return steps.serve(ctx, port, svc)
+	return serveErr
 }