Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 49 additions & 5 deletions go/cmd/compass-runner/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"log/slog"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
Expand Down Expand Up @@ -216,7 +217,8 @@ func setupOtel(ctx context.Context) (shutdown func(), err error) {
// backendFlags holds the runtime-backend selection flags, registered on the
// default flag set before flag.Parse and resolved into a runtime after it.
type backendFlags struct {
backend, vmm, virtiofsd, kernel, rootfs *string
backend, vmm, virtiofsd, kernel, rootfs, initrd, runRoot *string
cpus, memoryMB *int
}

// registerBackendFlags declares the backend-selection flags. Call before
Expand All @@ -234,19 +236,41 @@ func registerBackendFlags() backendFlags {
"Path to the guest kernel image (microvm backend). Defaults to $COMPASS_MICROVM_KERNEL."),
rootfs: flag.String("microvm-rootfs", "",
"Path to the guest rootfs image (microvm backend). Defaults to $COMPASS_MICROVM_ROOTFS."),
initrd: flag.String("microvm-initrd", "",
"Path to the guest initramfs image (microvm backend). Defaults to $COMPASS_MICROVM_INITRD."),
runRoot: flag.String("microvm-runroot", "",
"Root dir for per-session microVM runtime dirs (microvm backend). Defaults to $COMPASS_MICROVM_RUNROOT."),
cpus: flag.Int("microvm-cpus", 0,
"Default vCPU count per session guest (microvm backend); 0 leaves the VMM default. "+
"Defaults to $COMPASS_MICROVM_CPUS."),
memoryMB: flag.Int("microvm-memory-mb", 0,
"Default guest RAM in MiB per session (microvm backend); 0 leaves the VMM default. "+
"Defaults to $COMPASS_MICROVM_MEMORY_MB."),
}
}

// selectEngine resolves the configured runtime backend from the parsed flags
// and their environment fallbacks.
func (f backendFlags) selectEngine() (runtime.ContainerRuntime, error) {
cpus, err := intOrEnv(*f.cpus, "COMPASS_MICROVM_CPUS")
if err != nil {
return nil, err
}
memoryMB, err := intOrEnv(*f.memoryMB, "COMPASS_MICROVM_MEMORY_MB")
if err != nil {
return nil, err
}
return runtime.SelectBackend(runtime.BackendConfig{
Backend: orEnv(*f.backend, "COMPASS_RUNTIME_BACKEND"),
MicroVM: runtime.MicroVMConfig{
VMMPath: orEnv(*f.vmm, "COMPASS_MICROVM_VMM"),
VirtiofsdPath: orEnv(*f.virtiofsd, "COMPASS_MICROVM_VIRTIOFSD"),
KernelImage: orEnv(*f.kernel, "COMPASS_MICROVM_KERNEL"),
RootfsImage: orEnv(*f.rootfs, "COMPASS_MICROVM_ROOTFS"),
VMMPath: orEnv(*f.vmm, "COMPASS_MICROVM_VMM"),
VirtiofsdPath: orEnv(*f.virtiofsd, "COMPASS_MICROVM_VIRTIOFSD"),
KernelImage: orEnv(*f.kernel, "COMPASS_MICROVM_KERNEL"),
RootfsImage: orEnv(*f.rootfs, "COMPASS_MICROVM_ROOTFS"),
InitrdImage: orEnv(*f.initrd, "COMPASS_MICROVM_INITRD"),
RunRoot: orEnv(*f.runRoot, "COMPASS_MICROVM_RUNROOT"),
DefaultCPUs: cpus,
DefaultMemoryMB: memoryMB,
},
})
}
Expand All @@ -259,6 +283,26 @@ func orEnv(flagVal, envKey string) string {
return os.Getenv(envKey)
}

// intOrEnv returns flagVal when non-zero, else the named environment variable
// parsed as an int. An empty env var is 0 (unset — the config treats 0 as
// "leave the VMM default"); a present-but-non-numeric env var is an error
// naming the offending variable and value so a misconfiguration surfaces at
// startup rather than as a zero silently swallowing a typo.
func intOrEnv(flagVal int, envKey string) (int, error) {
if flagVal != 0 {
return flagVal, nil
}
raw := os.Getenv(envKey)
if raw == "" {
return 0, nil
}
parsed, err := strconv.Atoi(raw)
if err != nil {
return 0, fmt.Errorf("$%s=%q is not an integer: %w", envKey, raw, err)
}
return parsed, nil
}

// parseEgress parses the comma-separated allowlist into a validated EgressPolicy.
// An empty list is a valid default-deny policy (no host reachable).
func parseEgress(csv string) (runtime.EgressPolicy, error) {
Expand Down
98 changes: 95 additions & 3 deletions go/internal/guestd/boot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"context"
"errors"
"slices"
"syscall"
"testing"
)

Expand Down Expand Up @@ -59,8 +60,8 @@ func (f *fakeMount) Mount() error {
// terminal step, exactly as production does. served holds the service serve was
// handed, or nil if serve was never reached. cmdline is the kernel command line
// the readCmdline step returns; cmdlineErr, when non-nil, makes that step fail.
func newSteps(rec *recorder, apiErr, netErr, mountErr, cmdlineErr error, cmdline string) (bootSteps, **healthService, chan struct{}) {
served := new(*healthService)
func newSteps(rec *recorder, apiErr, netErr, mountErr, cmdlineErr error, cmdline string) (bootSteps, **supervisor, chan struct{}) {
served := new(*supervisor)
reached := make(chan struct{})
steps := bootSteps{
mountAPIFilesystems: func() error {
Expand All @@ -73,13 +74,17 @@ func newSteps(rec *recorder, apiErr, netErr, mountErr, cmdlineErr error, cmdline
},
net: &fakeNet{rec: rec, err: netErr},
workspace: &fakeMount{rec: rec, err: mountErr},
serve: func(ctx context.Context, _ uint32, svc *healthService) error {
serve: func(ctx context.Context, _ uint32, svc *supervisor) error {
rec.mark("serve")
*served = svc
close(reached)
<-ctx.Done()
return ctx.Err()
},
powerOff: func() error {
rec.mark("poweroff")
return nil
},
}
return steps, served, reached
}
Expand Down Expand Up @@ -279,3 +284,90 @@ func TestBootFailsClosedOnBadCmdline(t *testing.T) {
t.Fatal("a health service was constructed despite a bad cmdline")
}
}

// TestBootPowersOffOnRPCStop drives the full run() poweroff gate: a serve that
// simulates an RPC Stop (flags rpcStop + cancels serving via initiateStop) then
// drains clean must end in reboot(RB_POWER_OFF), since a bare PID-1 exit panics
// the kernel (§(d)). Exercises the integration TestRPCStopCancels... asserts in
// isolation.
func TestBootPowersOffOnRPCStop(t *testing.T) {
rec := &recorder{}
steps := bootSteps{
mountAPIFilesystems: func() error { rec.mark("api"); return nil },
readCmdline: func() ([]byte, error) { rec.mark("cmdline"); return []byte("compass.vsock_port=1024"), nil },
net: &fakeNet{rec: rec},
workspace: &fakeMount{rec: rec},
serve: func(_ context.Context, _ uint32, svc *supervisor) error {
rec.mark("serve")
svc.initiateStop(syscall.SIGTERM) // RPC Stop: sets rpcStop, cancels serving
return nil // clean drain
},
powerOff: func() error { rec.mark("poweroff"); return nil },
}
if err := run(t.Context(), config{}, steps); err != nil {
t.Fatalf("run after a clean RPC stop = %v, want nil", err)
}
if !rec.ran("poweroff") {
t.Fatalf("power-off did not run after an RPC stop; order was %v", rec.steps)
}
}

// TestBootPowersOffOnRPCStopDespiteDrainError is the reliability contract: an
// RPC Stop whose graceful drain overran (serve returns a non-nil error, e.g. a
// child that ignored SIGTERM held Shutdown past its deadline) must STILL power
// off, so the VMM observes a real guest shutdown within the host's timeout
// instead of burning it to a hard kill. The poweroff is gated on rpcStop alone,
// not on a clean serveErr.
func TestBootPowersOffOnRPCStopDespiteDrainError(t *testing.T) {
rec := &recorder{}
drainErr := errors.New("shutdown deadline exceeded")
steps := bootSteps{
mountAPIFilesystems: func() error { rec.mark("api"); return nil },
readCmdline: func() ([]byte, error) { rec.mark("cmdline"); return []byte("compass.vsock_port=1024"), nil },
net: &fakeNet{rec: rec},
workspace: &fakeMount{rec: rec},
serve: func(_ context.Context, _ uint32, svc *supervisor) error {
rec.mark("serve")
svc.initiateStop(syscall.SIGTERM)
return drainErr // drain overran
},
powerOff: func() error { rec.mark("poweroff"); return nil },
}
// run returns powerOff()'s result (nil), NOT the drain error — the guest
// powered off, so main never falls through to a bare PID-1 exit.
if err := run(t.Context(), config{}, steps); err != nil {
t.Fatalf("run after an RPC stop with a drain error = %v, want nil (powered off)", err)
}
if !rec.ran("poweroff") {
t.Fatalf("power-off was skipped on a drain error during an RPC stop; order was %v", rec.steps)
}
}

// TestBootDoesNotPowerOffOnSignalCancel is the negative gate: a Unix-signal
// shutdown (ctx cancelled, no RPC Stop) must NOT power off — rpcStop is false,
// so run returns the serve error and lets main exit (the V2a path), and the
// host observes the dial failure.
func TestBootDoesNotPowerOffOnSignalCancel(t *testing.T) {
rec := &recorder{}
steps := bootSteps{
mountAPIFilesystems: func() error { rec.mark("api"); return nil },
readCmdline: func() ([]byte, error) { rec.mark("cmdline"); return []byte("compass.vsock_port=1024"), nil },
net: &fakeNet{rec: rec},
workspace: &fakeMount{rec: rec},
serve: func(ctx context.Context, _ uint32, _ *supervisor) error {
rec.mark("serve")
<-ctx.Done()
return ctx.Err()
},
powerOff: func() error { rec.mark("poweroff"); return nil },
}
ctx, cancel := context.WithCancel(t.Context())
cancel() // simulate a Unix-signal shutdown: serve returns on ctx, rpcStop false
err := run(ctx, config{}, steps)
if !errors.Is(err, context.Canceled) {
t.Fatalf("run after a signal cancel = %v, want context.Canceled", err)
}
if rec.ran("poweroff") {
t.Fatalf("power-off ran on a plain signal cancel (rpcStop false); order was %v", rec.steps)
}
}
40 changes: 40 additions & 0 deletions go/internal/guestd/cmdline.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package guestd

import (
"encoding/hex"
"fmt"
"strconv"
"strings"
Expand Down Expand Up @@ -58,3 +59,42 @@ func parseVsockPort(procCmdline string) (uint32, error) {
}
return uint32(n), nil
}

// bootNonceKey is the kernel-cmdline parameter carrying the per-session boot
// nonce (§(e)) — a random hex value the host generates per session, passes on
// the cmdline beside compass.vsock_port, and expects guestd to echo in
// HealthResponse.boot_nonce. It binds the guest answering the handshake to THIS
// BootConfig (a liveness/identity check against a stale VMM on a recycled
// socket), not an authentication secret. It is OPTIONAL: a V2a-style cmdline
// carries no nonce, so an absent key echoes an empty nonce and Health still
// answers. A present-but-malformed value is a boot-config bug and fail-closes.
const bootNonceKey = "compass.boot_nonce"

// parseBootNonce extracts compass.boot_nonce=<hex> from a /proc/cmdline string,
// following the same last-occurrence-wins tokenisation as parseVsockPort. A
// missing key returns (nil, nil) — the nonce is optional hardening, not a
// fail-closed boot parameter. A present key with an empty or non-hex value is a
// malformed boot config and returns an error.
func parseBootNonce(procCmdline string) ([]byte, error) {
raw := ""
found := false
for tok := range strings.FieldsSeq(procCmdline) {
key, val, ok := strings.Cut(tok, "=")
if !ok || key != bootNonceKey {
continue
}
raw = val
found = true
}
if !found {
return nil, nil
}
if raw == "" {
return nil, fmt.Errorf("kernel cmdline %s has an empty value", bootNonceKey)
}
nonce, err := hex.DecodeString(raw)
if err != nil {
return nil, fmt.Errorf("kernel cmdline %s=%q is not valid hex: %w", bootNonceKey, raw, err)
}
return nonce, nil
}
71 changes: 70 additions & 1 deletion go/internal/guestd/cmdline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ package guestd
// the handshake without a valid non-zero vsock port, so every malformed cmdline
// is an error and only a well-formed compass.vsock_port=<n> yields a port.

import "testing"
import (
"bytes"
"testing"
)

func TestParseVsockPort(t *testing.T) {
tests := []struct {
Expand Down Expand Up @@ -107,3 +110,69 @@ func TestParseVsockPort(t *testing.T) {
})
}
}

// TestParseBootNonce defends the boot-nonce contract (§(e)): the nonce is
// OPTIONAL hardening, so an absent key is (nil, nil) and Health still answers;
// a present key must be valid hex; an empty or non-hex value is a malformed
// boot config and fail-closes.
func TestParseBootNonce(t *testing.T) {
tests := []struct {
name string
cmdline string
want []byte
wantErr bool
}{
{
name: "absent key echoes empty nonce",
cmdline: "console=ttyS0 compass.vsock_port=1024",
want: nil,
},
{
name: "valid hex nonce",
cmdline: "compass.vsock_port=1024 compass.boot_nonce=deadbeef",
want: []byte{0xde, 0xad, 0xbe, 0xef},
},
{
name: "trailing newline as /proc/cmdline yields",
cmdline: "compass.boot_nonce=00ff\n",
want: []byte{0x00, 0xff},
},
{
name: "last occurrence wins",
cmdline: "compass.boot_nonce=aa compass.boot_nonce=bb",
want: []byte{0xbb},
},
{
name: "empty value is an error",
cmdline: "compass.boot_nonce=",
wantErr: true,
},
{
name: "non-hex value is an error",
cmdline: "compass.boot_nonce=zzzz",
wantErr: true,
},
{
name: "odd-length hex is an error",
cmdline: "compass.boot_nonce=abc",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseBootNonce(tt.cmdline)
if tt.wantErr {
if err == nil {
t.Fatalf("parseBootNonce(%q) = %x, nil; want error", tt.cmdline, got)
}
return
}
if err != nil {
t.Fatalf("parseBootNonce(%q) unexpected error: %v", tt.cmdline, err)
}
if !bytes.Equal(got, tt.want) {
t.Fatalf("parseBootNonce(%q) = %x, want %x", tt.cmdline, got, tt.want)
}
})
}
}
Loading
Loading