diff --git a/go.mod b/go.mod index 38be10e9..7da704bd 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( golang.org/x/net v0.54.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/text v0.39.0 // indirect modernc.org/libc v1.72.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index a9fef723..4b1ca23d 100644 --- a/go.sum +++ b/go.sum @@ -152,22 +152,22 @@ golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/executor/executor.go b/internal/executor/executor.go index dd639402..972c9cef 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -2899,6 +2899,14 @@ func createTmuxWindow(daemonSession, windowName, workDir, script, allowedProject return "", fmt.Errorf("security: refusing to create tmux window with invalid workDir: %s", workDir) } + // Consult system memory pressure before adding another agent session. Warns by + // default and only defers when TY_MEMORY_GUARD=block (see memoryguard.go). + if note, gerr := guardMemoryForSpawn(taskID); gerr != nil { + return "", gerr + } else if note != "" { + log.Warn("memory guard: "+note, "task", taskID) + } + // Serialize check-then-create with the TUI/API spawn path (EnsureTaskWindow). // Best-effort on timeout so a wedged holder can't block the daemon forever. if release, lerr := executorlock.AcquireSpawn(executorSpawnLockDir(), taskID, spawnLockTimeout); lerr == nil { diff --git a/internal/executor/memoryguard.go b/internal/executor/memoryguard.go new file mode 100644 index 00000000..cad198d8 --- /dev/null +++ b/internal/executor/memoryguard.go @@ -0,0 +1,116 @@ +package executor + +// Memory admission guard for executor spawns. +// +// Why this exists: nothing in ty bounded how many agent sessions could be live at +// once. Each session is a claude process plus its MCP servers (~100-150MB resident, +// considerably more in footprint once macOS compresses the cold pages), so a queue +// that fans out freely can walk a workstation into swap thrashing — at which point +// every session, dev server and test run on the box gets slower, including the ones +// already doing useful work. +// +// Deliberately NOT a concurrency cap. A fixed "max N sessions" is a hidden wall: +// the right number depends entirely on how much RAM the machine has and what else +// is running, and a user who legitimately wants dozens of concurrent agents should +// not be told "no" by an arbitrary constant. So this gates on the machine's ACTUAL +// memory pressure instead. With headroom, spawn as many as you like; the guard only +// has an opinion once the kernel says memory is genuinely short. +// +// Default mode is "warn": log loudly, never block. That keeps behaviour unchanged +// for everyone else while making the condition visible. Set TY_MEMORY_GUARD=block +// to have spawns deferred under pressure instead. +// +// TY_MEMORY_GUARD=off|warn|block (default: warn) +// TY_MEMORY_GUARD_MIN_FREE_PCT=<0-100> (default: 20) +// +// The signal is "percent of memory still available", read per-platform by +// systemFreeMemoryPct (see memoryguard_darwin.go / memoryguard_linux.go). On any +// platform where it can't be read the guard is inert. +// +// Threshold semantics are the same everywhere: the fraction of memory still +// available, 0-100. On macOS that maps to Activity Monitor's pressure zones (green +// 100-50, yellow 50-30, red 30-0), so the default of 20 fires only well into the +// red — when the machine is already hurting, not merely busy. + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" +) + +type memoryGuardMode int + +const ( + memoryGuardOff memoryGuardMode = iota + memoryGuardWarn + memoryGuardBlock +) + +// defaultMemoryGuardMinFreePct is the free-memory percentage below which the guard +// considers the machine to be under real pressure. 20 sits inside Activity Monitor's +// red zone (30-0). +const defaultMemoryGuardMinFreePct = 20 + +// ErrMemoryPressure is returned by guardMemoryForSpawn in block mode when the +// machine is below the free-memory threshold. Callers should treat it as "try this +// task again shortly", not as a task failure. +var ErrMemoryPressure = errors.New("executor: deferring spawn, system memory pressure") + +func memoryGuardModeFromEnv() memoryGuardMode { + switch strings.ToLower(strings.TrimSpace(os.Getenv("TY_MEMORY_GUARD"))) { + case "off", "0", "false", "disabled": + return memoryGuardOff + case "block", "defer", "enforce": + return memoryGuardBlock + default: + // Unset or unrecognised: warn. Never silently block. + return memoryGuardWarn + } +} + +func memoryGuardMinFreePct() int { + raw := strings.TrimSpace(os.Getenv("TY_MEMORY_GUARD_MIN_FREE_PCT")) + if raw == "" { + return defaultMemoryGuardMinFreePct + } + n, err := strconv.Atoi(raw) + if err != nil || n < 0 || n > 100 { + return defaultMemoryGuardMinFreePct + } + return n +} + +// systemFreeMemoryPct is implemented per platform; see memoryguard_darwin.go, +// memoryguard_linux.go and memoryguard_unsupported.go. It returns ok=false whenever +// the signal can't be read, which makes the guard inert rather than guessing. + +// guardMemoryForSpawn consults system memory pressure before an executor spawn. +// +// It returns a human-readable note whenever the machine is under pressure (empty +// string otherwise) so the caller can surface it with whatever logger it has, and a +// non-nil error ONLY in block mode. Callers must always spawn when err == nil, even +// if note is non-empty — that is the whole point of the default warn mode. +func guardMemoryForSpawn(taskID int64) (note string, err error) { + mode := memoryGuardModeFromEnv() + if mode == memoryGuardOff { + return "", nil + } + + freePct, ok := systemFreeMemoryPct() + if !ok { + return "", nil // no signal, no opinion + } + minFree := memoryGuardMinFreePct() + if freePct >= minFree { + return "", nil + } + + if mode == memoryGuardBlock { + return "", fmt.Errorf("%w: %d%% memory free (threshold %d%%), task %d — set TY_MEMORY_GUARD=off or lower TY_MEMORY_GUARD_MIN_FREE_PCT to spawn anyway", + ErrMemoryPressure, freePct, minFree, taskID) + } + return fmt.Sprintf("system memory is low (%d%% free, threshold %d%%): spawning anyway; set TY_MEMORY_GUARD=block to defer spawns under pressure", + freePct, minFree), nil +} diff --git a/internal/executor/memoryguard_darwin.go b/internal/executor/memoryguard_darwin.go new file mode 100644 index 00000000..72c8dc16 --- /dev/null +++ b/internal/executor/memoryguard_darwin.go @@ -0,0 +1,39 @@ +//go:build darwin + +package executor + +import ( + "context" + "os/exec" + "strconv" + "strings" + "time" +) + +// memoryGuardSysctlTimeout bounds the sysctl call so a wedged exec can never delay +// a spawn. On timeout the guard reports "unknown" and stays out of the way. +const memoryGuardSysctlTimeout = 2 * time.Second + +// systemFreeMemoryPct reads kern.memorystatus_level, the kernel's own +// percent-of-memory-still-free figure. It is what Activity Monitor's memory +// pressure display is derived from, so the number a user sees there and the number +// this guard acts on are the same one. +// +// Note this is deliberately NOT "free RAM" in the naive sense: macOS aggressively +// fills RAM with cache and compressed pages, so free-page counts read alarmingly low +// on a perfectly healthy machine. memorystatus_level already accounts for what the +// kernel can reclaim, which is why it's the right signal here. +func systemFreeMemoryPct() (pct int, ok bool) { + ctx, cancel := context.WithTimeout(context.Background(), memoryGuardSysctlTimeout) + defer cancel() + + out, err := exec.CommandContext(ctx, "sysctl", "-n", "kern.memorystatus_level").Output() + if err != nil { + return 0, false + } + n, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil || n < 0 || n > 100 { + return 0, false + } + return n, true +} diff --git a/internal/executor/memoryguard_linux.go b/internal/executor/memoryguard_linux.go new file mode 100644 index 00000000..55ce1582 --- /dev/null +++ b/internal/executor/memoryguard_linux.go @@ -0,0 +1,45 @@ +//go:build linux + +package executor + +import "os" + +// cgroup v2 paths. Inside a container the container's own cgroup is mounted at the +// hierarchy root, so these read that container's limit. On a bare host the root +// cgroup has no memory.max and the read simply fails, falling through to +// /proc/meminfo — which is what we want there anyway. +const ( + cgroupV2MemoryMax = "/sys/fs/cgroup/memory.max" + cgroupV2MemoryCurrent = "/sys/fs/cgroup/memory.current" + cgroupV2MemoryStat = "/sys/fs/cgroup/memory.stat" + procMeminfo = "/proc/meminfo" +) + +// systemFreeMemoryPct returns percent-of-memory-available on Linux. +// +// cgroup v2 is consulted first so a containerised agent server is measured against +// its own memory limit rather than the host's total RAM — otherwise a 2GB container +// on a 64GB host would look like it had endless headroom right up until the OOM +// killer fired. Falls back to /proc/meminfo when there is no cgroup limit. +// +// Deliberately not using PSI (/proc/pressure/memory): it is the better *pressure* +// signal, but it reports stall time rather than a fraction of memory, so it can't +// share TY_MEMORY_GUARD_MIN_FREE_PCT's meaning with the macOS path. Keeping one +// threshold that means the same thing on every platform is worth more here than a +// slightly sharper Linux signal. +func systemFreeMemoryPct() (pct int, ok bool) { + maxRaw, errMax := os.ReadFile(cgroupV2MemoryMax) + currentRaw, errCur := os.ReadFile(cgroupV2MemoryCurrent) + if errMax == nil && errCur == nil { + statRaw, _ := os.ReadFile(cgroupV2MemoryStat) // optional; absent just means no cache adjustment + if pct, ok := parseCgroupFreePct(string(maxRaw), string(currentRaw), string(statRaw)); ok { + return pct, true + } + } + + data, err := os.ReadFile(procMeminfo) + if err != nil { + return 0, false + } + return parseMeminfoFreePct(string(data)) +} diff --git a/internal/executor/memoryguard_procparse.go b/internal/executor/memoryguard_procparse.go new file mode 100644 index 00000000..8fce3c03 --- /dev/null +++ b/internal/executor/memoryguard_procparse.go @@ -0,0 +1,115 @@ +package executor + +// Pure parsers for the Linux memory signals, deliberately kept free of build tags +// and of any file I/O so they can be unit-tested on any platform (including the +// macOS laptops where this is usually developed). The Linux-only file reading that +// feeds them lives in memoryguard_linux.go. + +import ( + "strconv" + "strings" +) + +// parseMeminfoFreePct computes percent-of-memory-available from /proc/meminfo. +// +// MemAvailable is the right numerator: it is the kernel's own estimate of memory +// obtainable without swapping, already accounting for reclaimable page cache and +// slab. Using MemFree instead would report single-digit percentages on a healthy +// Linux box that is simply using its RAM for cache — and in block mode that would +// wedge an agent fleet for no reason. +// +// MemAvailable has been present since Linux 3.14 (2014). Older kernels fall back to +// MemFree + Buffers + Cached, the approximation MemAvailable itself replaced. +func parseMeminfoFreePct(data string) (pct int, ok bool) { + var total, available, free, buffers, cached int64 + var haveAvailable bool + + for _, line := range strings.Split(data, "\n") { + key, valueField, found := strings.Cut(line, ":") + if !found { + continue + } + fields := strings.Fields(valueField) + if len(fields) == 0 { + continue + } + v, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + continue + } + switch key { + case "MemTotal": + total = v + case "MemAvailable": + available, haveAvailable = v, true + case "MemFree": + free = v + case "Buffers": + buffers = v + case "Cached": // must not match "SwapCached": strings.Cut keys are exact + cached = v + } + } + + if total <= 0 { + return 0, false + } + if !haveAvailable { + available = free + buffers + cached + } + return clampPct(available * 100 / total), true +} + +// parseCgroupFreePct computes percent-of-memory-available from cgroup v2 files, so +// a containerised agent server is measured against its own limit rather than the +// host's RAM. Returns ok=false when the cgroup is unlimited ("max"), which is the +// normal case on a bare VM — the caller then falls back to /proc/meminfo. +// +// memory.current counts page cache, which is reclaimable, so subtracting +// inactive_file from memory.stat is what keeps a cache-heavy container from looking +// permanently starved. Without that subtraction a long-running container would sit +// at ~0% "free" forever and, in block mode, stop spawning entirely. +func parseCgroupFreePct(maxRaw, currentRaw, statRaw string) (pct int, ok bool) { + maxRaw = strings.TrimSpace(maxRaw) + if maxRaw == "" || maxRaw == "max" { + return 0, false // no limit set; not a meaningful denominator + } + limit, err := strconv.ParseInt(maxRaw, 10, 64) + if err != nil || limit <= 0 { + return 0, false + } + current, err := strconv.ParseInt(strings.TrimSpace(currentRaw), 10, 64) + if err != nil || current < 0 { + return 0, false + } + + // Treat reclaimable file cache as available. + var inactiveFile int64 + for _, line := range strings.Split(statRaw, "\n") { + if name, value, found := strings.Cut(strings.TrimSpace(line), " "); found && name == "inactive_file" { + if v, err := strconv.ParseInt(value, 10, 64); err == nil && v >= 0 { + inactiveFile = v + } + break + } + } + + used := current - inactiveFile + if used < 0 { + used = 0 + } + if used > limit { + used = limit + } + return clampPct((limit - used) * 100 / limit), true +} + +func clampPct(v int64) int { + if v < 0 { + return 0 + } + if v > 100 { + return 100 + } + return int(v) +} diff --git a/internal/executor/memoryguard_procparse_test.go b/internal/executor/memoryguard_procparse_test.go new file mode 100644 index 00000000..3d0a979c --- /dev/null +++ b/internal/executor/memoryguard_procparse_test.go @@ -0,0 +1,138 @@ +package executor + +import "testing" + +// Real shape of /proc/meminfo, trimmed. 32GB box with most of RAM in cache. +const sampleMeminfo = `MemTotal: 32819104 kB +MemFree: 412300 kB +MemAvailable: 24117248 kB +Buffers: 204800 kB +Cached: 23068672 kB +SwapCached: 12345 kB +Active: 6291456 kB +SwapTotal: 2097148 kB +SwapFree: 2097148 kB +` + +func TestParseMeminfoFreePct(t *testing.T) { + pct, ok := parseMeminfoFreePct(sampleMeminfo) + if !ok { + t.Fatal("expected parse to succeed") + } + // 24117248 / 32819104 = 73% + if pct != 73 { + t.Fatalf("got %d%%, want 73%%", pct) + } +} + +// The whole point of preferring MemAvailable: MemFree alone would read 1% here and, +// in block mode, wedge a perfectly healthy box that is merely using RAM for cache. +func TestParseMeminfoPrefersAvailableOverFree(t *testing.T) { + pct, _ := parseMeminfoFreePct(sampleMeminfo) + if pct < 50 { + t.Fatalf("cache-heavy but healthy box reported %d%% free; MemAvailable is not being used", pct) + } +} + +// Pre-3.14 kernels have no MemAvailable; fall back to MemFree+Buffers+Cached. +func TestParseMeminfoFallbackWithoutMemAvailable(t *testing.T) { + old := `MemTotal: 1000000 kB +MemFree: 100000 kB +Buffers: 50000 kB +Cached: 350000 kB +` + pct, ok := parseMeminfoFreePct(old) + if !ok { + t.Fatal("expected parse to succeed") + } + if pct != 50 { // (100000+50000+350000)/1000000 + t.Fatalf("got %d%%, want 50%%", pct) + } +} + +// "Cached" must not be satisfied by "SwapCached" — an exact key match, not a prefix. +func TestParseMeminfoDoesNotConfuseSwapCached(t *testing.T) { + in := `MemTotal: 1000000 kB +MemFree: 100000 kB +Buffers: 0 kB +SwapCached: 900000 kB +Cached: 100000 kB +` + pct, ok := parseMeminfoFreePct(in) + if !ok { + t.Fatal("expected parse to succeed") + } + if pct != 20 { // (100000+0+100000)/1000000 — SwapCached must be ignored + t.Fatalf("got %d%%, want 20%%; SwapCached leaked into Cached", pct) + } +} + +func TestParseMeminfoRejectsGarbage(t *testing.T) { + for _, in := range []string{"", "not meminfo at all", "MemTotal: 0 kB\n"} { + if _, ok := parseMeminfoFreePct(in); ok { + t.Errorf("expected failure for %q", in) + } + } +} + +func TestParseCgroupFreePct(t *testing.T) { + // 2GB limit, 1GB charged, none of it reclaimable cache -> 50% free. + pct, ok := parseCgroupFreePct("2147483648", "1073741824", "anon 1073741824\ninactive_file 0\n") + if !ok || pct != 50 { + t.Fatalf("got %d%% ok=%v, want 50%% ok=true", pct, ok) + } +} + +// Page cache is reclaimable: a container whose charge is almost all cache is NOT +// under pressure. Without the inactive_file subtraction this reads 0% free and +// block mode would stop spawning permanently. +func TestParseCgroupTreatsPageCacheAsAvailable(t *testing.T) { + pct, ok := parseCgroupFreePct("2147483648", "2147483648", "anon 107374182\ninactive_file 2040109466\n") + if !ok { + t.Fatal("expected parse to succeed") + } + if pct < 90 { + t.Fatalf("cache-heavy container reported %d%% free; inactive_file not subtracted", pct) + } +} + +// An unlimited cgroup is not a meaningful denominator; caller must fall back. +func TestParseCgroupUnlimitedFallsBack(t *testing.T) { + for _, raw := range []string{"max", "max\n", "", " "} { + if _, ok := parseCgroupFreePct(raw, "1073741824", ""); ok { + t.Errorf("expected ok=false for memory.max=%q", raw) + } + } +} + +func TestParseCgroupRejectsGarbage(t *testing.T) { + cases := [][2]string{ + {"notanumber", "123"}, + {"0", "123"}, + {"-1", "123"}, + {"2147483648", "notanumber"}, + {"2147483648", "-5"}, + } + for _, c := range cases { + if _, ok := parseCgroupFreePct(c[0], c[1], ""); ok { + t.Errorf("expected ok=false for max=%q current=%q", c[0], c[1]) + } + } +} + +// Usage above the limit (transiently possible) must clamp to 0, never go negative. +func TestParseCgroupClampsOverLimit(t *testing.T) { + pct, ok := parseCgroupFreePct("1000", "5000", "") + if !ok || pct != 0 { + t.Fatalf("got %d%% ok=%v, want 0%% ok=true", pct, ok) + } +} + +func TestClampPct(t *testing.T) { + cases := map[int64]int{-5: 0, 0: 0, 42: 42, 100: 100, 101: 100, 1 << 40: 100} + for in, want := range cases { + if got := clampPct(in); got != want { + t.Errorf("clampPct(%d) = %d, want %d", in, got, want) + } + } +} diff --git a/internal/executor/memoryguard_test.go b/internal/executor/memoryguard_test.go new file mode 100644 index 00000000..94a5f096 --- /dev/null +++ b/internal/executor/memoryguard_test.go @@ -0,0 +1,99 @@ +package executor + +import ( + "errors" + "testing" +) + +func TestMemoryGuardModeFromEnv(t *testing.T) { + cases := map[string]memoryGuardMode{ + "": memoryGuardWarn, // unset must never block + "warn": memoryGuardWarn, + "nonsense": memoryGuardWarn, // unrecognised must never block + "off": memoryGuardOff, + "0": memoryGuardOff, + "false": memoryGuardOff, + "disabled": memoryGuardOff, + "block": memoryGuardBlock, + "BLOCK": memoryGuardBlock, + " block ": memoryGuardBlock, + "defer": memoryGuardBlock, + "enforce": memoryGuardBlock, + } + for in, want := range cases { + t.Setenv("TY_MEMORY_GUARD", in) + if got := memoryGuardModeFromEnv(); got != want { + t.Errorf("TY_MEMORY_GUARD=%q: got mode %d, want %d", in, got, want) + } + } +} + +func TestMemoryGuardMinFreePct(t *testing.T) { + cases := map[string]int{ + "": defaultMemoryGuardMinFreePct, + "35": 35, + "0": 0, + "100": 100, + "-1": defaultMemoryGuardMinFreePct, // out of range falls back + "101": defaultMemoryGuardMinFreePct, + "junk": defaultMemoryGuardMinFreePct, + } + for in, want := range cases { + t.Setenv("TY_MEMORY_GUARD_MIN_FREE_PCT", in) + if got := memoryGuardMinFreePct(); got != want { + t.Errorf("TY_MEMORY_GUARD_MIN_FREE_PCT=%q: got %d, want %d", in, got, want) + } + } +} + +// Off mode must be inert regardless of how low the threshold makes the machine look. +func TestGuardMemoryForSpawnOffIsInert(t *testing.T) { + t.Setenv("TY_MEMORY_GUARD", "off") + t.Setenv("TY_MEMORY_GUARD_MIN_FREE_PCT", "100") + note, err := guardMemoryForSpawn(1) + if err != nil || note != "" { + t.Fatalf("off mode must be inert, got note=%q err=%v", note, err) + } +} + +// The critical safety property: the default (unset) mode never returns an error, so +// no ty user hits a spawn wall they didn't opt into — even with the threshold pinned +// so high that the machine is guaranteed to be "under pressure". +func TestGuardMemoryForSpawnDefaultNeverBlocks(t *testing.T) { + t.Setenv("TY_MEMORY_GUARD", "") + t.Setenv("TY_MEMORY_GUARD_MIN_FREE_PCT", "100") + _, err := guardMemoryForSpawn(1) + if err != nil { + t.Fatalf("default mode must never block, got err=%v", err) + } +} + +// Block mode with a 100% threshold blocks iff the pressure signal is readable. +// Skips rather than fails where the signal is unavailable (non-Darwin CI). +func TestGuardMemoryForSpawnBlockMode(t *testing.T) { + if _, ok := systemFreeMemoryPct(); !ok { + t.Skip("kern.memorystatus_level unavailable on this platform") + } + t.Setenv("TY_MEMORY_GUARD", "block") + t.Setenv("TY_MEMORY_GUARD_MIN_FREE_PCT", "100") + if _, err := guardMemoryForSpawn(42); !errors.Is(err, ErrMemoryPressure) { + t.Fatalf("want ErrMemoryPressure, got %v", err) + } + + // With a 0% threshold nothing can be below it, so block mode must allow the spawn. + t.Setenv("TY_MEMORY_GUARD_MIN_FREE_PCT", "0") + if note, err := guardMemoryForSpawn(42); err != nil || note != "" { + t.Fatalf("threshold 0 must always allow, got note=%q err=%v", note, err) + } +} + +func TestSystemFreeMemoryPctInRange(t *testing.T) { + pct, ok := systemFreeMemoryPct() + if !ok { + t.Skip("kern.memorystatus_level unavailable on this platform") + } + if pct < 0 || pct > 100 { + t.Fatalf("free pct out of range: %d", pct) + } + t.Logf("kern.memorystatus_level = %d%% free", pct) +} diff --git a/internal/executor/memoryguard_unsupported.go b/internal/executor/memoryguard_unsupported.go new file mode 100644 index 00000000..5f4017e5 --- /dev/null +++ b/internal/executor/memoryguard_unsupported.go @@ -0,0 +1,11 @@ +//go:build !darwin && !linux + +package executor + +// systemFreeMemoryPct has no implementation on this platform, so the guard is +// inert: guardMemoryForSpawn returns "no opinion" and spawns proceed exactly as +// they did before the guard existed. Failing open is the correct default — a guard +// that can't measure anything must never be the reason a task doesn't start. +func systemFreeMemoryPct() (pct int, ok bool) { + return 0, false +} diff --git a/internal/executor/session.go b/internal/executor/session.go index 469b60fa..0f30eaa7 100644 --- a/internal/executor/session.go +++ b/internal/executor/session.go @@ -60,6 +60,14 @@ func (e *Executor) EnsureTaskWindow(ctx context.Context, task *db.Task, sessionI return target, false, nil } + // Consult system memory pressure before adding another agent session. Warns by + // default and only defers when TY_MEMORY_GUARD=block (see memoryguard.go). + if note, err := guardMemoryForSpawn(task.ID); err != nil { + return "", false, err + } else if note != "" { + e.logger.Warn("memory guard: "+note, "task", task.ID) + } + // Serialize the check-then-create against the daemon executor and any other // caller so two spawners can't both observe "no window yet" and each create // one — the double-spawn that leaves two executor sessions in one worktree with