From 75499667599e2d7986b4ce0511f04771cce0ba63 Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 17 Aug 2026 23:45:58 -0700 Subject: [PATCH 1/2] fix: default the monitor restart window to 1 minute A restart kept /monitor/workloads in alarm for 15 minutes after the pod had recovered, so a 1-minute poller re-notified on every check until the window expired. - MONITOR_RESTART_WINDOW_MIN default 15 -> 1 (config, .env.example, Helm values) - note in .env.example that the value should track the poller's interval A crash loop still alarms on its own via CrashLoopBackOff, so persistent failures stay visible; what the shorter window drops is a single restart that lands between two polls. Signed-off-by: vthwang --- .env.example | 5 +++-- helm/vtafarm-api/values.yaml | 2 +- internal/config/config.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 27ac9b1..c36e6d1 100644 --- a/.env.example +++ b/.env.example @@ -88,8 +88,9 @@ MONITOR_TOKEN= MONITOR_CPU_PCT=90 MONITOR_MEM_PCT=90 MONITOR_STORAGE_PCT=85 -# A container restart within this window alarms (time-based so it self-clears) -MONITOR_RESTART_WINDOW_MIN=15 +# A container restart within this window alarms (time-based so it self-clears). +# Keep it at the poller's interval, or a recovered pod keeps alarming. +MONITOR_RESTART_WINDOW_MIN=1 # How long a pod may sit Pending / not-Ready before alarming MONITOR_PENDING_GRACE_MIN=10 # Infra namespaces watched in addition to {K8S_NAMESPACE_PREFIX}-*. diff --git a/helm/vtafarm-api/values.yaml b/helm/vtafarm-api/values.yaml index ffb04a9..d5c24b8 100644 --- a/helm/vtafarm-api/values.yaml +++ b/helm/vtafarm-api/values.yaml @@ -66,7 +66,7 @@ monitor: cpuPct: "90" memPct: "90" storagePct: "85" - restartWindowMin: "15" + restartWindowMin: "1" pendingGraceMin: "10" extraNamespaces: "vault,vault-transit,longhorn-system" diff --git a/internal/config/config.go b/internal/config/config.go index e8cecc8..7c4d886 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -214,7 +214,7 @@ func Load() *Config { CPUPercent: getEnvInt("MONITOR_CPU_PCT", 90), MemPercent: getEnvInt("MONITOR_MEM_PCT", 90), StoragePercent: getEnvInt("MONITOR_STORAGE_PCT", 85), - RestartWindowMin: getEnvInt("MONITOR_RESTART_WINDOW_MIN", 15), + RestartWindowMin: getEnvInt("MONITOR_RESTART_WINDOW_MIN", 1), PendingGraceMin: getEnvInt("MONITOR_PENDING_GRACE_MIN", 10), ExtraNamespaces: splitComma(getEnv("MONITOR_EXTRA_NAMESPACES", "vault,vault-transit,longhorn-system")), }, From 8f4c946315fb96e4fe21ad696c9edbc90a971c1d Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 17 Aug 2026 23:58:36 -0700 Subject: [PATCH 2/2] docs: drop the restart-window note from .env.example The surrounding entries state what the value does, not how to pick it. Signed-off-by: vthwang --- .env.example | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.env.example b/.env.example index c36e6d1..e4e974d 100644 --- a/.env.example +++ b/.env.example @@ -88,8 +88,7 @@ MONITOR_TOKEN= MONITOR_CPU_PCT=90 MONITOR_MEM_PCT=90 MONITOR_STORAGE_PCT=85 -# A container restart within this window alarms (time-based so it self-clears). -# Keep it at the poller's interval, or a recovered pod keeps alarming. +# A container restart within this window alarms (time-based so it self-clears) MONITOR_RESTART_WINDOW_MIN=1 # How long a pod may sit Pending / not-Ready before alarming MONITOR_PENDING_GRACE_MIN=10