From 4ce22c99718a6ea9a69b8c7c94aa0f483be3a0bf Mon Sep 17 00:00:00 2001 From: nfebe Date: Tue, 25 Aug 2026 16:44:45 +0100 Subject: [PATCH 1/2] feat(observability): Add container memory utilization --- CHANGELOG.md | 1 + internal/observ/alert_api_test.go | 15 +++++++++++++++ internal/observ/alerts.go | 4 ++-- internal/observ/alerts_test.go | 9 +++++++++ internal/observ/api_test.go | 4 ++-- internal/observ/otlp.go | 3 ++- internal/observ/otlp_test.go | 3 +++ internal/observ/prometheus.go | 11 ++++++----- internal/observ/prometheus_test.go | 5 +++++ internal/observ/store.go | 19 ++++++++++++------- internal/observ/store_test.go | 19 +++++++++++++++++-- 11 files changed, 74 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b84ad27..2cf8abc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Seventh beta of the Albacore release, focused on reliable first deployments thro ### Added - Bind mount ownership declarations and file ownership changes +- Container memory utilization metrics for percentage-based alerts and dashboards - One-shot service jobs whose status and output follow the container process - Partial environment variable updates with plan support - OpenAPI metadata for accepted values and plan capable operations diff --git a/internal/observ/alert_api_test.go b/internal/observ/alert_api_test.go index 20eadd6..465d39d 100644 --- a/internal/observ/alert_api_test.go +++ b/internal/observ/alert_api_test.go @@ -116,6 +116,21 @@ func TestAlertRulesRoundTripThroughTheAPI(t *testing.T) { } } +func TestContainerMemoryUtilizationRuleThroughTheAPI(t *testing.T) { + h, engine, _ := alertHandler(t) + body := `[{"name":"Memory high","metric":"container.memory.utilization","comparison":"above","threshold":90,"for_seconds":300,"enabled":true}]` + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/alerts/rules", strings.NewReader(body))) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rec.Code, rec.Body.String()) + } + rules := engine.Rules() + if len(rules) != 1 || rules[0].Metric != MetricMemoryUtilization || rules[0].Threshold != 90 { + t.Errorf("engine rules = %+v", rules) + } +} + func TestAlertRulesRejectAnUnusableRule(t *testing.T) { h, engine, _ := alertHandler(t) diff --git a/internal/observ/alerts.go b/internal/observ/alerts.go index e20df1b..afa5108 100644 --- a/internal/observ/alerts.go +++ b/internal/observ/alerts.go @@ -77,7 +77,7 @@ func (r AlertRule) forDuration() time.Duration { func knownMetric(name string) bool { switch name { - case MetricCPUUsage, MetricMemoryUsage, MetricMemoryLimit, MetricNetworkRx, MetricNetworkTx, + case MetricCPUUsage, MetricMemoryUsage, MetricMemoryLimit, MetricMemoryUtilization, MetricNetworkRx, MetricNetworkTx, MetricHostCPU, MetricHostMemUtil, MetricHostMemUsage, MetricHostMemLimit, MetricHostDisk: return true } @@ -171,7 +171,7 @@ func (e AlertEvent) Message() string { func formatMetricValue(metric string, v float64) string { switch metric { - case MetricCPUUsage, MetricHostCPU, MetricHostMemUtil, MetricHostDisk: + case MetricCPUUsage, MetricMemoryUtilization, MetricHostCPU, MetricHostMemUtil, MetricHostDisk: return fmt.Sprintf("%.1f%%", v) case MetricNetworkRx, MetricNetworkTx: return formatBytes(v) + "/s" diff --git a/internal/observ/alerts_test.go b/internal/observ/alerts_test.go index 69586e8..a6a169f 100644 --- a/internal/observ/alerts_test.go +++ b/internal/observ/alerts_test.go @@ -238,4 +238,13 @@ func TestAlertEventMessage(t *testing.T) { if got := mem.Message(); got != "Memory: shop-db in shop is 5.0 GB, above 4.0 GB." { t.Errorf("Message() = %q", got) } + + utilization := AlertEvent{ + RuleName: "Memory utilization", Deployment: "shop", Container: "shop-db", + Metric: MetricMemoryUtilization, Value: 92.5, Threshold: 90, + Comparison: ComparisonAbove, State: AlertFiring, + } + if got := utilization.Message(); got != "Memory utilization: shop-db in shop is 92.5%, above 90.0%." { + t.Errorf("Message() = %q", got) + } } diff --git a/internal/observ/api_test.go b/internal/observ/api_test.go index 1bda438..decc03b 100644 --- a/internal/observ/api_test.go +++ b/internal/observ/api_test.go @@ -12,7 +12,7 @@ import ( func TestHandlerLatestGroupsByDeployment(t *testing.T) { store := NewStore(10) now := time.Unix(1_700_000_000, 0) - store.Record(ContainerSample{Deployment: "shop", Container: "shop-web", CPUPercent: 10, MemoryUsage: 200}, now) + store.Record(ContainerSample{Deployment: "shop", Container: "shop-web", CPUPercent: 10, MemoryUsage: 200, MemoryLimit: 1000}, now) store.Record(ContainerSample{Deployment: "shop", Container: "shop-db", CPUPercent: 4, MemoryUsage: 500}, now) rec := httptest.NewRecorder() @@ -37,7 +37,7 @@ func TestHandlerLatestGroupsByDeployment(t *testing.T) { web = &got[0].Containers[i] } } - if web == nil || web.Metrics[MetricCPUUsage] != 10 || web.Metrics[MetricMemoryUsage] != 200 { + if web == nil || web.Metrics[MetricCPUUsage] != 10 || web.Metrics[MetricMemoryUsage] != 200 || web.Metrics[MetricMemoryUtilization] != 20 { t.Errorf("shop-web container = %+v", web) } } diff --git a/internal/observ/otlp.go b/internal/observ/otlp.go index 2f23464..af8f29f 100644 --- a/internal/observ/otlp.go +++ b/internal/observ/otlp.go @@ -77,6 +77,7 @@ func registerGauges(meter metric.Meter, store *Store) error { MetricCPUUsage, MetricMemoryUsage, MetricMemoryLimit, + MetricMemoryUtilization, MetricNetworkRx, MetricNetworkTx, } { @@ -113,7 +114,7 @@ func registerGauges(meter metric.Meter, store *Store) error { // bytes as bytes rather than a bare number. func metricUnit(name string) string { switch name { - case MetricCPUUsage: + case MetricCPUUsage, MetricMemoryUtilization: return "%" case MetricMemoryUsage, MetricMemoryLimit, MetricNetworkRx, MetricNetworkTx: return "By" diff --git a/internal/observ/otlp_test.go b/internal/observ/otlp_test.go index ef46cbf..db6252c 100644 --- a/internal/observ/otlp_test.go +++ b/internal/observ/otlp_test.go @@ -108,4 +108,7 @@ func TestMetricUnits(t *testing.T) { if got := metricUnit(MetricCPUUsage); got != "%" { t.Errorf("cpu unit = %q, want %%", got) } + if got := metricUnit(MetricMemoryUtilization); got != "%" { + t.Errorf("memory utilization unit = %q, want %%", got) + } } diff --git a/internal/observ/prometheus.go b/internal/observ/prometheus.go index 20f92f2..049e82b 100644 --- a/internal/observ/prometheus.go +++ b/internal/observ/prometheus.go @@ -11,11 +11,12 @@ const prometheusContentType = "text/plain; version=0.0.4; charset=utf-8" // metricHelp describes each series for anything reading the exposition. var metricHelp = map[string]string{ - MetricCPUUsage: "Container CPU usage as a percentage of host capacity.", - MetricMemoryUsage: "Container memory usage in bytes.", - MetricMemoryLimit: "Container memory limit in bytes.", - MetricNetworkRx: "Bytes received by the container per second.", - MetricNetworkTx: "Bytes sent by the container per second.", + MetricCPUUsage: "Container CPU usage as a percentage of host capacity.", + MetricMemoryUsage: "Container memory usage in bytes.", + MetricMemoryLimit: "Container memory limit in bytes.", + MetricMemoryUtilization: "Container memory usage as a percentage of its effective limit.", + MetricNetworkRx: "Bytes received by the container per second.", + MetricNetworkTx: "Bytes sent by the container per second.", } // renderPrometheus writes the store's latest sample per series in Prometheus text exposition diff --git a/internal/observ/prometheus_test.go b/internal/observ/prometheus_test.go index 7f9e8c2..a8ca41c 100644 --- a/internal/observ/prometheus_test.go +++ b/internal/observ/prometheus_test.go @@ -13,6 +13,8 @@ func TestRenderPrometheus(t *testing.T) { Sample: Sample{Time: at, Value: 12.5}}, {SeriesKey: SeriesKey{Deployment: "shop", Container: "shop-db", Metric: MetricMemoryUsage}, Sample: Sample{Time: at, Value: 5_368_709_120}}, + {SeriesKey: SeriesKey{Deployment: "shop", Container: "shop-db", Metric: MetricMemoryUtilization}, + Sample: Sample{Time: at, Value: 92.5}}, }) // Prometheus allows only letters, digits and underscores, so the OTel dots convert. @@ -36,6 +38,9 @@ func TestRenderPrometheus(t *testing.T) { if !strings.Contains(out, `container_memory_usage{deployment="shop",container="shop-db"} 5.36870912e+09`) { t.Errorf("memory sample line wrong:\n%s", out) } + if !strings.Contains(out, `container_memory_utilization{deployment="shop",container="shop-db"} 92.5`) { + t.Errorf("memory utilization sample line wrong:\n%s", out) + } } func TestRenderPrometheusEscapesLabelValues(t *testing.T) { diff --git a/internal/observ/store.go b/internal/observ/store.go index 95a9f2a..489c2ac 100644 --- a/internal/observ/store.go +++ b/internal/observ/store.go @@ -14,11 +14,12 @@ import ( // OpenTelemetry container metric names (semconv). Emitting these verbatim keeps FlatRun's // metrics interoperable with any OTel backend. const ( - MetricCPUUsage = "container.cpu.usage" - MetricMemoryUsage = "container.memory.usage" - MetricMemoryLimit = "container.memory.limit" - MetricNetworkRx = "container.network.io.rx" - MetricNetworkTx = "container.network.io.tx" + MetricCPUUsage = "container.cpu.usage" + MetricMemoryUsage = "container.memory.usage" + MetricMemoryLimit = "container.memory.limit" + MetricMemoryUtilization = "container.memory.utilization" + MetricNetworkRx = "container.network.io.rx" + MetricNetworkTx = "container.network.io.tx" ) // Host (system-wide) metric names, semconv system.* conventions. These answer @@ -135,13 +136,17 @@ func (s *Store) add(key SeriesKey, sample Sample) { // not a total that only climbs. func (s *Store) Record(c ContainerSample, t time.Time) { rxRate, txRate := s.netRates(c, t) - s.record(SeriesKey{Deployment: c.Deployment, Container: c.Container}, map[string]float64{ + metrics := map[string]float64{ MetricCPUUsage: c.CPUPercent, MetricMemoryUsage: float64(c.MemoryUsage), MetricMemoryLimit: float64(c.MemoryLimit), MetricNetworkRx: rxRate, MetricNetworkTx: txRate, - }, t) + } + if c.MemoryLimit > 0 { + metrics[MetricMemoryUtilization] = float64(c.MemoryUsage) / float64(c.MemoryLimit) * 100 + } + s.record(SeriesKey{Deployment: c.Deployment, Container: c.Container}, metrics, t) } // netRates converts a container's cumulative network counters into a per-second diff --git a/internal/observ/store_test.go b/internal/observ/store_test.go index d45cf88..fb0c9cf 100644 --- a/internal/observ/store_test.go +++ b/internal/observ/store_test.go @@ -14,8 +14,8 @@ func TestStoreRecordExpandsSemconvSeries(t *testing.T) { }, t0) keys := s.Series() - if len(keys) != 5 { - t.Fatalf("expected 5 semconv series, got %d: %+v", len(keys), keys) + if len(keys) != 6 { + t.Fatalf("expected 6 semconv series, got %d: %+v", len(keys), keys) } cpu := s.Range(SeriesKey{Deployment: "shop", Container: "shop-web", Metric: MetricCPUUsage}, t0) @@ -26,6 +26,21 @@ func TestStoreRecordExpandsSemconvSeries(t *testing.T) { if len(mem) != 1 || mem[0].Value != 100 { t.Errorf("memory series = %+v, want one sample of 100", mem) } + utilization := s.Range(SeriesKey{Deployment: "shop", Container: "shop-web", Metric: MetricMemoryUtilization}, t0) + if len(utilization) != 1 || utilization[0].Value != 10 { + t.Errorf("memory utilization series = %+v, want one sample of 10", utilization) + } +} + +func TestStoreOmitsMemoryUtilizationWithoutALimit(t *testing.T) { + s := NewStore(10) + t0 := time.Unix(1_700_000_000, 0) + s.Record(ContainerSample{Deployment: "shop", Container: "shop-web", MemoryUsage: 100}, t0) + + got := s.Range(SeriesKey{Deployment: "shop", Container: "shop-web", Metric: MetricMemoryUtilization}, t0) + if got != nil { + t.Errorf("memory utilization without a limit = %+v, want no series", got) + } } func TestStoreNetworkCounterBecomesRate(t *testing.T) { From bcf484a1ea15f20dc7437dbc501ce5be2ada68ed Mon Sep 17 00:00:00 2001 From: nfebe Date: Tue, 25 Aug 2026 17:49:08 +0100 Subject: [PATCH 2/2] docs: Move new changes to Unreleased --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cf8abc..7a23966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,16 @@ # Changelog +## [Unreleased] + +### Added +- Container memory utilization metrics for percentage-based alerts and dashboards + ## [0.4.0-beta.7] - 2026-08-23 Seventh beta of the Albacore release, focused on reliable first deployments through the CLI. ### Added - Bind mount ownership declarations and file ownership changes -- Container memory utilization metrics for percentage-based alerts and dashboards - One-shot service jobs whose status and output follow the container process - Partial environment variable updates with plan support - OpenAPI metadata for accepted values and plan capable operations