From 2afdc267f8c5a824d2985529354c17627cbc533c Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Thu, 17 Sep 2026 09:22:50 +0000 Subject: [PATCH] fix(monitors): keep a down server down while its breach lasts after for_sec is raised --- server/internal/services/metricrules.go | 6 ++++++ server/internal/services/metricrules_test.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/server/internal/services/metricrules.go b/server/internal/services/metricrules.go index 5bab55e..77878b3 100644 --- a/server/internal/services/metricrules.go +++ b/server/internal/services/metricrules.go @@ -179,6 +179,12 @@ func nextServerState(prev *models.MonitorServerState, breach bool, forSec int, n if prev != nil && prev.BreachSince != nil { since = *prev.BreachSince } + // A server already down stays down while the breach lasts, even if for_sec + // was raised meanwhile. Dropping it to pending would never resolve the open + // incident, because only an up transition closes it. + if prev != nil && prev.Status == models.StatusDown { + return models.StatusDown, &since + } if now.Sub(since) >= time.Duration(forSec)*time.Second { return models.StatusDown, &since } diff --git a/server/internal/services/metricrules_test.go b/server/internal/services/metricrules_test.go index af37d89..9a36094 100644 --- a/server/internal/services/metricrules_test.go +++ b/server/internal/services/metricrules_test.go @@ -85,6 +85,12 @@ func TestNextServerState(t *testing.T) { if st != models.StatusDown { t.Fatalf("for_sec 0 is down immediately, got %s", st) } + // Raising for_sec mid-breach must not demote a down server to pending: + // pending never resolves an incident, so it would stay open forever. + st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(2 * time.Minute)}, true, 3600, evalNow) + if st != models.StatusDown || !since.Equal(*past(2 * time.Minute)) { + t.Fatalf("down with larger for_sec should stay down keeping since, got %s %v", st, since) + } st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(time.Hour)}, false, 300, evalNow) if st != models.StatusUp || since != nil { t.Fatalf("clear should be up with no since, got %s %v", st, since)