diff --git a/server/internal/services/metricrules.go b/server/internal/services/metricrules.go index 5bab55e..77878b3 100644 --- a/server/internal/services/metricrules.go +++ b/server/internal/services/metricrules.go @@ -179,6 +179,12 @@ func nextServerState(prev *models.MonitorServerState, breach bool, forSec int, n if prev != nil && prev.BreachSince != nil { since = *prev.BreachSince } + // A server already down stays down while the breach lasts, even if for_sec + // was raised meanwhile. Dropping it to pending would never resolve the open + // incident, because only an up transition closes it. + if prev != nil && prev.Status == models.StatusDown { + return models.StatusDown, &since + } if now.Sub(since) >= time.Duration(forSec)*time.Second { return models.StatusDown, &since } diff --git a/server/internal/services/metricrules_test.go b/server/internal/services/metricrules_test.go index af37d89..9a36094 100644 --- a/server/internal/services/metricrules_test.go +++ b/server/internal/services/metricrules_test.go @@ -85,6 +85,12 @@ func TestNextServerState(t *testing.T) { if st != models.StatusDown { t.Fatalf("for_sec 0 is down immediately, got %s", st) } + // Raising for_sec mid-breach must not demote a down server to pending: + // pending never resolves an incident, so it would stay open forever. + st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(2 * time.Minute)}, true, 3600, evalNow) + if st != models.StatusDown || !since.Equal(*past(2 * time.Minute)) { + t.Fatalf("down with larger for_sec should stay down keeping since, got %s %v", st, since) + } st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(time.Hour)}, false, 300, evalNow) if st != models.StatusUp || since != nil { t.Fatalf("clear should be up with no since, got %s %v", st, since)