fix(monitors): keep a down server down while its breach lasts after for_sec is raised

server/internal/services/metricrules.go

@@ -179,6 +179,12 @@ func nextServerState(prev *models.MonitorServerState, breach bool, forSec int, n
 	if prev != nil && prev.BreachSince != nil {
 		since = *prev.BreachSince
 	}
+	// A server already down stays down while the breach lasts, even if for_sec
+	// was raised meanwhile. Dropping it to pending would never resolve the open
+	// incident, because only an up transition closes it.
+	if prev != nil && prev.Status == models.StatusDown {
+		return models.StatusDown, &since
+	}
 	if now.Sub(since) >= time.Duration(forSec)*time.Second {
 		return models.StatusDown, &since
 	}

server/internal/services/metricrules_test.go

@@ -85,6 +85,12 @@ func TestNextServerState(t *testing.T) {
 	if st != models.StatusDown {
 		t.Fatalf("for_sec 0 is down immediately, got %s", st)
 	}
+	// Raising for_sec mid-breach must not demote a down server to pending:
+	// pending never resolves an incident, so it would stay open forever.
+	st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(2 * time.Minute)}, true, 3600, evalNow)
+	if st != models.StatusDown || !since.Equal(*past(2 * time.Minute)) {
+		t.Fatalf("down with larger for_sec should stay down keeping since, got %s %v", st, since)
+	}
 	st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(time.Hour)}, false, 300, evalNow)
 	if st != models.StatusUp || since != nil {
 		t.Fatalf("clear should be up with no since, got %s %v", st, since)