fix: file live outages as active, refuse "operational" over zero components
- A derived monitor outage with no `resolved_at` went to `History`, so an ongoing disruption was listed under "Past incidents" while the component pill beside it read Down. Unresolved now goes to `ActiveIncidents`. - `overallState` returned `up` when nothing was counted: "all systems operational" claimed from no evidence at all. A page with no components now reports `no_data`, which the view already renders as "Status unknown". - `EnsureStatusPageIndexes` returned on the first failure, so a transient failure on the `status_pages` index left `status_incidents` with no unique `(instance_id, incident_id)` index — a correctness property, not a scan optimisation. All three are attempted and the failures joined.
This commit is contained in:
@@ -44,22 +44,30 @@ func spCtx() (context.Context, context.CancelFunc) {
|
||||
// EnsureAuthIndexes: the unique page_id index is a correctness property, but a
|
||||
// missing secondary index on a small collection degrades to a scan, which is no
|
||||
// reason to refuse to serve the fleet. main.go warns rather than exiting.
|
||||
//
|
||||
// All three are attempted and the failures joined, rather than returning on
|
||||
// the first. The three are independent, and two of them are uniqueness
|
||||
// constraints — bailing out on the status_pages index meant a transient
|
||||
// failure there silently left status_incidents with no unique
|
||||
// (instance_id, incident_id) index at all.
|
||||
func EnsureStatusPageIndexes() error {
|
||||
ctx, cancel := spCtx()
|
||||
defer cancel()
|
||||
|
||||
var errs []error
|
||||
|
||||
if _, err := db.Col("status_pages").Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "page_id", Value: 1}},
|
||||
Options: options.Index().SetUnique(true),
|
||||
}); err != nil {
|
||||
return err
|
||||
errs = append(errs, fmt.Errorf("status_pages (instance_id, page_id): %w", err))
|
||||
}
|
||||
|
||||
if _, err := db.Col("status_incidents").Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "incident_id", Value: 1}},
|
||||
Options: options.Index().SetUnique(true),
|
||||
}); err != nil {
|
||||
return err
|
||||
errs = append(errs, fmt.Errorf("status_incidents (instance_id, incident_id): %w", err))
|
||||
}
|
||||
|
||||
// The public read filters by page and orders by recency, and it is the
|
||||
@@ -67,9 +75,10 @@ func EnsureStatusPageIndexes() error {
|
||||
if _, err := db.Col("status_incidents").Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "page_ids", Value: 1}, {Key: "started_at", Value: -1}},
|
||||
}); err != nil {
|
||||
return err
|
||||
errs = append(errs, fmt.Errorf("status_incidents (instance_id, page_ids, started_at): %w", err))
|
||||
}
|
||||
return nil
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -172,7 +172,7 @@ func assembleSnapshot(in snapshotInput) StatusSnapshot {
|
||||
if inc.StartedAt.Before(in.Now.AddDate(0, 0, -HistoryDays)) {
|
||||
continue
|
||||
}
|
||||
snap.History = append(snap.History, PublicIncident{
|
||||
derived := PublicIncident{
|
||||
ID: inc.IncidentID,
|
||||
Kind: models.StatusKindIncident,
|
||||
Title: name + " unavailable",
|
||||
@@ -180,7 +180,15 @@ func assembleSnapshot(in snapshotInput) StatusSnapshot {
|
||||
Affected: []string{name},
|
||||
StartedAt: inc.StartedAt,
|
||||
ResolvedAt: inc.ResolvedAt,
|
||||
})
|
||||
}
|
||||
// An outage that has not recovered is happening now. Filing it under
|
||||
// History while the component pill reads Down and Overall reads down
|
||||
// told the reader the disruption was over.
|
||||
if inc.ResolvedAt == nil {
|
||||
snap.ActiveIncidents = append(snap.ActiveIncidents, derived)
|
||||
} else {
|
||||
snap.History = append(snap.History, derived)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(snap.History, func(i, j int) bool {
|
||||
@@ -551,8 +559,10 @@ func storeSnapshot(instanceID, pageID string, snap StatusSnapshot) {
|
||||
func overallState(sections []PublicSection) string {
|
||||
worst := PublicUp
|
||||
anyDown, anyMaint, anyOther := false, false, false
|
||||
counted := 0
|
||||
for _, s := range sections {
|
||||
for _, c := range s.Components {
|
||||
counted++
|
||||
switch c.Status {
|
||||
case PublicDown:
|
||||
anyDown = true
|
||||
@@ -564,6 +574,12 @@ func overallState(sections []PublicSection) string {
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case counted == 0:
|
||||
// Nothing is being reported, so nothing is known. "All systems
|
||||
// operational" over zero components is a claim of health made from no
|
||||
// evidence at all; PublicNoData is what the view renders as
|
||||
// "Status unknown".
|
||||
worst = PublicNoData
|
||||
case anyDown:
|
||||
worst = PublicDown
|
||||
case anyMaint:
|
||||
|
||||
@@ -222,3 +222,59 @@ func TestAssembleSnapshotOnlyIncludesAuthoredIncidentsForThisPage(t *testing.T)
|
||||
t.Fatalf("active incidents = %+v, want only the one naming this page", snap.ActiveIncidents)
|
||||
}
|
||||
}
|
||||
|
||||
// A derived outage that has not recovered is happening now. It used to be
|
||||
// appended to History unconditionally, so a live outage was reported under
|
||||
// "Past incidents" while the component pill next to it read Down.
|
||||
func TestAssembleSnapshotDerivedIncidentActiveUntilResolved(t *testing.T) {
|
||||
now := time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC)
|
||||
resolved := now.Add(-30 * time.Minute)
|
||||
|
||||
in := testInput(now)
|
||||
in.AutoIncidents = []models.Incident{
|
||||
{
|
||||
IncidentID: "inc-open",
|
||||
MonitorID: "mon-2",
|
||||
StartedAt: now.Add(-2 * time.Hour),
|
||||
},
|
||||
{
|
||||
IncidentID: "inc-closed",
|
||||
MonitorID: "mon-1",
|
||||
StartedAt: now.Add(-3 * time.Hour),
|
||||
ResolvedAt: &resolved,
|
||||
},
|
||||
}
|
||||
|
||||
snap := assembleSnapshot(in)
|
||||
|
||||
if len(snap.ActiveIncidents) != 1 || snap.ActiveIncidents[0].ID != "inc-open" {
|
||||
t.Fatalf("unresolved incident should be active, got %+v", snap.ActiveIncidents)
|
||||
}
|
||||
if snap.ActiveIncidents[0].ResolvedAt != nil {
|
||||
t.Errorf("active incident carries a resolved_at: %v", snap.ActiveIncidents[0].ResolvedAt)
|
||||
}
|
||||
if len(snap.History) != 1 || snap.History[0].ID != "inc-closed" {
|
||||
t.Fatalf("resolved incident should be history, got %+v", snap.History)
|
||||
}
|
||||
if snap.History[0].ResolvedAt == nil {
|
||||
t.Errorf("history entry lost its resolved_at")
|
||||
}
|
||||
}
|
||||
|
||||
// A page with no components knows nothing, and claiming "all systems
|
||||
// operational" from no evidence is the one answer it must not give.
|
||||
func TestAssembleSnapshotEmptyPageIsNotOperational(t *testing.T) {
|
||||
now := time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
in := testInput(now)
|
||||
in.Page.Sections = nil
|
||||
if got := assembleSnapshot(in).Overall; got != PublicNoData {
|
||||
t.Errorf("overall for a page with no components = %q, want %q", got, PublicNoData)
|
||||
}
|
||||
|
||||
in = testInput(now)
|
||||
in.Page.Sections = []models.StatusPageSection{{Name: "API"}}
|
||||
if got := assembleSnapshot(in).Overall; got != PublicNoData {
|
||||
t.Errorf("overall for a page with an empty section = %q, want %q", got, PublicNoData)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user