fix: file live outages as active, refuse "operational" over zero components

- A derived monitor outage with no `resolved_at` went to `History`, so an
  ongoing disruption was listed under "Past incidents" while the component pill
  beside it read Down. Unresolved now goes to `ActiveIncidents`.
- `overallState` returned `up` when nothing was counted: "all systems
  operational" claimed from no evidence at all. A page with no components now
  reports `no_data`, which the view already renders as "Status unknown".
- `EnsureStatusPageIndexes` returned on the first failure, so a transient
  failure on the `status_pages` index left `status_incidents` with no unique
  `(instance_id, incident_id)` index — a correctness property, not a scan
  optimisation. All three are attempted and the failures joined.
This commit is contained in:
2026-08-25 09:04:54 +00:00
parent da1dc90ac5
commit 7e1d67dba4
3 changed files with 87 additions and 6 deletions
+13 -4
View File
@@ -44,22 +44,30 @@ func spCtx() (context.Context, context.CancelFunc) {
// EnsureAuthIndexes: the unique page_id index is a correctness property, but a
// missing secondary index on a small collection degrades to a scan, which is no
// reason to refuse to serve the fleet. main.go warns rather than exiting.
//
// All three are attempted and the failures joined, rather than returning on
// the first. The three are independent, and two of them are uniqueness
// constraints — bailing out on the status_pages index meant a transient
// failure there silently left status_incidents with no unique
// (instance_id, incident_id) index at all.
func EnsureStatusPageIndexes() error {
ctx, cancel := spCtx()
defer cancel()
var errs []error
if _, err := db.Col("status_pages").Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "page_id", Value: 1}},
Options: options.Index().SetUnique(true),
}); err != nil {
return err
errs = append(errs, fmt.Errorf("status_pages (instance_id, page_id): %w", err))
}
if _, err := db.Col("status_incidents").Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "incident_id", Value: 1}},
Options: options.Index().SetUnique(true),
}); err != nil {
return err
errs = append(errs, fmt.Errorf("status_incidents (instance_id, incident_id): %w", err))
}
// The public read filters by page and orders by recency, and it is the
@@ -67,9 +75,10 @@ func EnsureStatusPageIndexes() error {
if _, err := db.Col("status_incidents").Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "page_ids", Value: 1}, {Key: "started_at", Value: -1}},
}); err != nil {
return err
errs = append(errs, fmt.Errorf("status_incidents (instance_id, page_ids, started_at): %w", err))
}
return nil
return errors.Join(errs...)
}
var (
+18 -2
View File
@@ -172,7 +172,7 @@ func assembleSnapshot(in snapshotInput) StatusSnapshot {
if inc.StartedAt.Before(in.Now.AddDate(0, 0, -HistoryDays)) {
continue
}
snap.History = append(snap.History, PublicIncident{
derived := PublicIncident{
ID: inc.IncidentID,
Kind: models.StatusKindIncident,
Title: name + " unavailable",
@@ -180,7 +180,15 @@ func assembleSnapshot(in snapshotInput) StatusSnapshot {
Affected: []string{name},
StartedAt: inc.StartedAt,
ResolvedAt: inc.ResolvedAt,
})
}
// An outage that has not recovered is happening now. Filing it under
// History while the component pill reads Down and Overall reads down
// told the reader the disruption was over.
if inc.ResolvedAt == nil {
snap.ActiveIncidents = append(snap.ActiveIncidents, derived)
} else {
snap.History = append(snap.History, derived)
}
}
sort.Slice(snap.History, func(i, j int) bool {
@@ -551,8 +559,10 @@ func storeSnapshot(instanceID, pageID string, snap StatusSnapshot) {
func overallState(sections []PublicSection) string {
worst := PublicUp
anyDown, anyMaint, anyOther := false, false, false
counted := 0
for _, s := range sections {
for _, c := range s.Components {
counted++
switch c.Status {
case PublicDown:
anyDown = true
@@ -564,6 +574,12 @@ func overallState(sections []PublicSection) string {
}
}
switch {
case counted == 0:
// Nothing is being reported, so nothing is known. "All systems
// operational" over zero components is a claim of health made from no
// evidence at all; PublicNoData is what the view renders as
// "Status unknown".
worst = PublicNoData
case anyDown:
worst = PublicDown
case anyMaint:
@@ -222,3 +222,59 @@ func TestAssembleSnapshotOnlyIncludesAuthoredIncidentsForThisPage(t *testing.T)
t.Fatalf("active incidents = %+v, want only the one naming this page", snap.ActiveIncidents)
}
}
// A derived outage that has not recovered is happening now. It used to be
// appended to History unconditionally, so a live outage was reported under
// "Past incidents" while the component pill next to it read Down.
func TestAssembleSnapshotDerivedIncidentActiveUntilResolved(t *testing.T) {
now := time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC)
resolved := now.Add(-30 * time.Minute)
in := testInput(now)
in.AutoIncidents = []models.Incident{
{
IncidentID: "inc-open",
MonitorID: "mon-2",
StartedAt: now.Add(-2 * time.Hour),
},
{
IncidentID: "inc-closed",
MonitorID: "mon-1",
StartedAt: now.Add(-3 * time.Hour),
ResolvedAt: &resolved,
},
}
snap := assembleSnapshot(in)
if len(snap.ActiveIncidents) != 1 || snap.ActiveIncidents[0].ID != "inc-open" {
t.Fatalf("unresolved incident should be active, got %+v", snap.ActiveIncidents)
}
if snap.ActiveIncidents[0].ResolvedAt != nil {
t.Errorf("active incident carries a resolved_at: %v", snap.ActiveIncidents[0].ResolvedAt)
}
if len(snap.History) != 1 || snap.History[0].ID != "inc-closed" {
t.Fatalf("resolved incident should be history, got %+v", snap.History)
}
if snap.History[0].ResolvedAt == nil {
t.Errorf("history entry lost its resolved_at")
}
}
// A page with no components knows nothing, and claiming "all systems
// operational" from no evidence is the one answer it must not give.
func TestAssembleSnapshotEmptyPageIsNotOperational(t *testing.T) {
now := time.Date(2026, 8, 24, 12, 0, 0, 0, time.UTC)
in := testInput(now)
in.Page.Sections = nil
if got := assembleSnapshot(in).Overall; got != PublicNoData {
t.Errorf("overall for a page with no components = %q, want %q", got, PublicNoData)
}
in = testInput(now)
in.Page.Sections = []models.StatusPageSection{{Name: "API"}}
if got := assembleSnapshot(in).Overall; got != PublicNoData {
t.Errorf("overall for a page with an empty section = %q, want %q", got, PublicNoData)
}
}