diff --git a/server/internal/models/monitor.go b/server/internal/models/monitor.go index 2893be4..1dd7920 100644 --- a/server/internal/models/monitor.go +++ b/server/internal/models/monitor.go @@ -7,12 +7,21 @@ import ( ) const ( - MonitorHTTP = "http" - MonitorTCP = "tcp" - MonitorICMP = "icmp" - MonitorTLS = "tls" + MonitorHTTP = "http" + MonitorTCP = "tcp" + MonitorICMP = "icmp" + MonitorTLS = "tls" + MonitorMetric = "metric" + MonitorHeartbeat = "heartbeat" ) +// IsPassiveMonitor reports whether a monitor type is evaluated from data that +// arrives (a ping, an agent report) rather than by running a check. Passive +// monitors are never handed to monitorsched or to an agent. +func IsPassiveMonitor(t string) bool { + return t == MonitorMetric || t == MonitorHeartbeat +} + const ( StatusUp = "up" StatusDown = "down" @@ -38,6 +47,14 @@ type MonitorTarget struct { Keyword string `bson:"keyword,omitempty" json:"keyword,omitempty"` TLSWarnDays int `bson:"tls_warn_days,omitempty" json:"tls_warn_days,omitempty"` Insecure bool `bson:"insecure,omitempty" json:"insecure,omitempty"` + // Metric monitors. + Selector map[string]string `bson:"selector,omitempty" json:"selector,omitempty"` + Metric string `bson:"metric,omitempty" json:"metric,omitempty"` + Threshold float64 `bson:"threshold,omitempty" json:"threshold,omitempty"` + Mount string `bson:"mount,omitempty" json:"mount,omitempty"` + // Heartbeat monitors. + PeriodSec int `bson:"period_sec,omitempty" json:"period_sec,omitempty"` + GraceSec int `bson:"grace_sec,omitempty" json:"grace_sec,omitempty"` } type MonitorState struct { @@ -48,6 +65,8 @@ type MonitorState struct { CertExpiryAt *time.Time `bson:"cert_expiry_at,omitempty" json:"cert_expiry_at,omitempty"` Fails int `bson:"fails" json:"fails"` LastNotifiedAt *time.Time `bson:"last_notified_at,omitempty" json:"last_notified_at,omitempty"` + LastPingAt *time.Time `bson:"last_ping_at,omitempty" json:"last_ping_at,omitempty"` + StartedAt *time.Time `bson:"started_at,omitempty" json:"started_at,omitempty"` } type Monitor struct { @@ -68,6 +87,11 @@ type Monitor struct { ChannelIDs []string `bson:"channel_ids,omitempty" json:"channel_ids,omitempty"` State MonitorState `bson:"state" json:"state"` CreatedAt time.Time `bson:"created_at" json:"created_at"` + // ForSec is how long a metric condition must hold before a server is down. + ForSec int `bson:"for_sec,omitempty" json:"for_sec,omitempty"` + // HeartbeatTokenHash is the SHA-256 of the ping token. The token itself is + // shown once, on create or rotate, and never stored. + HeartbeatTokenHash string `bson:"heartbeat_token_hash,omitempty" json:"-"` } type Incident struct { @@ -77,6 +101,22 @@ type Incident struct { StartedAt time.Time `bson:"started_at" json:"started_at"` ResolvedAt *time.Time `bson:"resolved_at,omitempty" json:"resolved_at,omitempty"` Cause string `bson:"cause,omitempty" json:"cause,omitempty"` + // ServerID is set only for metric monitors, which keep one incident per + // breaching server. + ServerID string `bson:"server_id,omitempty" json:"server_id,omitempty"` +} + +// MonitorServerState is one metric monitor's view of one matching server. +type MonitorServerState struct { + InstanceID string `bson:"instance_id" json:"instance_id"` + MonitorID string `bson:"monitor_id" json:"monitor_id"` + ServerID string `bson:"server_id" json:"server_id"` + Hostname string `bson:"-" json:"hostname,omitempty"` + Status string `bson:"status" json:"status"` + BreachSince *time.Time `bson:"breach_since,omitempty" json:"breach_since,omitempty"` + Value float64 `bson:"value" json:"value"` + Message string `bson:"message,omitempty" json:"message,omitempty"` + UpdatedAt time.Time `bson:"updated_at" json:"updated_at"` } // MonitorSample is one check result, kept only long enough to draw the diff --git a/server/internal/models/server.go b/server/internal/models/server.go index 2672ac3..8f6a5cd 100644 --- a/server/internal/models/server.go +++ b/server/internal/models/server.go @@ -48,6 +48,8 @@ type Inventory struct { // BootTime is the host's last reported boot time, stored on every report // that carries one so a patch reboot can be proven by a changed boot. BootTime *time.Time `bson:"boot_time,omitempty" json:"boot_time,omitempty"` + // RebootRequiredSince is when the host first reported a pending reboot. + RebootRequiredSince *time.Time `bson:"reboot_required_since,omitempty" json:"reboot_required_since,omitempty"` } type Server struct { diff --git a/server/internal/notify/dispatch.go b/server/internal/notify/dispatch.go index bd247f4..d93a97b 100644 --- a/server/internal/notify/dispatch.go +++ b/server/internal/notify/dispatch.go @@ -17,6 +17,7 @@ const TypePatch = "patch" type Event struct { MonitorName string + ServerName string Type string OldStatus string NewStatus string @@ -42,7 +43,13 @@ func (e Event) title() string { } s = fmt.Sprintf("[Vantage] Server %s %s", e.MonitorName, verb) } else { - s = fmt.Sprintf("[Vantage] %s (%s) %s", e.MonitorName, e.Type, verb) + name := e.MonitorName + if e.ServerName != "" { + name = fmt.Sprintf("%s (%s) on %s", e.MonitorName, e.Type, e.ServerName) + } else { + name = fmt.Sprintf("%s (%s)", e.MonitorName, e.Type) + } + s = fmt.Sprintf("[Vantage] %s %s", name, verb) } if e.Message != "" { s += ": " + e.Message diff --git a/server/internal/notify/dispatch_title_test.go b/server/internal/notify/dispatch_title_test.go new file mode 100644 index 0000000..0094d29 --- /dev/null +++ b/server/internal/notify/dispatch_title_test.go @@ -0,0 +1,24 @@ +package notify + +import ( + "strings" + "testing" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" +) + +func TestTitleIncludesServerName(t *testing.T) { + ev := Event{MonitorName: "Disk full", Type: models.MonitorMetric, NewStatus: models.StatusDown, ServerName: "web-01", Message: "/var 94.2% used"} + got := ev.title() + want := "[Vantage] Disk full (metric) on web-01 is DOWN: /var 94.2% used" + if got != want { + t.Fatalf("title = %q, want %q", got, want) + } +} + +func TestTitleWithoutServerNameUnchanged(t *testing.T) { + ev := Event{MonitorName: "Site", Type: models.MonitorHTTP, NewStatus: models.StatusUp} + if got := ev.title(); strings.Contains(got, " on ") || got != "[Vantage] Site (http) recovered" { + t.Fatalf("title = %q", got) + } +} diff --git a/server/internal/notify/http.go b/server/internal/notify/http.go index eb64ca3..69a7f0a 100644 --- a/server/internal/notify/http.go +++ b/server/internal/notify/http.go @@ -33,14 +33,18 @@ func dispatchWebhook(ch models.NotificationChannel, ev Event) error { if target == "" { return fmt.Errorf("webhook: missing url") } - return postJSON(target, map[string]any{ + payload := map[string]any{ "monitor": ev.MonitorName, "type": ev.Type, "old_status": ev.OldStatus, "new_status": ev.NewStatus, "message": ev.Message, "time": ev.Time.Format(time.RFC3339), - }) + } + if ev.ServerName != "" { + payload["server_name"] = ev.ServerName + } + return postJSON(target, payload) } func dispatchDiscord(ch models.NotificationChannel, ev Event) error { diff --git a/server/internal/services/monitors.go b/server/internal/services/monitors.go index 9c5afeb..07515bb 100644 --- a/server/internal/services/monitors.go +++ b/server/internal/services/monitors.go @@ -103,7 +103,8 @@ func ListServerScheduledMonitors() ([]models.Monitor, error) { func listMonitorsForRunner(instanceID, runner string) ([]models.Monitor, error) { ctx, cancel := monCtx() defer cancel() - filter := bson.M{"runner": runner, "enabled": true} + filter := bson.M{"runner": runner, "enabled": true, + "type": bson.M{"$nin": []string{models.MonitorMetric, models.MonitorHeartbeat}}} if instanceID != "" { filter["instance_id"] = instanceID } @@ -342,24 +343,7 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro now := time.Now() prev := m.State.Status - retries := m.Retries - if retries < 1 { - retries = 1 - } - - newStatus := prev - fails := m.State.Fails - if res.Up { - fails = 0 - newStatus = models.StatusUp - } else { - fails++ - if fails >= retries { - newStatus = models.StatusDown - } else if prev == "" || prev == models.StatusPending { - newStatus = models.StatusPending - } - } + newStatus, fails := decideStatus(prev, m.State.Fails, m.Retries, res.Up) state := bson.M{ "state.status": newStatus, @@ -400,31 +384,11 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro }, options.UpdateOne().SetUpsert(true)) - if newStatus != prev { - switch newStatus { - case models.StatusDown: - inc := models.Incident{ - InstanceID: m.InstanceID, - IncidentID: uuid.NewString(), - MonitorID: monitorID, - StartedAt: now, - Cause: res.Message, - } - db.Col("incidents").InsertOne(ctx, inc) - notifyTransition(m, newStatus, res.Message) - case models.StatusUp: - if prev == models.StatusDown { - db.Col("incidents").UpdateOne(ctx, - bson.M{"monitor_id": monitorID, "instance_id": m.InstanceID, "resolved_at": nil}, - bson.M{"$set": bson.M{"resolved_at": now}}) - notifyTransition(m, newStatus, res.Message) - } - } - } + applyTransition(ctx, m, "", "", prev, newStatus, res.Message, now) return nil } -func notifyTransition(m *models.Monitor, newStatus, message string) { +func notifyTransition(m *models.Monitor, serverName, oldStatus, newStatus, message string) { if len(m.ChannelIDs) == 0 { return } @@ -435,8 +399,9 @@ func notifyTransition(m *models.Monitor, newStatus, message string) { } ev := notify.Event{ MonitorName: m.Name, + ServerName: serverName, Type: m.Type, - OldStatus: m.State.Status, + OldStatus: oldStatus, NewStatus: newStatus, Message: message, Time: time.Now(), diff --git a/server/internal/services/monitortransition.go b/server/internal/services/monitortransition.go new file mode 100644 index 0000000..93ab47c --- /dev/null +++ b/server/internal/services/monitortransition.go @@ -0,0 +1,78 @@ +package services + +import ( + "context" + "log" + "time" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "github.com/google/uuid" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// decideStatus is the pull-check retry state machine: a success is always up, +// failures below the retry count leave the status alone (or pending when there +// is none yet), and reaching the count is down. +func decideStatus(prev string, fails, retries int, up bool) (string, int) { + if retries < 1 { + retries = 1 + } + if up { + return models.StatusUp, 0 + } + fails++ + if fails >= retries { + return models.StatusDown, fails + } + if prev == "" || prev == models.StatusPending { + return models.StatusPending, fails + } + return prev, fails +} + +// applyTransition is the one place a status change becomes an incident and a +// notification. serverID is empty for every monitor except a metric monitor, +// which keeps one incident per server so each breach opens and resolves on its +// own. +func applyTransition(ctx context.Context, m *models.Monitor, serverID, serverName, prev, next, message string, now time.Time) { + if next == prev { + return + } + switch next { + case models.StatusDown: + inc := models.Incident{ + InstanceID: m.InstanceID, + IncidentID: uuid.NewString(), + MonitorID: m.MonitorID, + ServerID: serverID, + StartedAt: now, + Cause: message, + } + if _, err := db.Col("incidents").InsertOne(ctx, inc); err != nil { + log.Printf("monitors: open incident for %s: %v", m.MonitorID, err) + } + notifyTransition(m, serverName, prev, next, message) + case models.StatusUp: + if prev != models.StatusDown { + return + } + resolveIncident(ctx, m, serverID, now) + notifyTransition(m, serverName, prev, next, message) + } +} + +// resolveIncident closes the open incident for a monitor (and server, for a +// metric monitor) without notifying. It is also used when a server stops +// matching a metric monitor's selector: nothing recovered, so nobody is told. +func resolveIncident(ctx context.Context, m *models.Monitor, serverID string, now time.Time) { + filter := bson.M{"monitor_id": m.MonitorID, "instance_id": m.InstanceID, "resolved_at": nil} + if serverID != "" { + filter["server_id"] = serverID + } else { + filter["server_id"] = bson.M{"$exists": false} + } + if _, err := db.Col("incidents").UpdateMany(ctx, filter, bson.M{"$set": bson.M{"resolved_at": now}}); err != nil { + log.Printf("monitors: resolve incident for %s: %v", m.MonitorID, err) + } +} diff --git a/server/internal/services/monitortransition_test.go b/server/internal/services/monitortransition_test.go new file mode 100644 index 0000000..616ce43 --- /dev/null +++ b/server/internal/services/monitortransition_test.go @@ -0,0 +1,36 @@ +package services + +import ( + "testing" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" +) + +// decideStatus is the retry state machine lifted out of ingestResult. These +// cases pin its existing behaviour so the refactor cannot change it. +func TestDecideStatus(t *testing.T) { + cases := []struct { + name string + prev string + fails int + retries int + up bool + wantState string + wantFails int + }{ + {"up resets fails", models.StatusDown, 3, 2, true, models.StatusUp, 0}, + {"first failure below retries is pending", models.StatusPending, 0, 3, false, models.StatusPending, 1}, + {"failure below retries keeps up", models.StatusUp, 0, 3, false, models.StatusUp, 1}, + {"failure reaching retries is down", models.StatusUp, 2, 3, false, models.StatusDown, 3}, + {"retries below one treated as one", models.StatusUp, 0, 0, false, models.StatusDown, 1}, + {"empty prev failing below retries is pending", "", 0, 2, false, models.StatusPending, 1}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, fails := decideStatus(c.prev, c.fails, c.retries, c.up) + if got != c.wantState || fails != c.wantFails { + t.Fatalf("got (%s,%d), want (%s,%d)", got, fails, c.wantState, c.wantFails) + } + }) + } +}