Vantage/vantage-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(monitors): extract applyTransition and add passive monitor model fields

Browse Source
This commit is contained in:
mrhid6
2026-09-17 08:14:24 +00:00
parent aaad7db09d
commit 64dbad1d20
8 changed files with 205 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/models/monitor.go
+44 -4
View File
@@ -7,12 +7,21 @@ import (
)
const (
MonitorHTTP = "http"
MonitorTCP = "tcp"
MonitorICMP = "icmp"
MonitorTLS = "tls"
MonitorHTTP = "http"
MonitorTCP = "tcp"
MonitorICMP = "icmp"
MonitorTLS = "tls"
MonitorMetric = "metric"
MonitorHeartbeat = "heartbeat"
)
// IsPassiveMonitor reports whether a monitor type is evaluated from data that
// arrives (a ping, an agent report) rather than by running a check. Passive
// monitors are never handed to monitorsched or to an agent.
func IsPassiveMonitor(t string) bool {
return t == MonitorMetric || t == MonitorHeartbeat
}
const (
StatusUp = "up"
StatusDown = "down"
@@ -38,6 +47,14 @@ type MonitorTarget struct {
Keyword string `bson:"keyword,omitempty" json:"keyword,omitempty"`
TLSWarnDays int `bson:"tls_warn_days,omitempty" json:"tls_warn_days,omitempty"`
Insecure bool `bson:"insecure,omitempty" json:"insecure,omitempty"`
// Metric monitors.
Selector map[string]string `bson:"selector,omitempty" json:"selector,omitempty"`
Metric string `bson:"metric,omitempty" json:"metric,omitempty"`
Threshold float64 `bson:"threshold,omitempty" json:"threshold,omitempty"`
Mount string `bson:"mount,omitempty" json:"mount,omitempty"`
// Heartbeat monitors.
PeriodSec int `bson:"period_sec,omitempty" json:"period_sec,omitempty"`
GraceSec int `bson:"grace_sec,omitempty" json:"grace_sec,omitempty"`
}
type MonitorState struct {
@@ -48,6 +65,8 @@ type MonitorState struct {
CertExpiryAt *time.Time `bson:"cert_expiry_at,omitempty" json:"cert_expiry_at,omitempty"`
Fails int `bson:"fails" json:"fails"`
LastNotifiedAt *time.Time `bson:"last_notified_at,omitempty" json:"last_notified_at,omitempty"`
LastPingAt *time.Time `bson:"last_ping_at,omitempty" json:"last_ping_at,omitempty"`
StartedAt *time.Time `bson:"started_at,omitempty" json:"started_at,omitempty"`
}
type Monitor struct {
@@ -68,6 +87,11 @@ type Monitor struct {
ChannelIDs []string `bson:"channel_ids,omitempty" json:"channel_ids,omitempty"`
State MonitorState `bson:"state" json:"state"`
CreatedAt time.Time `bson:"created_at" json:"created_at"`
// ForSec is how long a metric condition must hold before a server is down.
ForSec int `bson:"for_sec,omitempty" json:"for_sec,omitempty"`
// HeartbeatTokenHash is the SHA-256 of the ping token. The token itself is
// shown once, on create or rotate, and never stored.
HeartbeatTokenHash string `bson:"heartbeat_token_hash,omitempty" json:"-"`
}
type Incident struct {
@@ -77,6 +101,22 @@ type Incident struct {
StartedAt time.Time `bson:"started_at" json:"started_at"`
ResolvedAt *time.Time `bson:"resolved_at,omitempty" json:"resolved_at,omitempty"`
Cause string `bson:"cause,omitempty" json:"cause,omitempty"`
// ServerID is set only for metric monitors, which keep one incident per
// breaching server.
ServerID string `bson:"server_id,omitempty" json:"server_id,omitempty"`
}
// MonitorServerState is one metric monitor's view of one matching server.
type MonitorServerState struct {
InstanceID string `bson:"instance_id" json:"instance_id"`
MonitorID string `bson:"monitor_id" json:"monitor_id"`
ServerID string `bson:"server_id" json:"server_id"`
Hostname string `bson:"-" json:"hostname,omitempty"`
Status string `bson:"status" json:"status"`
BreachSince *time.Time `bson:"breach_since,omitempty" json:"breach_since,omitempty"`
Value float64 `bson:"value" json:"value"`
Message string `bson:"message,omitempty" json:"message,omitempty"`
UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
}
// MonitorSample is one check result, kept only long enough to draw the
server/internal/models/server.go
+2
View File
@@ -48,6 +48,8 @@ type Inventory struct {
// BootTime is the host's last reported boot time, stored on every report
// that carries one so a patch reboot can be proven by a changed boot.
BootTime *time.Time `bson:"boot_time,omitempty" json:"boot_time,omitempty"`
// RebootRequiredSince is when the host first reported a pending reboot.
RebootRequiredSince *time.Time `bson:"reboot_required_since,omitempty" json:"reboot_required_since,omitempty"`
}
type Server struct {
server/internal/notify/dispatch.go
+8 -1
View File
@@ -17,6 +17,7 @@ const TypePatch = "patch"
type Event struct {
MonitorName string
ServerName string
Type string
OldStatus string
NewStatus string
@@ -42,7 +43,13 @@ func (e Event) title() string {
}
s = fmt.Sprintf("[Vantage] Server %s %s", e.MonitorName, verb)
} else {
s = fmt.Sprintf("[Vantage] %s (%s) %s", e.MonitorName, e.Type, verb)
name := e.MonitorName
if e.ServerName != "" {
name = fmt.Sprintf("%s (%s) on %s", e.MonitorName, e.Type, e.ServerName)
} else {
name = fmt.Sprintf("%s (%s)", e.MonitorName, e.Type)
}
s = fmt.Sprintf("[Vantage] %s %s", name, verb)
}
if e.Message != "" {
s += ": " + e.Message
server/internal/notify/dispatch_title_test.go
+24
View File
@@ -0,0 +1,24 @@
package notify
import (
"strings"
"testing"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
)
func TestTitleIncludesServerName(t *testing.T) {
ev := Event{MonitorName: "Disk full", Type: models.MonitorMetric, NewStatus: models.StatusDown, ServerName: "web-01", Message: "/var 94.2% used"}
got := ev.title()
want := "[Vantage] Disk full (metric) on web-01 is DOWN: /var 94.2% used"
if got != want {
t.Fatalf("title = %q, want %q", got, want)
}
}
func TestTitleWithoutServerNameUnchanged(t *testing.T) {
ev := Event{MonitorName: "Site", Type: models.MonitorHTTP, NewStatus: models.StatusUp}
if got := ev.title(); strings.Contains(got, " on ") || got != "[Vantage] Site (http) recovered" {
t.Fatalf("title = %q", got)
}
}
server/internal/notify/http.go
+6 -2
View File
@@ -33,14 +33,18 @@ func dispatchWebhook(ch models.NotificationChannel, ev Event) error {
if target == "" {
return fmt.Errorf("webhook: missing url")
}
return postJSON(target, map[string]any{
payload := map[string]any{
"monitor": ev.MonitorName,
"type": ev.Type,
"old_status": ev.OldStatus,
"new_status": ev.NewStatus,
"message": ev.Message,
"time": ev.Time.Format(time.RFC3339),
})
}
if ev.ServerName != "" {
payload["server_name"] = ev.ServerName
}
return postJSON(target, payload)
}
func dispatchDiscord(ch models.NotificationChannel, ev Event) error {
server/internal/services/monitors.go
+7 -42
View File
@@ -103,7 +103,8 @@ func ListServerScheduledMonitors() ([]models.Monitor, error) {
func listMonitorsForRunner(instanceID, runner string) ([]models.Monitor, error) {
ctx, cancel := monCtx()
defer cancel()
filter := bson.M{"runner": runner, "enabled": true}
filter := bson.M{"runner": runner, "enabled": true,
"type": bson.M{"$nin": []string{models.MonitorMetric, models.MonitorHeartbeat}}}
if instanceID != "" {
filter["instance_id"] = instanceID
}
@@ -342,24 +343,7 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro
now := time.Now()
prev := m.State.Status
retries := m.Retries
if retries < 1 {
retries = 1
}
newStatus := prev
fails := m.State.Fails
if res.Up {
fails = 0
newStatus = models.StatusUp
} else {
fails++
if fails >= retries {
newStatus = models.StatusDown
} else if prev == "" || prev == models.StatusPending {
newStatus = models.StatusPending
}
}
newStatus, fails := decideStatus(prev, m.State.Fails, m.Retries, res.Up)
state := bson.M{
"state.status": newStatus,
@@ -400,31 +384,11 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro
},
options.UpdateOne().SetUpsert(true))
if newStatus != prev {
switch newStatus {
case models.StatusDown:
inc := models.Incident{
InstanceID: m.InstanceID,
IncidentID: uuid.NewString(),
MonitorID: monitorID,
StartedAt: now,
Cause: res.Message,
}
db.Col("incidents").InsertOne(ctx, inc)
notifyTransition(m, newStatus, res.Message)
case models.StatusUp:
if prev == models.StatusDown {
db.Col("incidents").UpdateOne(ctx,
bson.M{"monitor_id": monitorID, "instance_id": m.InstanceID, "resolved_at": nil},
bson.M{"$set": bson.M{"resolved_at": now}})
notifyTransition(m, newStatus, res.Message)
}
}
}
applyTransition(ctx, m, "", "", prev, newStatus, res.Message, now)
return nil
}
func notifyTransition(m *models.Monitor, newStatus, message string) {
func notifyTransition(m *models.Monitor, serverName, oldStatus, newStatus, message string) {
if len(m.ChannelIDs) == 0 {
return
}
@@ -435,8 +399,9 @@ func notifyTransition(m *models.Monitor, newStatus, message string) {
}
ev := notify.Event{
MonitorName: m.Name,
ServerName: serverName,
Type: m.Type,
OldStatus: m.State.Status,
OldStatus: oldStatus,
NewStatus: newStatus,
Message: message,
Time: time.Now(),
server/internal/services/monitortransition.go
+78
View File
@@ -0,0 +1,78 @@
package services
import (
"context"
"log"
"time"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/v2/bson"
)
// decideStatus is the pull-check retry state machine: a success is always up,
// failures below the retry count leave the status alone (or pending when there
// is none yet), and reaching the count is down.
func decideStatus(prev string, fails, retries int, up bool) (string, int) {
if retries < 1 {
retries = 1
}
if up {
return models.StatusUp, 0
}
fails++
if fails >= retries {
return models.StatusDown, fails
}
if prev == "" || prev == models.StatusPending {
return models.StatusPending, fails
}
return prev, fails
}
// applyTransition is the one place a status change becomes an incident and a
// notification. serverID is empty for every monitor except a metric monitor,
// which keeps one incident per server so each breach opens and resolves on its
// own.
func applyTransition(ctx context.Context, m *models.Monitor, serverID, serverName, prev, next, message string, now time.Time) {
if next == prev {
return
}
switch next {
case models.StatusDown:
inc := models.Incident{
InstanceID: m.InstanceID,
IncidentID: uuid.NewString(),
MonitorID: m.MonitorID,
ServerID: serverID,
StartedAt: now,
Cause: message,
}
if _, err := db.Col("incidents").InsertOne(ctx, inc); err != nil {
log.Printf("monitors: open incident for %s: %v", m.MonitorID, err)
}
notifyTransition(m, serverName, prev, next, message)
case models.StatusUp:
if prev != models.StatusDown {
return
}
resolveIncident(ctx, m, serverID, now)
notifyTransition(m, serverName, prev, next, message)
}
}
// resolveIncident closes the open incident for a monitor (and server, for a
// metric monitor) without notifying. It is also used when a server stops
// matching a metric monitor's selector: nothing recovered, so nobody is told.
func resolveIncident(ctx context.Context, m *models.Monitor, serverID string, now time.Time) {
filter := bson.M{"monitor_id": m.MonitorID, "instance_id": m.InstanceID, "resolved_at": nil}
if serverID != "" {
filter["server_id"] = serverID
} else {
filter["server_id"] = bson.M{"$exists": false}
}
if _, err := db.Col("incidents").UpdateMany(ctx, filter, bson.M{"$set": bson.M{"resolved_at": now}}); err != nil {
log.Printf("monitors: resolve incident for %s: %v", m.MonitorID, err)
}
}
server/internal/services/monitortransition_test.go
+36
View File
@@ -0,0 +1,36 @@
package services
import (
"testing"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
)
// decideStatus is the retry state machine lifted out of ingestResult. These
// cases pin its existing behaviour so the refactor cannot change it.
func TestDecideStatus(t *testing.T) {
cases := []struct {
name string
prev string
fails int
retries int
up bool
wantState string
wantFails int
}{
{"up resets fails", models.StatusDown, 3, 2, true, models.StatusUp, 0},
{"first failure below retries is pending", models.StatusPending, 0, 3, false, models.StatusPending, 1},
{"failure below retries keeps up", models.StatusUp, 0, 3, false, models.StatusUp, 1},
{"failure reaching retries is down", models.StatusUp, 2, 3, false, models.StatusDown, 3},
{"retries below one treated as one", models.StatusUp, 0, 0, false, models.StatusDown, 1},
{"empty prev failing below retries is pending", "", 0, 2, false, models.StatusPending, 1},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, fails := decideStatus(c.prev, c.fails, c.retries, c.up)
if got != c.wantState || fails != c.wantFails {
t.Fatalf("got (%s,%d), want (%s,%d)", got, fails, c.wantState, c.wantFails)
}
})
}
}