Vantage/vantage-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(monitors): extract applyTransition and add passive monitor model fields

Browse Source
This commit is contained in:
mrhid6
2026-09-17 08:14:24 +00:00
parent aaad7db09d
commit 64dbad1d20
8 changed files with 205 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/services/monitors.go
+7 -42
View File
@@ -103,7 +103,8 @@ func ListServerScheduledMonitors() ([]models.Monitor, error) {
func listMonitorsForRunner(instanceID, runner string) ([]models.Monitor, error) {
ctx, cancel := monCtx()
defer cancel()
filter := bson.M{"runner": runner, "enabled": true}
filter := bson.M{"runner": runner, "enabled": true,
"type": bson.M{"$nin": []string{models.MonitorMetric, models.MonitorHeartbeat}}}
if instanceID != "" {
filter["instance_id"] = instanceID
}
@@ -342,24 +343,7 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro
now := time.Now()
prev := m.State.Status
retries := m.Retries
if retries < 1 {
retries = 1
}
newStatus := prev
fails := m.State.Fails
if res.Up {
fails = 0
newStatus = models.StatusUp
} else {
fails++
if fails >= retries {
newStatus = models.StatusDown
} else if prev == "" || prev == models.StatusPending {
newStatus = models.StatusPending
}
}
newStatus, fails := decideStatus(prev, m.State.Fails, m.Retries, res.Up)
state := bson.M{
"state.status": newStatus,
@@ -400,31 +384,11 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro
},
options.UpdateOne().SetUpsert(true))
if newStatus != prev {
switch newStatus {
case models.StatusDown:
inc := models.Incident{
InstanceID: m.InstanceID,
IncidentID: uuid.NewString(),
MonitorID: monitorID,
StartedAt: now,
Cause: res.Message,
}
db.Col("incidents").InsertOne(ctx, inc)
notifyTransition(m, newStatus, res.Message)
case models.StatusUp:
if prev == models.StatusDown {
db.Col("incidents").UpdateOne(ctx,
bson.M{"monitor_id": monitorID, "instance_id": m.InstanceID, "resolved_at": nil},
bson.M{"$set": bson.M{"resolved_at": now}})
notifyTransition(m, newStatus, res.Message)
}
}
}
applyTransition(ctx, m, "", "", prev, newStatus, res.Message, now)
return nil
}
func notifyTransition(m *models.Monitor, newStatus, message string) {
func notifyTransition(m *models.Monitor, serverName, oldStatus, newStatus, message string) {
if len(m.ChannelIDs) == 0 {
return
}
@@ -435,8 +399,9 @@ func notifyTransition(m *models.Monitor, newStatus, message string) {
}
ev := notify.Event{
MonitorName: m.Name,
ServerName: serverName,
Type: m.Type,
OldStatus: m.State.Status,
OldStatus: oldStatus,
NewStatus: newStatus,
Message: message,
Time: time.Now(),
server/internal/services/monitortransition.go
+78
View File
@@ -0,0 +1,78 @@
package services
import (
"context"
"log"
"time"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/v2/bson"
)
// decideStatus is the pull-check retry state machine: a success is always up,
// failures below the retry count leave the status alone (or pending when there
// is none yet), and reaching the count is down.
func decideStatus(prev string, fails, retries int, up bool) (string, int) {
if retries < 1 {
retries = 1
}
if up {
return models.StatusUp, 0
}
fails++
if fails >= retries {
return models.StatusDown, fails
}
if prev == "" || prev == models.StatusPending {
return models.StatusPending, fails
}
return prev, fails
}
// applyTransition is the one place a status change becomes an incident and a
// notification. serverID is empty for every monitor except a metric monitor,
// which keeps one incident per server so each breach opens and resolves on its
// own.
func applyTransition(ctx context.Context, m *models.Monitor, serverID, serverName, prev, next, message string, now time.Time) {
if next == prev {
return
}
switch next {
case models.StatusDown:
inc := models.Incident{
InstanceID: m.InstanceID,
IncidentID: uuid.NewString(),
MonitorID: m.MonitorID,
ServerID: serverID,
StartedAt: now,
Cause: message,
}
if _, err := db.Col("incidents").InsertOne(ctx, inc); err != nil {
log.Printf("monitors: open incident for %s: %v", m.MonitorID, err)
}
notifyTransition(m, serverName, prev, next, message)
case models.StatusUp:
if prev != models.StatusDown {
return
}
resolveIncident(ctx, m, serverID, now)
notifyTransition(m, serverName, prev, next, message)
}
}
// resolveIncident closes the open incident for a monitor (and server, for a
// metric monitor) without notifying. It is also used when a server stops
// matching a metric monitor's selector: nothing recovered, so nobody is told.
func resolveIncident(ctx context.Context, m *models.Monitor, serverID string, now time.Time) {
filter := bson.M{"monitor_id": m.MonitorID, "instance_id": m.InstanceID, "resolved_at": nil}
if serverID != "" {
filter["server_id"] = serverID
} else {
filter["server_id"] = bson.M{"$exists": false}
}
if _, err := db.Col("incidents").UpdateMany(ctx, filter, bson.M{"$set": bson.M{"resolved_at": now}}); err != nil {
log.Printf("monitors: resolve incident for %s: %v", m.MonitorID, err)
}
}
server/internal/services/monitortransition_test.go
+36
View File
@@ -0,0 +1,36 @@
package services
import (
"testing"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
)
// decideStatus is the retry state machine lifted out of ingestResult. These
// cases pin its existing behaviour so the refactor cannot change it.
func TestDecideStatus(t *testing.T) {
cases := []struct {
name string
prev string
fails int
retries int
up bool
wantState string
wantFails int
}{
{"up resets fails", models.StatusDown, 3, 2, true, models.StatusUp, 0},
{"first failure below retries is pending", models.StatusPending, 0, 3, false, models.StatusPending, 1},
{"failure below retries keeps up", models.StatusUp, 0, 3, false, models.StatusUp, 1},
{"failure reaching retries is down", models.StatusUp, 2, 3, false, models.StatusDown, 3},
{"retries below one treated as one", models.StatusUp, 0, 0, false, models.StatusDown, 1},
{"empty prev failing below retries is pending", "", 0, 2, false, models.StatusPending, 1},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, fails := decideStatus(c.prev, c.fails, c.retries, c.up)
if got != c.wantState || fails != c.wantFails {
t.Fatalf("got (%s,%d), want (%s,%d)", got, fails, c.wantState, c.wantFails)
}
})
}
}