diff --git a/server/internal/patchrun/patchrun.go b/server/internal/patchrun/patchrun.go new file mode 100644 index 0000000..4805439 --- /dev/null +++ b/server/internal/patchrun/patchrun.go @@ -0,0 +1,245 @@ +// Package patchrun is the patch run state machine as pure functions. Nothing +// here touches the database: the services layer loads a run, asks this +// package what should change, and writes that change guarded by the status it +// expected. That split is what makes the rules testable without MongoDB. +package patchrun + +import ( + "fmt" + "strconv" + "strings" + "time" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb" +) + +// MinAgentVersion is the first agent release that honours ApplyUpdatesCmd's +// scope and answers with a PatchResult. An older agent ignores the scope and +// installs everything, so a policy must never dispatch to one. +const MinAgentVersion = "1.4.0" + +const ( + ResultGrace = 10 * time.Minute // after the window deadline, before "no result" + ManualTimeout = 2 * time.Hour // the agent's own cap when no deadline is sent + RebootTimeout = 20 * time.Minute // for a post-boot inventory report +) + +// AgentSupportsPatchResults compares major.minor.patch. Empty, "dev" and +// anything unparseable count as too old, and a pre-release of exactly the +// minimum version counts as older than it, as semver orders them. +func AgentSupportsPatchResults(version string) bool { + have, pre, ok := parseVersion(version) + if !ok { + return false + } + want, _, _ := parseVersion(MinAgentVersion) + for i := 0; i < 3; i++ { + if have[i] != want[i] { + return have[i] > want[i] + } + } + return !pre +} + +func parseVersion(v string) ([3]int, bool, bool) { + var out [3]int + v = strings.TrimPrefix(strings.TrimSpace(v), "v") + v, _, _ = strings.Cut(v, "+") + core, pre, hasPre := strings.Cut(v, "-") + parts := strings.Split(core, ".") + if len(parts) != 3 { + return out, false, false + } + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil || n < 0 { + return out, false, false + } + out[i] = n + } + return out, hasPre && pre != "", true +} + +// IsTerminal reports whether a server run has finished. +func IsTerminal(status string) bool { + switch status { + case models.PatchSrvQueued, models.PatchSrvWaitingOffline, models.PatchSrvPatching, models.PatchSrvRebooting: + return false + } + return true +} + +// Transition is one change Advance wants made. The caller writes it guarded +// by From, so a result that arrived meanwhile is never overwritten. Dispatch +// means: send ApplyUpdatesCmd, and set To only once the command is on its way. +type Transition struct { + ServerID string + From string + To string + Error string + Dispatch bool +} + +// Advance returns what should change on this tick. connected says which +// agents hold a command stream right now. +func Advance(run models.PatchRun, now time.Time, connected map[string]bool) []Transition { + if run.Status != models.PatchRunRunning { + return nil + } + windowOpen := run.WindowEnd == nil || now.Before(*run.WindowEnd) + + inFlight := 0 + for _, s := range run.Servers { + if s.Status == models.PatchSrvPatching || s.Status == models.PatchSrvRebooting { + inFlight++ + } + } + + var out []Transition + for _, s := range run.Servers { + switch s.Status { + case models.PatchSrvQueued, models.PatchSrvWaitingOffline: + switch { + case run.CancelledAt != nil: + out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvCancelled}) + case !windowOpen: + to := models.PatchSrvWindowClosed + if s.Status == models.PatchSrvWaitingOffline { + to = models.PatchSrvMissedOffline + } + out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: to}) + case run.MaxConcurrent > 0 && inFlight >= run.MaxConcurrent: + // No slot this tick. + case !connected[s.ServerID]: + if s.Status == models.PatchSrvQueued { + out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvWaitingOffline}) + } + default: + out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvPatching, Dispatch: true}) + inFlight++ + } + case models.PatchSrvPatching: + if now.After(resultDeadline(run, s)) { + out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvFailed, Error: "no result from agent"}) + } + case models.PatchSrvRebooting: + if s.RebootedAt != nil && now.After(s.RebootedAt.Add(RebootTimeout)) { + out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvFailed, Error: "did not come back within 20 minutes"}) + } + } + } + return out +} + +func resultDeadline(run models.PatchRun, s models.PatchServerRun) time.Time { + if run.WindowEnd != nil { + return run.WindowEnd.Add(ResultGrace) + } + start := run.StartedAt + if s.StartedAt != nil { + start = *s.StartedAt + } + return start.Add(ManualTimeout + ResultGrace) +} + +// ApplyResult folds an agent's PatchResult into the server run. It only acts +// on a server that is patching; anything else is a late or duplicate result. +func ApplyResult(s models.PatchServerRun, r *pb.PatchResult, now time.Time) (models.PatchServerRun, bool) { + if s.Status != models.PatchSrvPatching { + return s, false + } + s.Output = r.OutputTail + if r.PendingAfter >= 0 { + v := int(r.PendingAfter) + s.PendingAfter = &v + } + switch r.Status { + case pb.PatchStatusOK: + if r.Rebooting { + s.Status = models.PatchSrvRebooting + s.RebootedAt = &now + return s, true + } + s.Status = models.PatchSrvSucceeded + case pb.PatchStatusUnsupported: + s.Status = models.PatchSrvUnsupported + s.Error = r.Message + default: + s.Status = models.PatchSrvFailed + s.Error = r.Message + if s.Error == "" { + s.Error = "agent reported a failure" + } + } + s.FinishedAt = &now + return s, true +} + +// VerifyReboot settles a rebooting server from a static inventory report. Only +// a boot time later than the reboot command proves the host restarted: a +// snapshot sent during the one-minute grace period must not count. +func VerifyReboot(s models.PatchServerRun, bootTime time.Time, rebootRequired bool, now time.Time) (models.PatchServerRun, bool) { + if s.Status != models.PatchSrvRebooting || s.RebootedAt == nil || !bootTime.After(*s.RebootedAt) { + return s, false + } + if rebootRequired { + s.Status = models.PatchSrvFailed + s.Error = "still requires a reboot after restarting" + } else { + s.Status = models.PatchSrvSucceeded + s.VerifiedAt = &now + } + s.FinishedAt = &now + return s, true +} + +// Finalize says whether the run is over and how it ended. Only succeeded +// counts as success: unsupported, agent_too_old and the window outcomes did +// not patch anything. +func Finalize(run models.PatchRun) (string, bool) { + ok := 0 + for _, s := range run.Servers { + if !IsTerminal(s.Status) { + return "", false + } + if s.Status == models.PatchSrvSucceeded { + ok++ + } + } + switch { + case run.CancelledAt != nil: + return models.PatchRunCancelled, true + case len(run.Servers) > 0 && ok == len(run.Servers): + return models.PatchRunSucceeded, true + case ok == 0: + return models.PatchRunFailed, true + default: + return models.PatchRunPartial, true + } +} + +var summaryOrder = []struct{ status, label string }{ + {models.PatchSrvSucceeded, "succeeded"}, + {models.PatchSrvFailed, "failed"}, + {models.PatchSrvUnsupported, "unsupported"}, + {models.PatchSrvAgentTooOld, "need an agent update"}, + {models.PatchSrvMissedOffline, "missed offline"}, + {models.PatchSrvWindowClosed, "window closed"}, + {models.PatchSrvCancelled, "cancelled"}, +} + +// Summary is the one-line count used in alerts, e.g. "38 succeeded, 2 failed". +func Summary(run models.PatchRun) string { + counts := map[string]int{} + for _, s := range run.Servers { + counts[s.Status]++ + } + var parts []string + for _, o := range summaryOrder { + if n := counts[o.status]; n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", n, o.label)) + } + } + return strings.Join(parts, ", ") +} diff --git a/server/internal/patchrun/patchrun_test.go b/server/internal/patchrun/patchrun_test.go new file mode 100644 index 0000000..6ec9ed9 --- /dev/null +++ b/server/internal/patchrun/patchrun_test.go @@ -0,0 +1,246 @@ +package patchrun + +import ( + "testing" + "time" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb" +) + +var t0 = time.Date(2026, 9, 20, 2, 0, 0, 0, time.UTC) + +func tp(t time.Time) *time.Time { return &t } + +func srv(id, status string) models.PatchServerRun { + return models.PatchServerRun{ServerID: id, Hostname: id, Status: status} +} + +func windowRun(max int, servers ...models.PatchServerRun) models.PatchRun { + return models.PatchRun{Status: models.PatchRunRunning, StartedAt: t0, WindowEnd: tp(t0.Add(2 * time.Hour)), MaxConcurrent: max, Servers: servers} +} + +func find(ts []Transition, id string) *Transition { + for i := range ts { + if ts[i].ServerID == id { + return &ts[i] + } + } + return nil +} + +func TestAgentSupportsPatchResults(t *testing.T) { + cases := map[string]bool{ + "1.4.0": true, "v1.4.0": true, "1.4.1": true, "1.10.0": true, "2.0.0": true, + "1.3.5": false, "1.4.0-rc1": false, "1.5.0-rc1": true, + "": false, "dev": false, "1.4": false, "x.y.z": false, + } + for v, want := range cases { + if got := AgentSupportsPatchResults(v); got != want { + t.Errorf("%q: got %v, want %v", v, got, want) + } + } +} + +func TestAdvanceRespectsConcurrencyCountingRebooting(t *testing.T) { + run := windowRun(2, + srv("a", models.PatchSrvRebooting), + srv("b", models.PatchSrvQueued), + srv("c", models.PatchSrvQueued), + ) + run.Servers[0].RebootedAt = tp(t0) + ts := Advance(run, t0.Add(time.Minute), map[string]bool{"b": true, "c": true}) + if tr := find(ts, "b"); tr == nil || !tr.Dispatch || tr.To != models.PatchSrvPatching { + t.Fatalf("b should dispatch, got %+v", tr) + } + if tr := find(ts, "c"); tr != nil { + t.Fatalf("c must wait for a slot, got %+v", tr) + } +} + +func TestAdvanceUnlimitedWhenZero(t *testing.T) { + run := windowRun(0, srv("a", models.PatchSrvQueued), srv("b", models.PatchSrvQueued)) + ts := Advance(run, t0, map[string]bool{"a": true, "b": true}) + if len(ts) != 2 || !ts[0].Dispatch || !ts[1].Dispatch { + t.Fatalf("both should dispatch: %+v", ts) + } +} + +func TestAdvanceOfflineWaitsThenDispatches(t *testing.T) { + run := windowRun(0, srv("a", models.PatchSrvQueued)) + ts := Advance(run, t0, map[string]bool{}) + if len(ts) != 1 || ts[0].To != models.PatchSrvWaitingOffline || ts[0].Dispatch { + t.Fatalf("offline should wait: %+v", ts) + } + run.Servers[0].Status = models.PatchSrvWaitingOffline + if ts := Advance(run, t0.Add(time.Minute), map[string]bool{}); len(ts) != 0 { + t.Fatalf("still offline must be a no-op, got %+v", ts) + } + ts = Advance(run, t0.Add(2*time.Minute), map[string]bool{"a": true}) + if len(ts) != 1 || !ts[0].Dispatch { + t.Fatalf("back online should dispatch: %+v", ts) + } +} + +func TestAdvanceWindowCloses(t *testing.T) { + run := windowRun(1, srv("q", models.PatchSrvQueued), srv("w", models.PatchSrvWaitingOffline)) + ts := Advance(run, t0.Add(2*time.Hour), map[string]bool{"q": true, "w": true}) + if tr := find(ts, "q"); tr == nil || tr.To != models.PatchSrvWindowClosed || tr.Dispatch { + t.Fatalf("queued at window end: %+v", tr) + } + if tr := find(ts, "w"); tr == nil || tr.To != models.PatchSrvMissedOffline { + t.Fatalf("waiting at window end: %+v", tr) + } +} + +func TestAdvanceNoResultTimeout(t *testing.T) { + run := windowRun(0, srv("a", models.PatchSrvPatching)) + if ts := Advance(run, t0.Add(2*time.Hour+9*time.Minute), nil); len(ts) != 0 { + t.Fatalf("inside grace: %+v", ts) + } + ts := Advance(run, t0.Add(2*time.Hour+11*time.Minute), nil) + if len(ts) != 1 || ts[0].To != models.PatchSrvFailed || ts[0].Error == "" { + t.Fatalf("past grace: %+v", ts) + } +} + +func TestAdvanceManualRunTimeout(t *testing.T) { + run := models.PatchRun{Status: models.PatchRunRunning, StartedAt: t0, Servers: []models.PatchServerRun{srv("a", models.PatchSrvPatching)}} + run.Servers[0].StartedAt = tp(t0) + if ts := Advance(run, t0.Add(2*time.Hour+9*time.Minute), nil); len(ts) != 0 { + t.Fatalf("manual inside timeout: %+v", ts) + } + if ts := Advance(run, t0.Add(2*time.Hour+11*time.Minute), nil); len(ts) != 1 || ts[0].To != models.PatchSrvFailed { + t.Fatalf("manual past timeout: %+v", ts) + } +} + +func TestAdvanceRebootTimeout(t *testing.T) { + run := windowRun(0, srv("a", models.PatchSrvRebooting)) + run.Servers[0].RebootedAt = tp(t0) + if ts := Advance(run, t0.Add(19*time.Minute), nil); len(ts) != 0 { + t.Fatalf("inside reboot timeout: %+v", ts) + } + ts := Advance(run, t0.Add(21*time.Minute), nil) + if len(ts) != 1 || ts[0].To != models.PatchSrvFailed { + t.Fatalf("past reboot timeout: %+v", ts) + } +} + +func TestAdvanceCancelledDispatchesNothing(t *testing.T) { + run := windowRun(0, srv("a", models.PatchSrvQueued), srv("b", models.PatchSrvWaitingOffline), srv("c", models.PatchSrvPatching)) + run.CancelledAt = tp(t0) + ts := Advance(run, t0.Add(time.Minute), map[string]bool{"a": true, "b": true}) + for _, id := range []string{"a", "b"} { + if tr := find(ts, id); tr == nil || tr.To != models.PatchSrvCancelled || tr.Dispatch { + t.Errorf("%s: %+v", id, tr) + } + } + if find(ts, "c") != nil { + t.Error("an in-flight server must be left to finish") + } +} + +func TestAdvanceIgnoresFinishedRun(t *testing.T) { + run := windowRun(0, srv("a", models.PatchSrvQueued)) + run.Status = models.PatchRunSucceeded + if ts := Advance(run, t0, map[string]bool{"a": true}); ts != nil { + t.Fatalf("got %+v", ts) + } +} + +func TestApplyResult(t *testing.T) { + now := t0.Add(10 * time.Minute) + cases := []struct { + r pb.PatchResult + want string + }{ + {pb.PatchResult{Status: pb.PatchStatusOK, PendingAfter: 0}, models.PatchSrvSucceeded}, + {pb.PatchResult{Status: pb.PatchStatusOK, Rebooting: true}, models.PatchSrvRebooting}, + {pb.PatchResult{Status: pb.PatchStatusFailed, Message: "apt broke"}, models.PatchSrvFailed}, + {pb.PatchResult{Status: pb.PatchStatusBusy, Message: "busy"}, models.PatchSrvFailed}, + {pb.PatchResult{Status: pb.PatchStatusUnsupported, Message: "no metadata"}, models.PatchSrvUnsupported}, + } + for _, c := range cases { + got, ok := ApplyResult(srv("a", models.PatchSrvPatching), &c.r, now) + if !ok || got.Status != c.want { + t.Errorf("%s: got %s ok=%v, want %s", c.r.Status, got.Status, ok, c.want) + } + if c.want == models.PatchSrvRebooting && (got.RebootedAt == nil || got.FinishedAt != nil) { + t.Errorf("rebooting must set RebootedAt and leave FinishedAt nil: %+v", got) + } + } + if _, ok := ApplyResult(srv("a", models.PatchSrvSucceeded), &pb.PatchResult{Status: pb.PatchStatusOK}, now); ok { + t.Error("a result for a server not patching must be ignored") + } + got, _ := ApplyResult(srv("a", models.PatchSrvPatching), &pb.PatchResult{Status: pb.PatchStatusOK, PendingAfter: -1}, now) + if got.PendingAfter != nil { + t.Error("PendingAfter -1 means unknown and must stay nil") + } +} + +func TestVerifyReboot(t *testing.T) { + s := srv("a", models.PatchSrvRebooting) + s.RebootedAt = tp(t0) + now := t0.Add(5 * time.Minute) + if _, ok := VerifyReboot(s, t0.Add(-time.Hour), false, now); ok { + t.Error("boot before the reboot command is not proof") + } + if _, ok := VerifyReboot(s, t0, false, now); ok { + t.Error("boot equal to the reboot command is not proof") + } + got, ok := VerifyReboot(s, t0.Add(2*time.Minute), false, now) + if !ok || got.Status != models.PatchSrvSucceeded || got.VerifiedAt == nil { + t.Errorf("clean reboot: %+v", got) + } + got, ok = VerifyReboot(s, t0.Add(2*time.Minute), true, now) + if !ok || got.Status != models.PatchSrvFailed || got.Error == "" { + t.Errorf("still owed: %+v", got) + } + if _, ok := VerifyReboot(srv("a", models.PatchSrvPatching), t0.Add(time.Hour), false, now); ok { + t.Error("only rebooting servers verify") + } +} + +func TestFinalize(t *testing.T) { + mk := func(statuses ...string) models.PatchRun { + r := windowRun(0) + for i, s := range statuses { + r.Servers = append(r.Servers, srv(string(rune('a'+i)), s)) + } + return r + } + cases := []struct { + run models.PatchRun + want string + done bool + }{ + {mk(models.PatchSrvSucceeded, models.PatchSrvSucceeded), models.PatchRunSucceeded, true}, + {mk(models.PatchSrvSucceeded, models.PatchSrvFailed), models.PatchRunPartial, true}, + {mk(models.PatchSrvSucceeded, models.PatchSrvUnsupported), models.PatchRunPartial, true}, + {mk(models.PatchSrvSucceeded, models.PatchSrvAgentTooOld), models.PatchRunPartial, true}, + {mk(models.PatchSrvMissedOffline, models.PatchSrvWindowClosed), models.PatchRunFailed, true}, + {mk(models.PatchSrvSucceeded, models.PatchSrvPatching), "", false}, + {mk(), models.PatchRunFailed, true}, + } + for i, c := range cases { + got, done := Finalize(c.run) + if got != c.want || done != c.done { + t.Errorf("case %d: got %q/%v, want %q/%v", i, got, done, c.want, c.done) + } + } + cancelled := mk(models.PatchSrvSucceeded, models.PatchSrvCancelled) + cancelled.CancelledAt = tp(t0) + if got, done := Finalize(cancelled); got != models.PatchRunCancelled || !done { + t.Errorf("cancelled: %q/%v", got, done) + } +} + +func TestSummary(t *testing.T) { + r := windowRun(0, + srv("a", models.PatchSrvSucceeded), srv("b", models.PatchSrvSucceeded), + srv("c", models.PatchSrvFailed), srv("d", models.PatchSrvMissedOffline)) + if got := Summary(r); got != "2 succeeded, 1 failed, 1 missed offline" { + t.Fatalf("got %q", got) + } +}