feat: patchrun - pure state machine for patch runs

Implements the patchrun package with a pure functional state machine for managing
patch runs. Contains no database dependencies - the services layer loads a run,
asks this package what should change, and writes changes guarded by expected status.

All 14 test cases pass, covering:
- Agent version parsing and support detection
- Concurrency limits and queueing
- Window deadlines and offline handling
- Result timeouts (ResultGrace, ManualTimeout, RebootTimeout)
- Reboot verification with boot time proof
- Run finalization logic
- Summary generation for alerts
This commit is contained in:
2026-09-15 08:39:21 +00:00
parent 8f1ea6d5a0
commit b60daf0461
2 changed files with 491 additions and 0 deletions
+245
View File
@@ -0,0 +1,245 @@
// Package patchrun is the patch run state machine as pure functions. Nothing
// here touches the database: the services layer loads a run, asks this
// package what should change, and writes that change guarded by the status it
// expected. That split is what makes the rules testable without MongoDB.
package patchrun
import (
"fmt"
"strconv"
"strings"
"time"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
)
// MinAgentVersion is the first agent release that honours ApplyUpdatesCmd's
// scope and answers with a PatchResult. An older agent ignores the scope and
// installs everything, so a policy must never dispatch to one.
const MinAgentVersion = "1.4.0"
const (
ResultGrace = 10 * time.Minute // after the window deadline, before "no result"
ManualTimeout = 2 * time.Hour // the agent's own cap when no deadline is sent
RebootTimeout = 20 * time.Minute // for a post-boot inventory report
)
// AgentSupportsPatchResults compares major.minor.patch. Empty, "dev" and
// anything unparseable count as too old, and a pre-release of exactly the
// minimum version counts as older than it, as semver orders them.
func AgentSupportsPatchResults(version string) bool {
have, pre, ok := parseVersion(version)
if !ok {
return false
}
want, _, _ := parseVersion(MinAgentVersion)
for i := 0; i < 3; i++ {
if have[i] != want[i] {
return have[i] > want[i]
}
}
return !pre
}
func parseVersion(v string) ([3]int, bool, bool) {
var out [3]int
v = strings.TrimPrefix(strings.TrimSpace(v), "v")
v, _, _ = strings.Cut(v, "+")
core, pre, hasPre := strings.Cut(v, "-")
parts := strings.Split(core, ".")
if len(parts) != 3 {
return out, false, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false, false
}
out[i] = n
}
return out, hasPre && pre != "", true
}
// IsTerminal reports whether a server run has finished.
func IsTerminal(status string) bool {
switch status {
case models.PatchSrvQueued, models.PatchSrvWaitingOffline, models.PatchSrvPatching, models.PatchSrvRebooting:
return false
}
return true
}
// Transition is one change Advance wants made. The caller writes it guarded
// by From, so a result that arrived meanwhile is never overwritten. Dispatch
// means: send ApplyUpdatesCmd, and set To only once the command is on its way.
type Transition struct {
ServerID string
From string
To string
Error string
Dispatch bool
}
// Advance returns what should change on this tick. connected says which
// agents hold a command stream right now.
func Advance(run models.PatchRun, now time.Time, connected map[string]bool) []Transition {
if run.Status != models.PatchRunRunning {
return nil
}
windowOpen := run.WindowEnd == nil || now.Before(*run.WindowEnd)
inFlight := 0
for _, s := range run.Servers {
if s.Status == models.PatchSrvPatching || s.Status == models.PatchSrvRebooting {
inFlight++
}
}
var out []Transition
for _, s := range run.Servers {
switch s.Status {
case models.PatchSrvQueued, models.PatchSrvWaitingOffline:
switch {
case run.CancelledAt != nil:
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvCancelled})
case !windowOpen:
to := models.PatchSrvWindowClosed
if s.Status == models.PatchSrvWaitingOffline {
to = models.PatchSrvMissedOffline
}
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: to})
case run.MaxConcurrent > 0 && inFlight >= run.MaxConcurrent:
// No slot this tick.
case !connected[s.ServerID]:
if s.Status == models.PatchSrvQueued {
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvWaitingOffline})
}
default:
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvPatching, Dispatch: true})
inFlight++
}
case models.PatchSrvPatching:
if now.After(resultDeadline(run, s)) {
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvFailed, Error: "no result from agent"})
}
case models.PatchSrvRebooting:
if s.RebootedAt != nil && now.After(s.RebootedAt.Add(RebootTimeout)) {
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvFailed, Error: "did not come back within 20 minutes"})
}
}
}
return out
}
func resultDeadline(run models.PatchRun, s models.PatchServerRun) time.Time {
if run.WindowEnd != nil {
return run.WindowEnd.Add(ResultGrace)
}
start := run.StartedAt
if s.StartedAt != nil {
start = *s.StartedAt
}
return start.Add(ManualTimeout + ResultGrace)
}
// ApplyResult folds an agent's PatchResult into the server run. It only acts
// on a server that is patching; anything else is a late or duplicate result.
func ApplyResult(s models.PatchServerRun, r *pb.PatchResult, now time.Time) (models.PatchServerRun, bool) {
if s.Status != models.PatchSrvPatching {
return s, false
}
s.Output = r.OutputTail
if r.PendingAfter >= 0 {
v := int(r.PendingAfter)
s.PendingAfter = &v
}
switch r.Status {
case pb.PatchStatusOK:
if r.Rebooting {
s.Status = models.PatchSrvRebooting
s.RebootedAt = &now
return s, true
}
s.Status = models.PatchSrvSucceeded
case pb.PatchStatusUnsupported:
s.Status = models.PatchSrvUnsupported
s.Error = r.Message
default:
s.Status = models.PatchSrvFailed
s.Error = r.Message
if s.Error == "" {
s.Error = "agent reported a failure"
}
}
s.FinishedAt = &now
return s, true
}
// VerifyReboot settles a rebooting server from a static inventory report. Only
// a boot time later than the reboot command proves the host restarted: a
// snapshot sent during the one-minute grace period must not count.
func VerifyReboot(s models.PatchServerRun, bootTime time.Time, rebootRequired bool, now time.Time) (models.PatchServerRun, bool) {
if s.Status != models.PatchSrvRebooting || s.RebootedAt == nil || !bootTime.After(*s.RebootedAt) {
return s, false
}
if rebootRequired {
s.Status = models.PatchSrvFailed
s.Error = "still requires a reboot after restarting"
} else {
s.Status = models.PatchSrvSucceeded
s.VerifiedAt = &now
}
s.FinishedAt = &now
return s, true
}
// Finalize says whether the run is over and how it ended. Only succeeded
// counts as success: unsupported, agent_too_old and the window outcomes did
// not patch anything.
func Finalize(run models.PatchRun) (string, bool) {
ok := 0
for _, s := range run.Servers {
if !IsTerminal(s.Status) {
return "", false
}
if s.Status == models.PatchSrvSucceeded {
ok++
}
}
switch {
case run.CancelledAt != nil:
return models.PatchRunCancelled, true
case len(run.Servers) > 0 && ok == len(run.Servers):
return models.PatchRunSucceeded, true
case ok == 0:
return models.PatchRunFailed, true
default:
return models.PatchRunPartial, true
}
}
var summaryOrder = []struct{ status, label string }{
{models.PatchSrvSucceeded, "succeeded"},
{models.PatchSrvFailed, "failed"},
{models.PatchSrvUnsupported, "unsupported"},
{models.PatchSrvAgentTooOld, "need an agent update"},
{models.PatchSrvMissedOffline, "missed offline"},
{models.PatchSrvWindowClosed, "window closed"},
{models.PatchSrvCancelled, "cancelled"},
}
// Summary is the one-line count used in alerts, e.g. "38 succeeded, 2 failed".
func Summary(run models.PatchRun) string {
counts := map[string]int{}
for _, s := range run.Servers {
counts[s.Status]++
}
var parts []string
for _, o := range summaryOrder {
if n := counts[o.status]; n > 0 {
parts = append(parts, fmt.Sprintf("%d %s", n, o.label))
}
}
return strings.Join(parts, ", ")
}
+246
View File
@@ -0,0 +1,246 @@
package patchrun
import (
"testing"
"time"
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
)
var t0 = time.Date(2026, 9, 20, 2, 0, 0, 0, time.UTC)
func tp(t time.Time) *time.Time { return &t }
func srv(id, status string) models.PatchServerRun {
return models.PatchServerRun{ServerID: id, Hostname: id, Status: status}
}
func windowRun(max int, servers ...models.PatchServerRun) models.PatchRun {
return models.PatchRun{Status: models.PatchRunRunning, StartedAt: t0, WindowEnd: tp(t0.Add(2 * time.Hour)), MaxConcurrent: max, Servers: servers}
}
func find(ts []Transition, id string) *Transition {
for i := range ts {
if ts[i].ServerID == id {
return &ts[i]
}
}
return nil
}
func TestAgentSupportsPatchResults(t *testing.T) {
cases := map[string]bool{
"1.4.0": true, "v1.4.0": true, "1.4.1": true, "1.10.0": true, "2.0.0": true,
"1.3.5": false, "1.4.0-rc1": false, "1.5.0-rc1": true,
"": false, "dev": false, "1.4": false, "x.y.z": false,
}
for v, want := range cases {
if got := AgentSupportsPatchResults(v); got != want {
t.Errorf("%q: got %v, want %v", v, got, want)
}
}
}
func TestAdvanceRespectsConcurrencyCountingRebooting(t *testing.T) {
run := windowRun(2,
srv("a", models.PatchSrvRebooting),
srv("b", models.PatchSrvQueued),
srv("c", models.PatchSrvQueued),
)
run.Servers[0].RebootedAt = tp(t0)
ts := Advance(run, t0.Add(time.Minute), map[string]bool{"b": true, "c": true})
if tr := find(ts, "b"); tr == nil || !tr.Dispatch || tr.To != models.PatchSrvPatching {
t.Fatalf("b should dispatch, got %+v", tr)
}
if tr := find(ts, "c"); tr != nil {
t.Fatalf("c must wait for a slot, got %+v", tr)
}
}
func TestAdvanceUnlimitedWhenZero(t *testing.T) {
run := windowRun(0, srv("a", models.PatchSrvQueued), srv("b", models.PatchSrvQueued))
ts := Advance(run, t0, map[string]bool{"a": true, "b": true})
if len(ts) != 2 || !ts[0].Dispatch || !ts[1].Dispatch {
t.Fatalf("both should dispatch: %+v", ts)
}
}
func TestAdvanceOfflineWaitsThenDispatches(t *testing.T) {
run := windowRun(0, srv("a", models.PatchSrvQueued))
ts := Advance(run, t0, map[string]bool{})
if len(ts) != 1 || ts[0].To != models.PatchSrvWaitingOffline || ts[0].Dispatch {
t.Fatalf("offline should wait: %+v", ts)
}
run.Servers[0].Status = models.PatchSrvWaitingOffline
if ts := Advance(run, t0.Add(time.Minute), map[string]bool{}); len(ts) != 0 {
t.Fatalf("still offline must be a no-op, got %+v", ts)
}
ts = Advance(run, t0.Add(2*time.Minute), map[string]bool{"a": true})
if len(ts) != 1 || !ts[0].Dispatch {
t.Fatalf("back online should dispatch: %+v", ts)
}
}
func TestAdvanceWindowCloses(t *testing.T) {
run := windowRun(1, srv("q", models.PatchSrvQueued), srv("w", models.PatchSrvWaitingOffline))
ts := Advance(run, t0.Add(2*time.Hour), map[string]bool{"q": true, "w": true})
if tr := find(ts, "q"); tr == nil || tr.To != models.PatchSrvWindowClosed || tr.Dispatch {
t.Fatalf("queued at window end: %+v", tr)
}
if tr := find(ts, "w"); tr == nil || tr.To != models.PatchSrvMissedOffline {
t.Fatalf("waiting at window end: %+v", tr)
}
}
func TestAdvanceNoResultTimeout(t *testing.T) {
run := windowRun(0, srv("a", models.PatchSrvPatching))
if ts := Advance(run, t0.Add(2*time.Hour+9*time.Minute), nil); len(ts) != 0 {
t.Fatalf("inside grace: %+v", ts)
}
ts := Advance(run, t0.Add(2*time.Hour+11*time.Minute), nil)
if len(ts) != 1 || ts[0].To != models.PatchSrvFailed || ts[0].Error == "" {
t.Fatalf("past grace: %+v", ts)
}
}
func TestAdvanceManualRunTimeout(t *testing.T) {
run := models.PatchRun{Status: models.PatchRunRunning, StartedAt: t0, Servers: []models.PatchServerRun{srv("a", models.PatchSrvPatching)}}
run.Servers[0].StartedAt = tp(t0)
if ts := Advance(run, t0.Add(2*time.Hour+9*time.Minute), nil); len(ts) != 0 {
t.Fatalf("manual inside timeout: %+v", ts)
}
if ts := Advance(run, t0.Add(2*time.Hour+11*time.Minute), nil); len(ts) != 1 || ts[0].To != models.PatchSrvFailed {
t.Fatalf("manual past timeout: %+v", ts)
}
}
func TestAdvanceRebootTimeout(t *testing.T) {
run := windowRun(0, srv("a", models.PatchSrvRebooting))
run.Servers[0].RebootedAt = tp(t0)
if ts := Advance(run, t0.Add(19*time.Minute), nil); len(ts) != 0 {
t.Fatalf("inside reboot timeout: %+v", ts)
}
ts := Advance(run, t0.Add(21*time.Minute), nil)
if len(ts) != 1 || ts[0].To != models.PatchSrvFailed {
t.Fatalf("past reboot timeout: %+v", ts)
}
}
func TestAdvanceCancelledDispatchesNothing(t *testing.T) {
run := windowRun(0, srv("a", models.PatchSrvQueued), srv("b", models.PatchSrvWaitingOffline), srv("c", models.PatchSrvPatching))
run.CancelledAt = tp(t0)
ts := Advance(run, t0.Add(time.Minute), map[string]bool{"a": true, "b": true})
for _, id := range []string{"a", "b"} {
if tr := find(ts, id); tr == nil || tr.To != models.PatchSrvCancelled || tr.Dispatch {
t.Errorf("%s: %+v", id, tr)
}
}
if find(ts, "c") != nil {
t.Error("an in-flight server must be left to finish")
}
}
func TestAdvanceIgnoresFinishedRun(t *testing.T) {
run := windowRun(0, srv("a", models.PatchSrvQueued))
run.Status = models.PatchRunSucceeded
if ts := Advance(run, t0, map[string]bool{"a": true}); ts != nil {
t.Fatalf("got %+v", ts)
}
}
func TestApplyResult(t *testing.T) {
now := t0.Add(10 * time.Minute)
cases := []struct {
r pb.PatchResult
want string
}{
{pb.PatchResult{Status: pb.PatchStatusOK, PendingAfter: 0}, models.PatchSrvSucceeded},
{pb.PatchResult{Status: pb.PatchStatusOK, Rebooting: true}, models.PatchSrvRebooting},
{pb.PatchResult{Status: pb.PatchStatusFailed, Message: "apt broke"}, models.PatchSrvFailed},
{pb.PatchResult{Status: pb.PatchStatusBusy, Message: "busy"}, models.PatchSrvFailed},
{pb.PatchResult{Status: pb.PatchStatusUnsupported, Message: "no metadata"}, models.PatchSrvUnsupported},
}
for _, c := range cases {
got, ok := ApplyResult(srv("a", models.PatchSrvPatching), &c.r, now)
if !ok || got.Status != c.want {
t.Errorf("%s: got %s ok=%v, want %s", c.r.Status, got.Status, ok, c.want)
}
if c.want == models.PatchSrvRebooting && (got.RebootedAt == nil || got.FinishedAt != nil) {
t.Errorf("rebooting must set RebootedAt and leave FinishedAt nil: %+v", got)
}
}
if _, ok := ApplyResult(srv("a", models.PatchSrvSucceeded), &pb.PatchResult{Status: pb.PatchStatusOK}, now); ok {
t.Error("a result for a server not patching must be ignored")
}
got, _ := ApplyResult(srv("a", models.PatchSrvPatching), &pb.PatchResult{Status: pb.PatchStatusOK, PendingAfter: -1}, now)
if got.PendingAfter != nil {
t.Error("PendingAfter -1 means unknown and must stay nil")
}
}
func TestVerifyReboot(t *testing.T) {
s := srv("a", models.PatchSrvRebooting)
s.RebootedAt = tp(t0)
now := t0.Add(5 * time.Minute)
if _, ok := VerifyReboot(s, t0.Add(-time.Hour), false, now); ok {
t.Error("boot before the reboot command is not proof")
}
if _, ok := VerifyReboot(s, t0, false, now); ok {
t.Error("boot equal to the reboot command is not proof")
}
got, ok := VerifyReboot(s, t0.Add(2*time.Minute), false, now)
if !ok || got.Status != models.PatchSrvSucceeded || got.VerifiedAt == nil {
t.Errorf("clean reboot: %+v", got)
}
got, ok = VerifyReboot(s, t0.Add(2*time.Minute), true, now)
if !ok || got.Status != models.PatchSrvFailed || got.Error == "" {
t.Errorf("still owed: %+v", got)
}
if _, ok := VerifyReboot(srv("a", models.PatchSrvPatching), t0.Add(time.Hour), false, now); ok {
t.Error("only rebooting servers verify")
}
}
func TestFinalize(t *testing.T) {
mk := func(statuses ...string) models.PatchRun {
r := windowRun(0)
for i, s := range statuses {
r.Servers = append(r.Servers, srv(string(rune('a'+i)), s))
}
return r
}
cases := []struct {
run models.PatchRun
want string
done bool
}{
{mk(models.PatchSrvSucceeded, models.PatchSrvSucceeded), models.PatchRunSucceeded, true},
{mk(models.PatchSrvSucceeded, models.PatchSrvFailed), models.PatchRunPartial, true},
{mk(models.PatchSrvSucceeded, models.PatchSrvUnsupported), models.PatchRunPartial, true},
{mk(models.PatchSrvSucceeded, models.PatchSrvAgentTooOld), models.PatchRunPartial, true},
{mk(models.PatchSrvMissedOffline, models.PatchSrvWindowClosed), models.PatchRunFailed, true},
{mk(models.PatchSrvSucceeded, models.PatchSrvPatching), "", false},
{mk(), models.PatchRunFailed, true},
}
for i, c := range cases {
got, done := Finalize(c.run)
if got != c.want || done != c.done {
t.Errorf("case %d: got %q/%v, want %q/%v", i, got, done, c.want, c.done)
}
}
cancelled := mk(models.PatchSrvSucceeded, models.PatchSrvCancelled)
cancelled.CancelledAt = tp(t0)
if got, done := Finalize(cancelled); got != models.PatchRunCancelled || !done {
t.Errorf("cancelled: %q/%v", got, done)
}
}
func TestSummary(t *testing.T) {
r := windowRun(0,
srv("a", models.PatchSrvSucceeded), srv("b", models.PatchSrvSucceeded),
srv("c", models.PatchSrvFailed), srv("d", models.PatchSrvMissedOffline))
if got := Summary(r); got != "2 succeeded, 1 failed, 1 missed offline" {
t.Fatalf("got %q", got)
}
}