fix(patch): gate phases on the window deadline, queue undelivered results, retry startup inventory
The window deadline no longer kills a running package manager: it only gates the start of each phase, and a started upgrade runs under a 2 hour backstop that sends SIGTERM on Linux. A PatchResult whose send fails is queued and flushed on the next command stream, retaking the reboot decision. deb822 folded Suites continuation lines are filtered with the field. The startup static inventory report is retried until it succeeds.
This commit is contained in:
+42
-3
@@ -57,10 +57,15 @@ func handleApplyUpdates(send func(*pb.AgentMessage) error, cfg *config.Config, c
|
||||
pr.RebootRequired = updates.RebootRequired()
|
||||
pr.Rebooting = pr.Status == pb.PatchStatusOK && shouldReboot(c.RebootIfRequired, pr.RebootRequired, time.Now(), deadline)
|
||||
|
||||
if err := send(&pb.AgentMessage{ServerId: cfg.ServerID, AgentToken: cfg.AgentToken, PatchResult: pr}); err != nil {
|
||||
log.Printf("send patch result (cmd=%s): %v", cmd.CommandId, err)
|
||||
msg := &pb.AgentMessage{ServerId: cfg.ServerID, AgentToken: cfg.AgentToken, PatchResult: pr}
|
||||
if err := send(msg); err != nil {
|
||||
log.Printf("send patch result (cmd=%s): %v; keeping it for the next command stream", cmd.CommandId, err)
|
||||
// A reboot nobody was told about looks like a crash. Without a
|
||||
// delivered result, do not reboot.
|
||||
// delivered result, do not reboot: the queued result retakes the
|
||||
// decision when it is finally sent.
|
||||
if pendingResults.push(pendingResult{msg: msg, requested: c.RebootIfRequired, deadline: deadline}) {
|
||||
log.Printf("patch result queue full: dropped the oldest undelivered result")
|
||||
}
|
||||
return
|
||||
}
|
||||
log.Printf("patch result sent (cmd=%s status=%s pending_after=%d rebooting=%v)", cmd.CommandId, pr.Status, pr.PendingAfter, pr.Rebooting)
|
||||
@@ -71,6 +76,40 @@ func handleApplyUpdates(send func(*pb.AgentMessage) error, cfg *config.Config, c
|
||||
}
|
||||
}
|
||||
|
||||
// pendingResults holds PatchResults whose send failed, typically because the
|
||||
// command stream reconnected while a patch ran. The server guards result
|
||||
// writes on status and command id, so a late result is safe to deliver.
|
||||
var pendingResults = &resultQueue{max: maxPendingResults}
|
||||
|
||||
// flushPendingResults delivers queued results on a newly established stream.
|
||||
// A queued result that announced a reboot has that decision taken again now:
|
||||
// time has passed, so the window may be too close to its end, or the reboot
|
||||
// may no longer be owed. Rebooting is cleared before sending when it no
|
||||
// longer holds, and the host reboots only once the result is delivered.
|
||||
func flushPendingResults(send func(*pb.AgentMessage) error) {
|
||||
pendingResults.flush(func(p pendingResult) error {
|
||||
pr := p.msg.PatchResult
|
||||
if pr.Rebooting {
|
||||
owed := updates.RebootRequired()
|
||||
pr.RebootRequired = owed
|
||||
if !shouldReboot(p.requested, owed, time.Now(), p.deadline) {
|
||||
pr.Rebooting = false
|
||||
}
|
||||
}
|
||||
if err := send(p.msg); err != nil {
|
||||
log.Printf("send queued patch result (cmd=%s): %v", pr.CommandId, err)
|
||||
return err
|
||||
}
|
||||
log.Printf("queued patch result sent (cmd=%s status=%s rebooting=%v)", pr.CommandId, pr.Status, pr.Rebooting)
|
||||
if pr.Rebooting {
|
||||
if err := updates.ScheduleReboot(); err != nil {
|
||||
log.Printf("schedule reboot (cmd=%s): %v", pr.CommandId, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// reportPendingUpdates re-checks pending updates and reports them, returning
|
||||
// the count, or -1 if either step failed.
|
||||
func reportPendingUpdates(cfg *config.Config) int {
|
||||
|
||||
Reference in New Issue
Block a user