fix(patch): gate phases on the window deadline, queue undelivered results, retry startup inventory

The window deadline no longer kills a running package manager: it only gates
the start of each phase, and a started upgrade runs under a 2 hour backstop
that sends SIGTERM on Linux. A PatchResult whose send fails is queued and
flushed on the next command stream, retaking the reboot decision. deb822
folded Suites continuation lines are filtered with the field. The startup
static inventory report is retried until it succeeds.

internal/sync/patch.go

@@ -57,10 +57,15 @@ func handleApplyUpdates(send func(*pb.AgentMessage) error, cfg *config.Config, c
 	pr.RebootRequired = updates.RebootRequired()
 	pr.Rebooting = pr.Status == pb.PatchStatusOK && shouldReboot(c.RebootIfRequired, pr.RebootRequired, time.Now(), deadline)
 
-	if err := send(&pb.AgentMessage{ServerId: cfg.ServerID, AgentToken: cfg.AgentToken, PatchResult: pr}); err != nil {
-		log.Printf("send patch result (cmd=%s): %v", cmd.CommandId, err)
+	msg := &pb.AgentMessage{ServerId: cfg.ServerID, AgentToken: cfg.AgentToken, PatchResult: pr}
+	if err := send(msg); err != nil {
+		log.Printf("send patch result (cmd=%s): %v; keeping it for the next command stream", cmd.CommandId, err)
 		// A reboot nobody was told about looks like a crash. Without a
-		// delivered result, do not reboot.
+		// delivered result, do not reboot: the queued result retakes the
+		// decision when it is finally sent.
+		if pendingResults.push(pendingResult{msg: msg, requested: c.RebootIfRequired, deadline: deadline}) {
+			log.Printf("patch result queue full: dropped the oldest undelivered result")
+		}
 		return
 	}
 	log.Printf("patch result sent (cmd=%s status=%s pending_after=%d rebooting=%v)", cmd.CommandId, pr.Status, pr.PendingAfter, pr.Rebooting)
@@ -71,6 +76,40 @@ func handleApplyUpdates(send func(*pb.AgentMessage) error, cfg *config.Config, c
 	}
 }
 
+// pendingResults holds PatchResults whose send failed, typically because the
+// command stream reconnected while a patch ran. The server guards result
+// writes on status and command id, so a late result is safe to deliver.
+var pendingResults = &resultQueue{max: maxPendingResults}
+
+// flushPendingResults delivers queued results on a newly established stream.
+// A queued result that announced a reboot has that decision taken again now:
+// time has passed, so the window may be too close to its end, or the reboot
+// may no longer be owed. Rebooting is cleared before sending when it no
+// longer holds, and the host reboots only once the result is delivered.
+func flushPendingResults(send func(*pb.AgentMessage) error) {
+	pendingResults.flush(func(p pendingResult) error {
+		pr := p.msg.PatchResult
+		if pr.Rebooting {
+			owed := updates.RebootRequired()
+			pr.RebootRequired = owed
+			if !shouldReboot(p.requested, owed, time.Now(), p.deadline) {
+				pr.Rebooting = false
+			}
+		}
+		if err := send(p.msg); err != nil {
+			log.Printf("send queued patch result (cmd=%s): %v", pr.CommandId, err)
+			return err
+		}
+		log.Printf("queued patch result sent (cmd=%s status=%s rebooting=%v)", pr.CommandId, pr.Status, pr.Rebooting)
+		if pr.Rebooting {
+			if err := updates.ScheduleReboot(); err != nil {
+				log.Printf("schedule reboot (cmd=%s): %v", pr.CommandId, err)
+			}
+		}
+		return nil
+	})
+}
+
 // reportPendingUpdates re-checks pending updates and reports them, returning
 // the count, or -1 if either step failed.
 func reportPendingUpdates(cfg *config.Config) int {

internal/sync/queue.go

@@ -0,0 +1,56 @@
+package agentsync
+
+import (
+	"sync"
+	"time"
+
+	"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
+)
+
+// maxPendingResults bounds the results kept for a later stream. A host that
+// cannot reach the control plane for a very long time drops its oldest
+// results rather than growing without limit.
+const maxPendingResults = 32
+
+// pendingResult is a PatchResult that could not be sent, with what the reboot
+// decision needs to be taken again when it finally can be.
+type pendingResult struct {
+	msg       *pb.AgentMessage
+	requested bool      // the command's RebootIfRequired
+	deadline  time.Time // the command's deadline, zero for a manual run
+}
+
+// resultQueue holds undelivered PatchResults until the next command stream.
+type resultQueue struct {
+	mu    sync.Mutex
+	items []pendingResult
+	max   int
+}
+
+// push appends p, dropping the oldest entry when the queue is full. It
+// reports whether an entry was dropped.
+func (q *resultQueue) push(p pendingResult) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	q.items = append(q.items, p)
+	if len(q.items) > q.max {
+		q.items = q.items[len(q.items)-q.max:]
+		return true
+	}
+	return false
+}
+
+// flush sends the queued results oldest first, removing each once sent. It
+// stops at the first failure and keeps that entry and everything after it
+// for the next stream. The lock is held throughout so two streams never
+// deliver the same result.
+func (q *resultQueue) flush(send func(pendingResult) error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	for len(q.items) > 0 {
+		if err := send(q.items[0]); err != nil {
+			return
+		}
+		q.items = q.items[1:]
+	}
+}

internal/sync/queue_test.go

@@ -0,0 +1,88 @@
+package agentsync
+
+import (
+	"errors"
+	"testing"
+
+	"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
+)
+
+func pending(id string) pendingResult {
+	return pendingResult{msg: &pb.AgentMessage{PatchResult: &pb.PatchResult{CommandId: id}}}
+}
+
+func ids(q *resultQueue) []string {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	var out []string
+	for _, p := range q.items {
+		out = append(out, p.msg.PatchResult.CommandId)
+	}
+	return out
+}
+
+func TestResultQueueEnqueue(t *testing.T) {
+	q := &resultQueue{max: 3}
+	q.push(pending("a"))
+	q.push(pending("b"))
+	if got := ids(q); len(got) != 2 || got[0] != "a" || got[1] != "b" {
+		t.Fatalf("got %v", got)
+	}
+}
+
+func TestResultQueueBoundDropsOldest(t *testing.T) {
+	q := &resultQueue{max: 2}
+	q.push(pending("a"))
+	q.push(pending("b"))
+	if dropped := q.push(pending("c")); !dropped {
+		t.Fatal("push over the bound must report a drop")
+	}
+	if got := ids(q); len(got) != 2 || got[0] != "b" || got[1] != "c" {
+		t.Fatalf("got %v, want [b c]", got)
+	}
+}
+
+func TestResultQueueDefaultBound(t *testing.T) {
+	if maxPendingResults != 32 {
+		t.Fatalf("maxPendingResults = %d, want 32", maxPendingResults)
+	}
+}
+
+func TestResultQueueFlushOrderAndRemoval(t *testing.T) {
+	q := &resultQueue{max: 8}
+	for _, id := range []string{"a", "b", "c"} {
+		q.push(pending(id))
+	}
+	var sent []string
+	q.flush(func(p pendingResult) error {
+		sent = append(sent, p.msg.PatchResult.CommandId)
+		return nil
+	})
+	if len(sent) != 3 || sent[0] != "a" || sent[1] != "b" || sent[2] != "c" {
+		t.Fatalf("flush order %v, want [a b c]", sent)
+	}
+	if got := ids(q); len(got) != 0 {
+		t.Fatalf("sent entries must be removed, left %v", got)
+	}
+}
+
+func TestResultQueueFlushRetainsOnFailure(t *testing.T) {
+	q := &resultQueue{max: 8}
+	for _, id := range []string{"a", "b", "c"} {
+		q.push(pending(id))
+	}
+	var sent []string
+	q.flush(func(p pendingResult) error {
+		if p.msg.PatchResult.CommandId == "b" {
+			return errors.New("stream gone")
+		}
+		sent = append(sent, p.msg.PatchResult.CommandId)
+		return nil
+	})
+	if len(sent) != 1 || sent[0] != "a" {
+		t.Fatalf("sent %v, want [a]", sent)
+	}
+	if got := ids(q); len(got) != 2 || got[0] != "b" || got[1] != "c" {
+		t.Fatalf("retained %v, want [b c]", got)
+	}
+}

internal/sync/sync.go

@@ -243,6 +243,11 @@ func connectAndHandleStream(ctx context.Context, cfg *config.Config) error {
 		return stream.Send(msg)
 	}
 
+	// Results that could not be sent on an earlier stream go out first. In
+	// its own goroutine: a queued reboot re-check can take a while on
+	// Windows, and the receive loop below must start promptly.
+	go flushPendingResults(send)
+
 	// Stream liveness, tracked here rather than left to gRPC keepalive.
 	//
 	// Keepalive operates on the transport, and behind an L7 proxy the transport
@@ -446,7 +451,7 @@ func runInventory(ctx context.Context, cfg *config.Config) {
 	}
 	defer client.Close()
 
-	report := func(static bool) {
+	report := func(static bool) error {
 		r := inventory.Collect(static)
 		r.ServerId = cfg.ServerID
 		r.AgentToken = cfg.AgentToken
@@ -462,10 +467,18 @@ func runInventory(ctx context.Context, cfg *config.Config) {
 		}
 		if err := client.ReportInventory(r); err != nil {
 			log.Printf("report inventory: %v", err)
+			return err
 		}
+		return nil
 	}
 
-	report(true)
+	// The startup static report carries the boot time the server uses to
+	// prove a patch reboot happened, so a failure is retried on the next
+	// ticks (every 30 seconds, up to startupStaticAttempts in total) instead
+	// of waiting a quarter of an hour for the next static snapshot.
+	const startupStaticAttempts = 10
+	attempts := 1
+	startupPending := report(true) != nil
 	ticker := time.NewTicker(30 * time.Second)
 	defer ticker.Stop()
 	tick := 0
@@ -475,6 +488,12 @@ func runInventory(ctx context.Context, cfg *config.Config) {
 			return
 		case <-ticker.C:
 			tick++
+			if startupPending && attempts < startupStaticAttempts {
+				attempts++
+				startupPending = report(true) != nil
+				continue
+			}
+			startupPending = false
 			report(tick%30 == 0)
 		}
 	}