fix: Fixes to command stream

server/internal/services/dispatch.go

@@ -30,9 +30,12 @@ const (
 	dispatchAckTimeout = 5 * time.Second
 
 	// Presence must outlive a renew or two, or a momentarily slow pod would
-	// look offline and its agent would be declared unreachable.
+	// look offline and its agent would be declared unreachable. Three renewals
+	// inside one TTL, not two: a renewal is also how a claim is recovered after
+	// Redis loses it, and recovery time is what the operator experiences as the
+	// agent being undispatchable.
 	presenceTTL   = 30 * time.Second
-	presenceRenew = 10 * time.Second
+	presenceRenew = 8 * time.Second
 )
 
 // CommandEnvelope is what actually crosses the bus. It is the command plus the
@@ -106,8 +109,11 @@ func (d *commandDispatcher) Serve(ctx context.Context, serverID string) (<-chan
 
 	runCtx, cancel := context.WithCancel(ctx)
 
+	// A failure here is not fatal and is not special-cased: the renewal loop
+	// reclaims an unheld key, so the first tick repairs it. Anything else would
+	// need a second recovery path for a case the loop already covers.
 	if err := bus.SetPresence(runCtx, serverID, presenceTTL); err != nil {
-		log.Printf("dispatch: claim presence for %s: %v", serverID, err)
+		log.Printf("dispatch: claim presence for %s, renewal will retry: %v", serverID, err)
 	}
 	go func() {
 		t := time.NewTicker(presenceRenew)
@@ -117,12 +123,7 @@ func (d *commandDispatcher) Serve(ctx context.Context, serverID string) (<-chan
 			case <-runCtx.Done():
 				return
 			case <-t.C:
-				// Renew only while this pod still holds the claim. Losing it
-				// means a newer stream for the same agent was established
-				// elsewhere, and this one is a half-open leftover: it must stop
-				// renewing rather than overwrite the live owner every 10s.
-				if !bus.RenewPresence(runCtx, serverID, presenceTTL) {
-					log.Printf("dispatch: presence for %s is held elsewhere, stopping renewal", serverID)
+				if !renewPresence(runCtx, serverID) {
 					return
 				}
 			}
@@ -154,6 +155,46 @@ func (d *commandDispatcher) Serve(ctx context.Context, serverID string) (<-chan
 	}
 }
 
+// renewPresence extends this pod's claim on serverID and reports whether the
+// claim is still this pod's to hold. False means, and only means, that another
+// pod now owns the agent's stream.
+//
+// A Redis failure returns true. It is tempting to read an error as loss and
+// give up, but nothing is known in that moment about who holds the claim, and
+// the stream this pod is serving is demonstrably alive — the caller is either a
+// ticker on that stream or a beat that just succeeded on it. Standing down on a
+// blip is precisely how an agent ends up connected, beating, and unreachable
+// until it happens to reconnect.
+func renewPresence(ctx context.Context, serverID string) bool {
+	switch bus.RenewPresence(ctx, serverID, presenceTTL) {
+	case bus.RenewLost:
+		// A newer stream for this agent exists elsewhere and this one is a
+		// half-open leftover. Stop, rather than overwrite the live owner every
+		// renewal interval and leave the key naming whichever wrote last.
+		log.Printf("dispatch: presence for %s is held elsewhere, standing down", serverID)
+		return false
+	case bus.RenewedClaim:
+		// Nobody held the key — Redis restarted, failed over, evicted it, or was
+		// unreachable for longer than the TTL. Worth a line: it is the only
+		// evidence that presence was lost and recovered rather than never lost.
+		log.Printf("dispatch: reclaimed presence for %s", serverID)
+	case bus.RenewFailed:
+		log.Printf("dispatch: renew presence for %s failed, will retry", serverID)
+	}
+	return true
+}
+
+// TouchPresence renews serverID's claim off the back of something that proves
+// the stream is alive, and reports whether this pod should keep serving it.
+//
+// The renewal ticker proves only that a goroutine is still scheduled. A beat
+// that the agent's stream accepted proves the stream itself still works, which
+// is the thing presence is supposed to advertise. A false answer means this
+// stream has been superseded and the handler should return.
+func TouchPresence(ctx context.Context, serverID string) bool {
+	return renewPresence(ctx, serverID)
+}
+
 // handleEnvelope performs the owner-pod side of a dispatch: any local setup the
 // command needs, then queueing it for the stream, then the ack.
 func (d *commandDispatcher) handleEnvelope(ctx context.Context, raw []byte, out chan *pb.ServerCommand) {