diff --git a/internal/sync/sync.go b/internal/sync/sync.go index 1db2fdd..de06199 100644 --- a/internal/sync/sync.go +++ b/internal/sync/sync.go @@ -118,9 +118,21 @@ func poll(client *grpcclient.Client, cfg *config.Config, version string) error { return nil } +// How long a command stream must survive before it counts as having worked. +// Past this, the next drop is treated as a fresh incident rather than as the +// continuation of a run of failures. +const streamHealthyAfter = time.Minute + func runCommandStream(ctx context.Context, cfg *config.Config) { backoff := time.Second - const maxBackoff = 2 * time.Minute + + // Two minutes was the old ceiling, and it was reached far too easily. The + // command stream is what makes this agent controllable at all: while it is + // down, workflows and console sessions fail as "agent offline" even though + // SyncKeys keeps polling happily and the fleet list still shows the server + // active. A shorter ceiling costs a few reconnect attempts; the old one cost + // two minutes of an agent that looks fine and answers nothing. + const maxBackoff = 30 * time.Second for { select { @@ -129,22 +141,40 @@ func runCommandStream(ctx context.Context, cfg *config.Config) { default: } - if err := connectAndHandleStream(ctx, cfg); err != nil { - if ctx.Err() != nil { - return - } - log.Printf("command stream error: %v, reconnecting in %s", err, backoff) - select { - case <-ctx.Done(): - return - case <-time.After(backoff): - } - if backoff < maxBackoff { - backoff *= 2 - } - } else { + started := time.Now() + err := connectAndHandleStream(ctx, cfg) + if ctx.Err() != nil { + return + } + + // A stream that stayed up is evidence the control plane is reachable, + // whatever ended it. Without this the backoff only ever climbed: + // connectAndHandleStream returns an error on *every* stream end, + // including a healthy one dropped by a routine deploy, so an agent + // pinned itself at the ceiling after a handful of ordinary restarts and + // stayed there for the rest of its life. + if time.Since(started) >= streamHealthyAfter { backoff = time.Second } + + if err != nil { + log.Printf("command stream error: %v, reconnecting in %s", err, backoff) + } else { + log.Printf("command stream closed, reconnecting in %s", backoff) + } + + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + + if backoff < maxBackoff { + backoff *= 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + } } }