fix(admin): reconcile once at boot, not only on the ticker
StartReconciler only fired on its 15-minute ticker, so nothing reconciled until a full interval had passed and restarting admin repaired nothing. Injection failures are most likely around a deploy or a crash, which is exactly when the backstop was asleep -- a paying customer could sit read-only for 15 minutes with the repair already computable. A restart is now also a supported way to force reconciliation. Found by the plan's own Step 8, which assumed this behaviour: verified by tampering with a control-plane blob, confirming the instance went invalid, and watching the boot pass restore it (checked 1, repaired 1). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -124,9 +124,17 @@ func Reconcile(ctx context.Context) (checked, repaired int, err error) {
|
||||
return checked, repaired, nil
|
||||
}
|
||||
|
||||
// StartReconciler runs Reconcile on a ticker until ctx is cancelled.
|
||||
// StartReconciler reconciles once at boot, then on a ticker until ctx is
|
||||
// cancelled.
|
||||
//
|
||||
// The pass at boot matters: injection failures are most likely around a deploy
|
||||
// or a crash, and waiting a full interval to notice would leave a paying
|
||||
// customer read-only for that long. It also means a restart is a supported way
|
||||
// to force reconciliation.
|
||||
func StartReconciler(ctx context.Context) {
|
||||
go func() {
|
||||
runOnce(ctx)
|
||||
|
||||
t := time.NewTicker(ReconcileInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
@@ -134,17 +142,22 @@ func StartReconciler(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
runCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
checked, repaired, err := Reconcile(runCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("reconcile: %v", err)
|
||||
continue
|
||||
}
|
||||
if repaired > 0 {
|
||||
log.Printf("reconcile: checked %d, repaired %d", checked, repaired)
|
||||
}
|
||||
runOnce(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func runOnce(ctx context.Context) {
|
||||
runCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
checked, repaired, err := Reconcile(runCtx)
|
||||
if err != nil {
|
||||
log.Printf("reconcile: %v", err)
|
||||
return
|
||||
}
|
||||
if repaired > 0 {
|
||||
log.Printf("reconcile: checked %d, repaired %d", checked, repaired)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user