fix(admin): reconcile once at boot, not only on the ticker

StartReconciler only fired on its 15-minute ticker, so nothing reconciled
until a full interval had passed and restarting admin repaired nothing.

Injection failures are most likely around a deploy or a crash, which is
exactly when the backstop was asleep -- a paying customer could sit
read-only for 15 minutes with the repair already computable. A restart is
now also a supported way to force reconciliation.

Found by the plan's own Step 8, which assumed this behaviour: verified by
tampering with a control-plane blob, confirming the instance went invalid,
and watching the boot pass restore it (checked 1, repaired 1).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
mrhid6
2026-07-25 19:21:16 +01:00
co-authored by Claude Opus 5
parent 7e6d7074d6
commit b7221d8111
+24 -11
View File
@@ -124,9 +124,17 @@ func Reconcile(ctx context.Context) (checked, repaired int, err error) {
return checked, repaired, nil
}
// StartReconciler runs Reconcile on a ticker until ctx is cancelled.
// StartReconciler reconciles once at boot, then on a ticker until ctx is
// cancelled.
//
// The pass at boot matters: injection failures are most likely around a deploy
// or a crash, and waiting a full interval to notice would leave a paying
// customer read-only for that long. It also means a restart is a supported way
// to force reconciliation.
func StartReconciler(ctx context.Context) {
go func() {
runOnce(ctx)
t := time.NewTicker(ReconcileInterval)
defer t.Stop()
for {
@@ -134,17 +142,22 @@ func StartReconciler(ctx context.Context) {
case <-ctx.Done():
return
case <-t.C:
runCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
checked, repaired, err := Reconcile(runCtx)
cancel()
if err != nil {
log.Printf("reconcile: %v", err)
continue
}
if repaired > 0 {
log.Printf("reconcile: checked %d, repaired %d", checked, repaired)
}
runOnce(ctx)
}
}
}()
}
func runOnce(ctx context.Context) {
runCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
checked, repaired, err := Reconcile(runCtx)
if err != nil {
log.Printf("reconcile: %v", err)
return
}
if repaired > 0 {
log.Printf("reconcile: checked %d, repaired %d", checked, repaired)
}
}