diff --git a/server/internal/services/reap.go b/server/internal/services/reap.go index 56af864..0ce0fc4 100644 --- a/server/internal/services/reap.go +++ b/server/internal/services/reap.go @@ -58,12 +58,35 @@ func reapAfter() time.Duration { return d } +// freeReapFilter selects Free instances past the reap window. A locked instance +// is excluded: it is under an open dispute in Vantage HQ, and a restore must +// find it intact. Its fate is the dispute's to decide. +func freeReapFilter(cutoff time.Time) bson.M { + return bson.M{ + "license_tier": license.TierFree, + "license_expiry": bson.M{"$ne": nil, "$lt": cutoff}, + "locked_at": bson.M{"$exists": false}, + } +} + +// terminatedPurgeFilter selects instances whose dispute failed in Vantage HQ. +// Both fields are positive assertions and only HQ writes them: locked_at alone +// is an open dispute, which must never be purged, and a self-hosted control +// plane never has either. +func terminatedPurgeFilter(now time.Time) bson.M { + return bson.M{ + "locked_at": bson.M{"$exists": true}, + "purge_after": bson.M{"$exists": true, "$lt": now}, + } +} + // purgeInstance deletes an instance and every document scoped to it. // // Unexported and unguarded: it trusts its caller completely and performs an // irreversible delete on whatever instance ID it is handed. The tier and expiry // gate - Free tier, an expiry that exists, an expiry past the window - lives in -// ReapFreeInstances, which is the only caller. Do not export this. +// ReapFreeInstances and ReapTerminatedInstances, which are the only callers. +// Do not export this. // // Idempotent: re-running over a half-deleted instance completes it. The instance // document goes last, so an interrupted purge is retried on the next sweep @@ -107,10 +130,7 @@ func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) { } cutoff := time.Now().UTC().Add(-window) - cur, err := db.Col("instances").Find(ctx, bson.M{ - "license_tier": license.TierFree, - "license_expiry": bson.M{"$ne": nil, "$lt": cutoff}, - }) + cur, err := db.Col("instances").Find(ctx, freeReapFilter(cutoff)) if err != nil { return 0, 0, err } @@ -152,20 +172,54 @@ func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) { return checked, purged, nil } -// StartReaper sweeps once at boot, then on a ticker until ctx is cancelled, and -// logs loudly which mode it is in. -// -// The pass at boot follows inject.StartReconciler's precedent and earns its keep -// the same way: it makes a restart a supported way to force a sweep, which is -// the only way this code can be exercised on demand - the ticker is hourly and -// deletion is measured in days. -func StartReaper(ctx context.Context) { - window := reapAfter() - if window == 0 { - log.Printf("reaper: DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)") - return +// ReapTerminatedInstances deletes instances Vantage HQ has marked for purge +// after a failed dispute. It does not depend on FREE_INSTANCE_REAP_AFTER: +// this is an explicit staff decision, not a policy default. +func ReapTerminatedInstances(ctx context.Context) (checked, purged int, err error) { + cur, err := db.Col("instances").Find(ctx, terminatedPurgeFilter(time.Now().UTC())) + if err != nil { + return 0, 0, err } - log.Printf("reaper: ENABLED - Free instances are deleted %s after their licence expires", window) + var doomed []struct { + InstanceID string `bson:"instance_id"` + Name string `bson:"name"` + Slug string `bson:"slug"` + PurgeAfter time.Time `bson:"purge_after"` + } + if err := cur.All(ctx, &doomed); err != nil { + return 0, 0, err + } + + for _, d := range doomed { + checked++ + // Logged before the delete, for the same reason as the Free reaper: + // afterwards there is nothing left to describe. + log.Printf("REAPING terminated instance %s (%s, slug=%s) - dispute failed in Vantage HQ, purge authorised from %s", + d.InstanceID, d.Name, d.Slug, d.PurgeAfter.Format(time.RFC3339)) + LogEvent(d.InstanceID, "instance.purged_terminated", "system", "", "", + fmt.Sprintf("dispute failed in Vantage HQ, purge authorised from %s", d.PurgeAfter.Format(time.RFC3339))) + + counts, err := purgeInstance(ctx, d.InstanceID) + if err != nil { + log.Printf("reaper: purge of terminated %s failed after %v: %v", d.InstanceID, counts, err) + continue + } + purged++ + log.Printf("reaped terminated instance %s: %v", d.InstanceID, counts) + } + return checked, purged, nil +} + +// StartReaper sweeps once at boot, then hourly until ctx is cancelled. The +// terminated sweep always runs; the Free sweep keeps its own off switch, and +// the boot log names both modes. +func StartReaper(ctx context.Context) { + if window := reapAfter(); window == 0 { + log.Printf("reaper: Free reaping DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)") + } else { + log.Printf("reaper: Free reaping ENABLED - Free instances are deleted %s after their licence expires", window) + } + log.Printf("reaper: terminated purge ENABLED - instances Vantage HQ marks for purge are deleted once purge_after passes") go func() { reapOnce(ctx) @@ -187,12 +241,14 @@ func reapOnce(ctx context.Context) { runCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) defer cancel() - checked, purged, err := ReapFreeInstances(runCtx) - if err != nil { - log.Printf("reaper: %v", err) - return + if checked, purged, err := ReapFreeInstances(runCtx); err != nil { + log.Printf("reaper: free: %v", err) + } else if purged > 0 { + log.Printf("reaper: free: checked %d, purged %d", checked, purged) } - if purged > 0 { - log.Printf("reaper: checked %d, purged %d", checked, purged) + if checked, purged, err := ReapTerminatedInstances(runCtx); err != nil { + log.Printf("reaper: terminated: %v", err) + } else if purged > 0 { + log.Printf("reaper: terminated: checked %d, purged %d", checked, purged) } } diff --git a/server/internal/services/reap_test.go b/server/internal/services/reap_test.go new file mode 100644 index 0000000..f6dc52c --- /dev/null +++ b/server/internal/services/reap_test.go @@ -0,0 +1,33 @@ +package services + +import ( + "reflect" + "testing" + "time" + + "gitea.hostxtra.co.uk/vantage/vantage-shared/license" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func TestFreeReapFilterSkipsLockedInstances(t *testing.T) { + cutoff := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + want := bson.M{ + "license_tier": license.TierFree, + "license_expiry": bson.M{"$ne": nil, "$lt": cutoff}, + "locked_at": bson.M{"$exists": false}, + } + if got := freeReapFilter(cutoff); !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestTerminatedPurgeFilterNeedsBothFields(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + want := bson.M{ + "locked_at": bson.M{"$exists": true}, + "purge_after": bson.M{"$exists": true, "$lt": now}, + } + if got := terminatedPurgeFilter(now); !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +}