feat: purge instances whose HQ dispute failed; Free reaper skips locked instances

2026-09-10 13:25:33 +00:00
parent 913d3da386
commit ec22696402
2 changed files with 113 additions and 24 deletions
@@ -58,12 +58,35 @@ func reapAfter() time.Duration {
 	return d
 }

+// freeReapFilter selects Free instances past the reap window. A locked instance
+// is excluded: it is under an open dispute in Vantage HQ, and a restore must
+// find it intact. Its fate is the dispute's to decide.
+func freeReapFilter(cutoff time.Time) bson.M {
+	return bson.M{
+		"license_tier":   license.TierFree,
+		"license_expiry": bson.M{"$ne": nil, "$lt": cutoff},
+		"locked_at":      bson.M{"$exists": false},
+	}
+}
+
+// terminatedPurgeFilter selects instances whose dispute failed in Vantage HQ.
+// Both fields are positive assertions and only HQ writes them: locked_at alone
+// is an open dispute, which must never be purged, and a self-hosted control
+// plane never has either.
+func terminatedPurgeFilter(now time.Time) bson.M {
+	return bson.M{
+		"locked_at":   bson.M{"$exists": true},
+		"purge_after": bson.M{"$exists": true, "$lt": now},
+	}
+}
+
 // purgeInstance deletes an instance and every document scoped to it.
 //
 // Unexported and unguarded: it trusts its caller completely and performs an
 // irreversible delete on whatever instance ID it is handed. The tier and expiry
 // gate - Free tier, an expiry that exists, an expiry past the window - lives in
-// ReapFreeInstances, which is the only caller. Do not export this.
+// ReapFreeInstances and ReapTerminatedInstances, which are the only callers.
+// Do not export this.
 //
 // Idempotent: re-running over a half-deleted instance completes it. The instance
 // document goes last, so an interrupted purge is retried on the next sweep
@@ -107,10 +130,7 @@ func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) {
 	}
 	cutoff := time.Now().UTC().Add(-window)

-	cur, err := db.Col("instances").Find(ctx, bson.M{
-		"license_tier":   license.TierFree,
-		"license_expiry": bson.M{"$ne": nil, "$lt": cutoff},
-	})
+	cur, err := db.Col("instances").Find(ctx, freeReapFilter(cutoff))
 	if err != nil {
 		return 0, 0, err
 	}
@@ -152,20 +172,54 @@ func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) {
 	return checked, purged, nil
 }

-// StartReaper sweeps once at boot, then on a ticker until ctx is cancelled, and
-// logs loudly which mode it is in.
-//
-// The pass at boot follows inject.StartReconciler's precedent and earns its keep
-// the same way: it makes a restart a supported way to force a sweep, which is
-// the only way this code can be exercised on demand - the ticker is hourly and
-// deletion is measured in days.
-func StartReaper(ctx context.Context) {
-	window := reapAfter()
-	if window == 0 {
-		log.Printf("reaper: DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)")
-		return
+// ReapTerminatedInstances deletes instances Vantage HQ has marked for purge
+// after a failed dispute. It does not depend on FREE_INSTANCE_REAP_AFTER:
+// this is an explicit staff decision, not a policy default.
+func ReapTerminatedInstances(ctx context.Context) (checked, purged int, err error) {
+	cur, err := db.Col("instances").Find(ctx, terminatedPurgeFilter(time.Now().UTC()))
+	if err != nil {
+		return 0, 0, err
 	}
-	log.Printf("reaper: ENABLED - Free instances are deleted %s after their licence expires", window)
+	var doomed []struct {
+		InstanceID string    `bson:"instance_id"`
+		Name       string    `bson:"name"`
+		Slug       string    `bson:"slug"`
+		PurgeAfter time.Time `bson:"purge_after"`
+	}
+	if err := cur.All(ctx, &doomed); err != nil {
+		return 0, 0, err
+	}
+
+	for _, d := range doomed {
+		checked++
+		// Logged before the delete, for the same reason as the Free reaper:
+		// afterwards there is nothing left to describe.
+		log.Printf("REAPING terminated instance %s (%s, slug=%s) - dispute failed in Vantage HQ, purge authorised from %s",
+			d.InstanceID, d.Name, d.Slug, d.PurgeAfter.Format(time.RFC3339))
+		LogEvent(d.InstanceID, "instance.purged_terminated", "system", "", "",
+			fmt.Sprintf("dispute failed in Vantage HQ, purge authorised from %s", d.PurgeAfter.Format(time.RFC3339)))
+
+		counts, err := purgeInstance(ctx, d.InstanceID)
+		if err != nil {
+			log.Printf("reaper: purge of terminated %s failed after %v: %v", d.InstanceID, counts, err)
+			continue
+		}
+		purged++
+		log.Printf("reaped terminated instance %s: %v", d.InstanceID, counts)
+	}
+	return checked, purged, nil
+}
+
+// StartReaper sweeps once at boot, then hourly until ctx is cancelled. The
+// terminated sweep always runs; the Free sweep keeps its own off switch, and
+// the boot log names both modes.
+func StartReaper(ctx context.Context) {
+	if window := reapAfter(); window == 0 {
+		log.Printf("reaper: Free reaping DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)")
+	} else {
+		log.Printf("reaper: Free reaping ENABLED - Free instances are deleted %s after their licence expires", window)
+	}
+	log.Printf("reaper: terminated purge ENABLED - instances Vantage HQ marks for purge are deleted once purge_after passes")

 	go func() {
 		reapOnce(ctx)
@@ -187,12 +241,14 @@ func reapOnce(ctx context.Context) {
 	runCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer cancel()

-	checked, purged, err := ReapFreeInstances(runCtx)
-	if err != nil {
-		log.Printf("reaper: %v", err)
-		return
+	if checked, purged, err := ReapFreeInstances(runCtx); err != nil {
+		log.Printf("reaper: free: %v", err)
+	} else if purged > 0 {
+		log.Printf("reaper: free: checked %d, purged %d", checked, purged)
 	}
-	if purged > 0 {
-		log.Printf("reaper: checked %d, purged %d", checked, purged)
+	if checked, purged, err := ReapTerminatedInstances(runCtx); err != nil {
+		log.Printf("reaper: terminated: %v", err)
+	} else if purged > 0 {
+		log.Printf("reaper: terminated: checked %d, purged %d", checked, purged)
 	}
 }
@@ -0,0 +1,33 @@
+package services
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"gitea.hostxtra.co.uk/vantage/vantage-shared/license"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+func TestFreeReapFilterSkipsLockedInstances(t *testing.T) {
+	cutoff := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+	want := bson.M{
+		"license_tier":   license.TierFree,
+		"license_expiry": bson.M{"$ne": nil, "$lt": cutoff},
+		"locked_at":      bson.M{"$exists": false},
+	}
+	if got := freeReapFilter(cutoff); !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestTerminatedPurgeFilterNeedsBothFields(t *testing.T) {
+	now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)
+	want := bson.M{
+		"locked_at":   bson.M{"$exists": true},
+		"purge_after": bson.M{"$exists": true, "$lt": now},
+	}
+	if got := terminatedPurgeFilter(now); !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}