feat(server): reap Free instances whose licence lapsed

The control plane owns deletion because it is the only service that knows what an instance is made of; mirroring that collection list into admin would drift, and a drift here deletes the wrong rows. Defaults OFF. Eligibility is three positive assertions — Free tier, an expiry that exists, and an expiry past the window — so a missing or stale field is never eligible. The instance document is deleted last, making an interrupted purge retryable rather than orphaning rows. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-26 13:23:37 +01:00
parent 372a8c5ddf
commit 4e90e8619f
2 changed files with 200 additions and 0 deletions
@@ -113,6 +113,8 @@ func main() {

 	monitorsched.Start(context.Background())

+	services.StartReaper(context.Background())
+
 	r := gin.New()
 	r.Use(gin.Recovery())
 	r.Use(gin.LoggerWithConfig(gin.LoggerConfig{SkipPaths: []string{"/api/console/tunnel"}}))
@@ -0,0 +1,198 @@
+package services
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/mrhid6/vantage/server/internal/db"
+	"github.com/mrhid6/vantage/shared/license"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+// ReapInterval is how often eligibility is re-checked. Deletion is measured in
+// days, so an hour is ample and keeps the query cheap.
+const ReapInterval = time.Hour
+
+// scopedCollectionsForPurge is ScopedCollections minus `instances` itself.
+//
+// Derived rather than duplicated on purpose. ScopedCollections is the canonical
+// list and AssertNoScopedCollectionMissed fails boot when a collection outside
+// it holds a tenant key; a second hand-maintained copy here would silently miss
+// whatever that assertion catches, leaking rows that outlive their instance.
+//
+// `instances` is excluded because it is keyed by instance_id rather than scoped
+// by it, and is deleted last so an interrupted purge is retried rather than
+// orphaning rows.
+func scopedCollectionsForPurge() []string {
+	out := make([]string, 0, len(ScopedCollections))
+	for _, name := range ScopedCollections {
+		if name == "instances" {
+			continue
+		}
+		out = append(out, name)
+	}
+	return out
+}
+
+// reapAfter reads FREE_INSTANCE_REAP_AFTER.
+//
+// An empty or unparseable value returns 0, which disables the reaper. Defaulting
+// OFF is the whole safety design: a deployment that never heard of this variable
+// must never delete a customer's instance.
+func reapAfter() time.Duration {
+	v := os.Getenv("FREE_INSTANCE_REAP_AFTER")
+	if v == "" {
+		return 0
+	}
+	d, err := time.ParseDuration(v)
+	if err != nil {
+		log.Printf("reaper: FREE_INSTANCE_REAP_AFTER %q is not a duration; reaper stays OFF", v)
+		return 0
+	}
+	if d <= 0 {
+		return 0
+	}
+	return d
+}
+
+// purgeInstance deletes an instance and every document scoped to it.
+//
+// Unexported and unguarded: it trusts its caller completely and performs an
+// irreversible delete on whatever instance ID it is handed. The tier and expiry
+// gate — Free tier, an expiry that exists, an expiry past the window — lives in
+// ReapFreeInstances, which is the only caller. Do not export this.
+//
+// Idempotent: re-running over a half-deleted instance completes it. The instance
+// document goes last, so an interrupted purge is retried on the next sweep
+// instead of leaving rows behind with nothing pointing at them.
+func purgeInstance(ctx context.Context, instanceID string) (map[string]int64, error) {
+	counts := map[string]int64{}
+	for _, name := range scopedCollectionsForPurge() {
+		res, err := db.Col(name).DeleteMany(ctx, bson.M{"instance_id": instanceID})
+		if err != nil {
+			return counts, fmt.Errorf("purge %s: %w", name, err)
+		}
+		if res.DeletedCount > 0 {
+			counts[name] = res.DeletedCount
+		}
+	}
+	res, err := db.Col("instances").DeleteOne(ctx, bson.M{"instance_id": instanceID})
+	if err != nil {
+		return counts, fmt.Errorf("purge instances: %w", err)
+	}
+	if res.DeletedCount > 0 {
+		counts["instances"] = res.DeletedCount
+	}
+	return counts, nil
+}
+
+// ReapFreeInstances deletes Free cloud instances whose licence expired longer
+// ago than the configured window.
+//
+// Eligibility requires ALL of:
+//   - license_tier == "free"   — a paid instance is never eligible
+//   - license_expiry present   — an instance that was never licensed, or whose
+//     issuance failed, has no expiry and is never eligible whatever its age
+//   - license_expiry older than now minus the window
+//
+// Every one of those is a positive assertion. Nothing is eligible by default,
+// which is what makes a missing or stale field fail safe.
+func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) {
+	window := reapAfter()
+	if window == 0 {
+		return 0, 0, nil
+	}
+	cutoff := time.Now().UTC().Add(-window)
+
+	cur, err := db.Col("instances").Find(ctx, bson.M{
+		"license_tier":   license.TierFree,
+		"license_expiry": bson.M{"$ne": nil, "$lt": cutoff},
+	})
+	if err != nil {
+		return 0, 0, err
+	}
+	var doomed []struct {
+		InstanceID string    `bson:"instance_id"`
+		Name       string    `bson:"name"`
+		Slug       string    `bson:"slug"`
+		Expiry     time.Time `bson:"license_expiry"`
+	}
+	if err := cur.All(ctx, &doomed); err != nil {
+		return 0, 0, err
+	}
+
+	for _, d := range doomed {
+		checked++
+
+		// Logged BEFORE the delete. Afterwards there is nothing left to
+		// describe, and "why did this instance vanish" is the only question
+		// anyone will ever ask about this code.
+		//
+		// The process log is the durable record, not the audit row: the purge
+		// deletes this instance's audit_logs along with everything else, so an
+		// audit entry written here would delete itself moments later. It is
+		// written anyway, because an operator reading audit during the window
+		// should see it coming.
+		log.Printf("REAPING instance %s (%s, slug=%s) — Free licence expired %s, past the %s window",
+			d.InstanceID, d.Name, d.Slug, d.Expiry.Format(time.RFC3339), window)
+		LogEvent(d.InstanceID, "instance.reaped", "system", "", "",
+			fmt.Sprintf("free licence expired %s, window %s", d.Expiry.Format(time.RFC3339), window))
+
+		counts, err := purgeInstance(ctx, d.InstanceID)
+		if err != nil {
+			log.Printf("reaper: purge of %s failed after %v: %v", d.InstanceID, counts, err)
+			continue
+		}
+		purged++
+		log.Printf("reaped instance %s: %v", d.InstanceID, counts)
+	}
+	return checked, purged, nil
+}
+
+// StartReaper sweeps once at boot, then on a ticker until ctx is cancelled, and
+// logs loudly which mode it is in.
+//
+// The pass at boot follows inject.StartReconciler's precedent and earns its keep
+// the same way: it makes a restart a supported way to force a sweep, which is
+// the only way this code can be exercised on demand — the ticker is hourly and
+// deletion is measured in days.
+func StartReaper(ctx context.Context) {
+	window := reapAfter()
+	if window == 0 {
+		log.Printf("reaper: DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)")
+		return
+	}
+	log.Printf("reaper: ENABLED — Free instances are deleted %s after their licence expires", window)
+
+	go func() {
+		reapOnce(ctx)
+
+		t := time.NewTicker(ReapInterval)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				reapOnce(ctx)
+			}
+		}
+	}()
+}
+
+func reapOnce(ctx context.Context) {
+	runCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+
+	checked, purged, err := ReapFreeInstances(runCtx)
+	if err != nil {
+		log.Printf("reaper: %v", err)
+		return
+	}
+	if purged > 0 {
+		log.Printf("reaper: checked %d, purged %d", checked, purged)
+	}
+}