From 4e90e8619f36e730d9b4744fa1cb05edb8fbca08 Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Sun, 26 Jul 2026 13:22:04 +0100 Subject: [PATCH] feat(server): reap Free instances whose licence lapsed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The control plane owns deletion because it is the only service that knows what an instance is made of; mirroring that collection list into admin would drift, and a drift here deletes the wrong rows. Defaults OFF. Eligibility is three positive assertions — Free tier, an expiry that exists, and an expiry past the window — so a missing or stale field is never eligible. The instance document is deleted last, making an interrupted purge retryable rather than orphaning rows. Co-Authored-By: Claude Opus 5 --- server/cmd/main.go | 2 + server/internal/services/reap.go | 198 +++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 server/internal/services/reap.go diff --git a/server/cmd/main.go b/server/cmd/main.go index 12c8a22..1a84023 100644 --- a/server/cmd/main.go +++ b/server/cmd/main.go @@ -113,6 +113,8 @@ func main() { monitorsched.Start(context.Background()) + services.StartReaper(context.Background()) + r := gin.New() r.Use(gin.Recovery()) r.Use(gin.LoggerWithConfig(gin.LoggerConfig{SkipPaths: []string{"/api/console/tunnel"}})) diff --git a/server/internal/services/reap.go b/server/internal/services/reap.go new file mode 100644 index 0000000..99964a3 --- /dev/null +++ b/server/internal/services/reap.go @@ -0,0 +1,198 @@ +package services + +import ( + "context" + "fmt" + "log" + "os" + "time" + + "github.com/mrhid6/vantage/server/internal/db" + "github.com/mrhid6/vantage/shared/license" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// ReapInterval is how often eligibility is re-checked. Deletion is measured in +// days, so an hour is ample and keeps the query cheap. +const ReapInterval = time.Hour + +// scopedCollectionsForPurge is ScopedCollections minus `instances` itself. +// +// Derived rather than duplicated on purpose. ScopedCollections is the canonical +// list and AssertNoScopedCollectionMissed fails boot when a collection outside +// it holds a tenant key; a second hand-maintained copy here would silently miss +// whatever that assertion catches, leaking rows that outlive their instance. +// +// `instances` is excluded because it is keyed by instance_id rather than scoped +// by it, and is deleted last so an interrupted purge is retried rather than +// orphaning rows. +func scopedCollectionsForPurge() []string { + out := make([]string, 0, len(ScopedCollections)) + for _, name := range ScopedCollections { + if name == "instances" { + continue + } + out = append(out, name) + } + return out +} + +// reapAfter reads FREE_INSTANCE_REAP_AFTER. +// +// An empty or unparseable value returns 0, which disables the reaper. Defaulting +// OFF is the whole safety design: a deployment that never heard of this variable +// must never delete a customer's instance. +func reapAfter() time.Duration { + v := os.Getenv("FREE_INSTANCE_REAP_AFTER") + if v == "" { + return 0 + } + d, err := time.ParseDuration(v) + if err != nil { + log.Printf("reaper: FREE_INSTANCE_REAP_AFTER %q is not a duration; reaper stays OFF", v) + return 0 + } + if d <= 0 { + return 0 + } + return d +} + +// purgeInstance deletes an instance and every document scoped to it. +// +// Unexported and unguarded: it trusts its caller completely and performs an +// irreversible delete on whatever instance ID it is handed. The tier and expiry +// gate — Free tier, an expiry that exists, an expiry past the window — lives in +// ReapFreeInstances, which is the only caller. Do not export this. +// +// Idempotent: re-running over a half-deleted instance completes it. The instance +// document goes last, so an interrupted purge is retried on the next sweep +// instead of leaving rows behind with nothing pointing at them. +func purgeInstance(ctx context.Context, instanceID string) (map[string]int64, error) { + counts := map[string]int64{} + for _, name := range scopedCollectionsForPurge() { + res, err := db.Col(name).DeleteMany(ctx, bson.M{"instance_id": instanceID}) + if err != nil { + return counts, fmt.Errorf("purge %s: %w", name, err) + } + if res.DeletedCount > 0 { + counts[name] = res.DeletedCount + } + } + res, err := db.Col("instances").DeleteOne(ctx, bson.M{"instance_id": instanceID}) + if err != nil { + return counts, fmt.Errorf("purge instances: %w", err) + } + if res.DeletedCount > 0 { + counts["instances"] = res.DeletedCount + } + return counts, nil +} + +// ReapFreeInstances deletes Free cloud instances whose licence expired longer +// ago than the configured window. +// +// Eligibility requires ALL of: +// - license_tier == "free" — a paid instance is never eligible +// - license_expiry present — an instance that was never licensed, or whose +// issuance failed, has no expiry and is never eligible whatever its age +// - license_expiry older than now minus the window +// +// Every one of those is a positive assertion. Nothing is eligible by default, +// which is what makes a missing or stale field fail safe. +func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) { + window := reapAfter() + if window == 0 { + return 0, 0, nil + } + cutoff := time.Now().UTC().Add(-window) + + cur, err := db.Col("instances").Find(ctx, bson.M{ + "license_tier": license.TierFree, + "license_expiry": bson.M{"$ne": nil, "$lt": cutoff}, + }) + if err != nil { + return 0, 0, err + } + var doomed []struct { + InstanceID string `bson:"instance_id"` + Name string `bson:"name"` + Slug string `bson:"slug"` + Expiry time.Time `bson:"license_expiry"` + } + if err := cur.All(ctx, &doomed); err != nil { + return 0, 0, err + } + + for _, d := range doomed { + checked++ + + // Logged BEFORE the delete. Afterwards there is nothing left to + // describe, and "why did this instance vanish" is the only question + // anyone will ever ask about this code. + // + // The process log is the durable record, not the audit row: the purge + // deletes this instance's audit_logs along with everything else, so an + // audit entry written here would delete itself moments later. It is + // written anyway, because an operator reading audit during the window + // should see it coming. + log.Printf("REAPING instance %s (%s, slug=%s) — Free licence expired %s, past the %s window", + d.InstanceID, d.Name, d.Slug, d.Expiry.Format(time.RFC3339), window) + LogEvent(d.InstanceID, "instance.reaped", "system", "", "", + fmt.Sprintf("free licence expired %s, window %s", d.Expiry.Format(time.RFC3339), window)) + + counts, err := purgeInstance(ctx, d.InstanceID) + if err != nil { + log.Printf("reaper: purge of %s failed after %v: %v", d.InstanceID, counts, err) + continue + } + purged++ + log.Printf("reaped instance %s: %v", d.InstanceID, counts) + } + return checked, purged, nil +} + +// StartReaper sweeps once at boot, then on a ticker until ctx is cancelled, and +// logs loudly which mode it is in. +// +// The pass at boot follows inject.StartReconciler's precedent and earns its keep +// the same way: it makes a restart a supported way to force a sweep, which is +// the only way this code can be exercised on demand — the ticker is hourly and +// deletion is measured in days. +func StartReaper(ctx context.Context) { + window := reapAfter() + if window == 0 { + log.Printf("reaper: DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)") + return + } + log.Printf("reaper: ENABLED — Free instances are deleted %s after their licence expires", window) + + go func() { + reapOnce(ctx) + + t := time.NewTicker(ReapInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + reapOnce(ctx) + } + } + }() +} + +func reapOnce(ctx context.Context) { + runCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + + checked, purged, err := ReapFreeInstances(runCtx) + if err != nil { + log.Printf("reaper: %v", err) + return + } + if purged > 0 { + log.Printf("reaper: checked %d, purged %d", checked, purged) + } +}