Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2d96cb4224 | ||
|
|
d2672fc022 | ||
|
|
ec22696402 | ||
|
|
913d3da386 |
@@ -999,6 +999,15 @@ plane, each of which this codebase enforces:
|
||||
- **`FREE_INSTANCE_REAP_AFTER` must match admin's value.** Admin names the date
|
||||
in its warning emails; this side performs the delete, because it is the only
|
||||
service that knows which collections carry `instance_id`.
|
||||
- **Disputes lock and purge instances.** HQ writes `instances.locked_at` when
|
||||
an account is disputed and `instances.purge_after` only once the dispute
|
||||
fails. `services.InstanceLocked` (60s cache) refuses sessions, API tokens
|
||||
and agents on a locked instance, and `GetInstanceBySlug` hides it from the
|
||||
host resolver. The check in `auth.Middleware` is explicit because the host
|
||||
guard only runs when a host resolves. `ReapTerminatedInstances` purges once
|
||||
both fields are present and `purge_after` has passed, independent of
|
||||
`FREE_INSTANCE_REAP_AFTER`, and `ReapFreeInstances` skips locked instances
|
||||
so a restore finds them intact.
|
||||
|
||||
## MongoDB Collections
|
||||
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.2
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.3.0
|
||||
github.com/bytedance/sonic v1.15.3 // indirect
|
||||
github.com/bytedance/sonic/loader v0.5.2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
|
||||
+3
-12
@@ -1,7 +1,5 @@
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.1 h1:rPzXSRwU+4+F2pdkmDrIxKsIzqz3S6feJEWalGmKqfU=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.1/go.mod h1:dWjeOFLltQ8sv9Pnn1xRxGfWGgqa2fkG0esuaJLoPXQ=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.2 h1:ibUp4zoh3GQgM298o5Mxmfv2cFM09uwWAPIDrE9yUjI=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.2/go.mod h1:dWjeOFLltQ8sv9Pnn1xRxGfWGgqa2fkG0esuaJLoPXQ=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.3.0 h1:6nSWrXsulzSCMEcr/2viu+uMyUdNOBfZ1JYtdiIzJH4=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.3.0/go.mod h1:dWjeOFLltQ8sv9Pnn1xRxGfWGgqa2fkG0esuaJLoPXQ=
|
||||
github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986 h1:2a30xLN2sUZcMXl50hg+PJCIDdJgIvIbVcKqLJ/ZrtM=
|
||||
github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986/go.mod h1:NT+jyeCzXk6vXR5MTkdn4z64TgGfE5HMLC8qfj5unl8=
|
||||
github.com/aquasecurity/trivy-db v0.0.0-20260813095258-0e0340a01b57 h1:A3Lz/9ip/qigafSxqBWcu7S8i+tJbQS7DB2V0XibOKs=
|
||||
@@ -182,16 +180,12 @@ golang.org/x/arch v0.31.0 h1:22MlEb14/O/EPCYHFxsDdv5TuLD5dMjT5e2QeJw4ULk=
|
||||
golang.org/x/arch v0.31.0/go.mod h1:KcJSod3cqT2dKcjBxqTyGfbumNikqU9p5tHJinPJnuY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
||||
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
|
||||
@@ -214,8 +208,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
@@ -223,9 +215,8 @@ golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func TestRefuseLocked(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
prev := instanceLocked
|
||||
t.Cleanup(func() { instanceLocked = prev })
|
||||
instanceLocked = func(id string) bool { return id == "locked" }
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
if !refuseLocked(c, "locked") {
|
||||
t.Fatal("a locked instance must be refused")
|
||||
}
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", w.Code)
|
||||
}
|
||||
|
||||
w = httptest.NewRecorder()
|
||||
c, _ = gin.CreateTestContext(w)
|
||||
if refuseLocked(c, "open") {
|
||||
t.Fatal("an unlocked instance must pass")
|
||||
}
|
||||
if c.IsAborted() {
|
||||
t.Fatal("an unlocked instance must not abort")
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,20 @@ import (
|
||||
|
||||
const ctxSessionKey = "km_session"
|
||||
|
||||
// instanceLocked is a variable so tests can stub the Mongo-backed check.
|
||||
var instanceLocked = services.InstanceLocked
|
||||
|
||||
// refuseLocked answers 401 for a session or token on an instance Vantage HQ has
|
||||
// locked under a dispute. Same body as an expired session: a locked instance
|
||||
// is not announced as locked to whoever holds a credential for it.
|
||||
func refuseLocked(c *gin.Context, instanceID string) bool {
|
||||
if !instanceLocked(instanceID) {
|
||||
return false
|
||||
}
|
||||
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "session expired"})
|
||||
return true
|
||||
}
|
||||
|
||||
func GetSessionFromContext(c *gin.Context) *Session {
|
||||
v, _ := c.Get(ctxSessionKey)
|
||||
sess, _ := v.(*Session)
|
||||
@@ -81,6 +95,13 @@ func Middleware() gin.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// Explicit, because the host guard below cannot do this: the resolver
|
||||
// hides a locked instance, so its host resolves to nothing and that
|
||||
// guard is skipped rather than tripped.
|
||||
if refuseLocked(c, sess.InstanceID) {
|
||||
return
|
||||
}
|
||||
|
||||
c.Set(ctxSessionKey, sess)
|
||||
|
||||
// The host guard applies to both credential kinds. A token carries an
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
// instanceLockTTL matches the host resolver's cache, so a lock and an unlock
|
||||
// both take effect within the same minute everywhere.
|
||||
const instanceLockTTL = 60 * time.Second
|
||||
|
||||
type lockEntry struct {
|
||||
locked bool
|
||||
at time.Time
|
||||
}
|
||||
|
||||
var (
|
||||
lockMu sync.Mutex
|
||||
lockCache = map[string]lockEntry{}
|
||||
|
||||
// lockLookup and lockNow are variables so tests can replace them.
|
||||
lockLookup = func(ctx context.Context, instanceID string) (bool, error) {
|
||||
n, err := db.Col("instances").CountDocuments(ctx, bson.M{
|
||||
"instance_id": instanceID,
|
||||
"locked_at": bson.M{"$exists": true},
|
||||
})
|
||||
return n > 0, err
|
||||
}
|
||||
lockNow = time.Now
|
||||
)
|
||||
|
||||
// InstanceLocked reports whether Vantage HQ has locked this instance under an
|
||||
// account dispute. HQ writes instances.locked_at through its cloudprov package;
|
||||
// nothing on this side ever sets or clears it.
|
||||
//
|
||||
// A read error answers false and is not cached. A database that cannot answer
|
||||
// this cannot serve the request that asked either, so failing open here costs
|
||||
// nothing that failing closed would save.
|
||||
func InstanceLocked(instanceID string) bool {
|
||||
if instanceID == "" {
|
||||
return false
|
||||
}
|
||||
lockMu.Lock()
|
||||
if e, ok := lockCache[instanceID]; ok && lockNow().Sub(e.at) < instanceLockTTL {
|
||||
lockMu.Unlock()
|
||||
return e.locked
|
||||
}
|
||||
lockMu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
locked, err := lockLookup(ctx, instanceID)
|
||||
if err != nil {
|
||||
log.Printf("instance lock: read %s: %v", instanceID, err)
|
||||
return false
|
||||
}
|
||||
lockMu.Lock()
|
||||
lockCache[instanceID] = lockEntry{locked: locked, at: lockNow()}
|
||||
lockMu.Unlock()
|
||||
return locked
|
||||
}
|
||||
|
||||
// unlockedFilter narrows an instance query to instances HQ has not locked.
|
||||
// HQ clears the field with $unset, so absence is the whole test.
|
||||
func unlockedFilter(f bson.M) bson.M {
|
||||
f["locked_at"] = bson.M{"$exists": false}
|
||||
return f
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
// stubLock replaces the Mongo lookup and the clock, and empties the cache, for
|
||||
// one test.
|
||||
func stubLock(t *testing.T, lookup func(context.Context, string) (bool, error), at *time.Time) {
|
||||
t.Helper()
|
||||
prevLookup, prevNow := lockLookup, lockNow
|
||||
lockLookup = lookup
|
||||
lockNow = func() time.Time { return *at }
|
||||
lockMu.Lock()
|
||||
lockCache = map[string]lockEntry{}
|
||||
lockMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
lockLookup, lockNow = prevLookup, prevNow
|
||||
lockMu.Lock()
|
||||
lockCache = map[string]lockEntry{}
|
||||
lockMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func TestInstanceLockedCachesForTTL(t *testing.T) {
|
||||
calls := 0
|
||||
at := time.Date(2026, 9, 10, 10, 0, 0, 0, time.UTC)
|
||||
stubLock(t, func(context.Context, string) (bool, error) { calls++; return true, nil }, &at)
|
||||
|
||||
if !InstanceLocked("i1") {
|
||||
t.Fatal("want locked")
|
||||
}
|
||||
InstanceLocked("i1")
|
||||
if calls != 1 {
|
||||
t.Fatalf("lookups = %d within the TTL, want 1", calls)
|
||||
}
|
||||
at = at.Add(instanceLockTTL + time.Second)
|
||||
InstanceLocked("i1")
|
||||
if calls != 2 {
|
||||
t.Fatalf("lookups = %d after the TTL, want 2", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstanceLockedReadErrorIsNotCached(t *testing.T) {
|
||||
calls := 0
|
||||
at := time.Date(2026, 9, 10, 10, 0, 0, 0, time.UTC)
|
||||
stubLock(t, func(context.Context, string) (bool, error) {
|
||||
calls++
|
||||
return false, errors.New("mongo down")
|
||||
}, &at)
|
||||
|
||||
if InstanceLocked("i1") {
|
||||
t.Fatal("a read error must answer false")
|
||||
}
|
||||
InstanceLocked("i1")
|
||||
if calls != 2 {
|
||||
t.Fatalf("lookups = %d, want 2: an error must not be cached", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstanceLockedEmptyID(t *testing.T) {
|
||||
at := time.Now()
|
||||
stubLock(t, func(context.Context, string) (bool, error) {
|
||||
t.Fatal("no lookup for an empty id")
|
||||
return false, nil
|
||||
}, &at)
|
||||
if InstanceLocked("") {
|
||||
t.Fatal("empty id is never locked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnlockedFilter(t *testing.T) {
|
||||
got := unlockedFilter(bson.M{"slug": "acme"})
|
||||
want := bson.M{"slug": "acme", "locked_at": bson.M{"$exists": false}}
|
||||
if len(got) != 2 || got["slug"] != "acme" {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
if cond, ok := got["locked_at"].(bson.M); !ok || cond["$exists"] != false {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,7 @@ func GetInstanceBySlug(slug string) (*models.Instance, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
var o models.Instance
|
||||
err := db.Col("instances").FindOne(ctx, bson.M{"slug": slug}).Decode(&o)
|
||||
err := db.Col("instances").FindOne(ctx, unlockedFilter(bson.M{"slug": slug})).Decode(&o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -58,12 +58,35 @@ func reapAfter() time.Duration {
|
||||
return d
|
||||
}
|
||||
|
||||
// freeReapFilter selects Free instances past the reap window. A locked instance
|
||||
// is excluded: it is under an open dispute in Vantage HQ, and a restore must
|
||||
// find it intact. Its fate is the dispute's to decide.
|
||||
func freeReapFilter(cutoff time.Time) bson.M {
|
||||
return bson.M{
|
||||
"license_tier": license.TierFree,
|
||||
"license_expiry": bson.M{"$ne": nil, "$lt": cutoff},
|
||||
"locked_at": bson.M{"$exists": false},
|
||||
}
|
||||
}
|
||||
|
||||
// terminatedPurgeFilter selects instances whose dispute failed in Vantage HQ.
|
||||
// Both fields are positive assertions and only HQ writes them: locked_at alone
|
||||
// is an open dispute, which must never be purged, and a self-hosted control
|
||||
// plane never has either.
|
||||
func terminatedPurgeFilter(now time.Time) bson.M {
|
||||
return bson.M{
|
||||
"locked_at": bson.M{"$exists": true},
|
||||
"purge_after": bson.M{"$exists": true, "$lt": now},
|
||||
}
|
||||
}
|
||||
|
||||
// purgeInstance deletes an instance and every document scoped to it.
|
||||
//
|
||||
// Unexported and unguarded: it trusts its caller completely and performs an
|
||||
// irreversible delete on whatever instance ID it is handed. The tier and expiry
|
||||
// gate - Free tier, an expiry that exists, an expiry past the window - lives in
|
||||
// ReapFreeInstances, which is the only caller. Do not export this.
|
||||
// ReapFreeInstances and ReapTerminatedInstances, which are the only callers.
|
||||
// Do not export this.
|
||||
//
|
||||
// Idempotent: re-running over a half-deleted instance completes it. The instance
|
||||
// document goes last, so an interrupted purge is retried on the next sweep
|
||||
@@ -107,10 +130,7 @@ func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) {
|
||||
}
|
||||
cutoff := time.Now().UTC().Add(-window)
|
||||
|
||||
cur, err := db.Col("instances").Find(ctx, bson.M{
|
||||
"license_tier": license.TierFree,
|
||||
"license_expiry": bson.M{"$ne": nil, "$lt": cutoff},
|
||||
})
|
||||
cur, err := db.Col("instances").Find(ctx, freeReapFilter(cutoff))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
@@ -152,20 +172,54 @@ func ReapFreeInstances(ctx context.Context) (checked, purged int, err error) {
|
||||
return checked, purged, nil
|
||||
}
|
||||
|
||||
// StartReaper sweeps once at boot, then on a ticker until ctx is cancelled, and
|
||||
// logs loudly which mode it is in.
|
||||
//
|
||||
// The pass at boot follows inject.StartReconciler's precedent and earns its keep
|
||||
// the same way: it makes a restart a supported way to force a sweep, which is
|
||||
// the only way this code can be exercised on demand - the ticker is hourly and
|
||||
// deletion is measured in days.
|
||||
func StartReaper(ctx context.Context) {
|
||||
window := reapAfter()
|
||||
if window == 0 {
|
||||
log.Printf("reaper: DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)")
|
||||
return
|
||||
// ReapTerminatedInstances deletes instances Vantage HQ has marked for purge
|
||||
// after a failed dispute. It does not depend on FREE_INSTANCE_REAP_AFTER:
|
||||
// this is an explicit staff decision, not a policy default.
|
||||
func ReapTerminatedInstances(ctx context.Context) (checked, purged int, err error) {
|
||||
cur, err := db.Col("instances").Find(ctx, terminatedPurgeFilter(time.Now().UTC()))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
log.Printf("reaper: ENABLED - Free instances are deleted %s after their licence expires", window)
|
||||
var doomed []struct {
|
||||
InstanceID string `bson:"instance_id"`
|
||||
Name string `bson:"name"`
|
||||
Slug string `bson:"slug"`
|
||||
PurgeAfter time.Time `bson:"purge_after"`
|
||||
}
|
||||
if err := cur.All(ctx, &doomed); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
for _, d := range doomed {
|
||||
checked++
|
||||
// Logged before the delete, for the same reason as the Free reaper:
|
||||
// afterwards there is nothing left to describe.
|
||||
log.Printf("REAPING terminated instance %s (%s, slug=%s) - dispute failed in Vantage HQ, purge authorised from %s",
|
||||
d.InstanceID, d.Name, d.Slug, d.PurgeAfter.Format(time.RFC3339))
|
||||
LogEvent(d.InstanceID, "instance.purged_terminated", "system", "", "",
|
||||
fmt.Sprintf("dispute failed in Vantage HQ, purge authorised from %s", d.PurgeAfter.Format(time.RFC3339)))
|
||||
|
||||
counts, err := purgeInstance(ctx, d.InstanceID)
|
||||
if err != nil {
|
||||
log.Printf("reaper: purge of terminated %s failed after %v: %v", d.InstanceID, counts, err)
|
||||
continue
|
||||
}
|
||||
purged++
|
||||
log.Printf("reaped terminated instance %s: %v", d.InstanceID, counts)
|
||||
}
|
||||
return checked, purged, nil
|
||||
}
|
||||
|
||||
// StartReaper sweeps once at boot, then hourly until ctx is cancelled. The
|
||||
// terminated sweep always runs; the Free sweep keeps its own off switch, and
|
||||
// the boot log names both modes.
|
||||
func StartReaper(ctx context.Context) {
|
||||
if window := reapAfter(); window == 0 {
|
||||
log.Printf("reaper: Free reaping DISABLED (FREE_INSTANCE_REAP_AFTER is unset or zero)")
|
||||
} else {
|
||||
log.Printf("reaper: Free reaping ENABLED - Free instances are deleted %s after their licence expires", window)
|
||||
}
|
||||
log.Printf("reaper: terminated purge ENABLED - instances Vantage HQ marks for purge are deleted once purge_after passes")
|
||||
|
||||
go func() {
|
||||
reapOnce(ctx)
|
||||
@@ -187,12 +241,14 @@ func reapOnce(ctx context.Context) {
|
||||
runCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
checked, purged, err := ReapFreeInstances(runCtx)
|
||||
if err != nil {
|
||||
log.Printf("reaper: %v", err)
|
||||
return
|
||||
if checked, purged, err := ReapFreeInstances(runCtx); err != nil {
|
||||
log.Printf("reaper: free: %v", err)
|
||||
} else if purged > 0 {
|
||||
log.Printf("reaper: free: checked %d, purged %d", checked, purged)
|
||||
}
|
||||
if purged > 0 {
|
||||
log.Printf("reaper: checked %d, purged %d", checked, purged)
|
||||
if checked, purged, err := ReapTerminatedInstances(runCtx); err != nil {
|
||||
log.Printf("reaper: terminated: %v", err)
|
||||
} else if purged > 0 {
|
||||
log.Printf("reaper: terminated: checked %d, purged %d", checked, purged)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.hostxtra.co.uk/vantage/vantage-shared/license"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
func TestFreeReapFilterSkipsLockedInstances(t *testing.T) {
|
||||
cutoff := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
|
||||
want := bson.M{
|
||||
"license_tier": license.TierFree,
|
||||
"license_expiry": bson.M{"$ne": nil, "$lt": cutoff},
|
||||
"locked_at": bson.M{"$exists": false},
|
||||
}
|
||||
if got := freeReapFilter(cutoff); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminatedPurgeFilterNeedsBothFields(t *testing.T) {
|
||||
now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)
|
||||
want := bson.M{
|
||||
"locked_at": bson.M{"$exists": true},
|
||||
"purge_after": bson.M{"$exists": true, "$lt": now},
|
||||
}
|
||||
if got := terminatedPurgeFilter(now); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -201,6 +201,12 @@ func ValidateAgentToken(serverID, agentToken string) (*models.Server, error) {
|
||||
if s.InstanceID == "" {
|
||||
return nil, fmt.Errorf("server %s has no org", serverID)
|
||||
}
|
||||
|
||||
// An agent of a locked instance is refused exactly like a bad token, so it
|
||||
// keeps retrying with backoff and reconnects on its own if HQ restores it.
|
||||
if InstanceLocked(s.InstanceID) {
|
||||
return nil, fmt.Errorf("invalid agent token")
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user