mrhid6/vantage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

fix: Harden instance rename against interleaving and lost unwinds

Browse Source
This commit is contained in:
mrhid6
2026-08-12 11:08:36 +00:00
parent f534b74066
commit cdc50b7aaf
10 changed files with 220 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/internal/api/customer.go
+91 -13
View File
@@ -1,6 +1,7 @@
package api
import (
"context"
"errors"
"fmt"
"log"
@@ -23,6 +24,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
)
// ownedInstance resolves an instance and confirms the session's account owns it.
@@ -576,50 +578,126 @@ func renameInstance(c *gin.Context) {
return
}
if inst.RenamedAt != nil {
if until := inst.RenamedAt.Add(models.RenameCooldown); time.Now().UTC().Before(until) {
ctx := c.Request.Context()
// The unwind and the audit write run on a context detached from the request.
// The commonest reason the admin-side write fails at all is the caller
// walking away, and an unwind sharing that context fails with it — leaving
// the control plane renamed and admin's row not, which is the exact
// divergence this handler is arranged to prevent.
bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
// Claim the cooldown atomically BEFORE the control-plane call. Checking it
// and then acting lets two parallel PUTs both pass the check and then
// interleave their two-database writes, which ends with the two databases
// disagreeing about the host — a worse outcome than either rename losing.
// The conditional update IS the cooldown; there is no second reading of it.
now := time.Now().UTC()
var claimed models.Instance
err := db.Admin("admin_instances").FindOneAndUpdate(ctx,
bson.M{
"instance_id": inst.InstanceID,
"account_id": inst.AccountID,
"$or": []bson.M{
{"renamed_at": bson.M{"$exists": false}},
{"renamed_at": bson.M{"$lte": now.Add(-models.RenameCooldown)}},
},
},
bson.M{"$set": bson.M{"renamed_at": now}}).Decode(&claimed)
if err != nil {
if !errors.Is(err, mongo.ErrNoDocuments) {
log.Printf("renameInstance: claiming the cooldown on %s: %v", inst.InstanceID, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "could not rename the instance"})
return
}
// No match means the cooldown is live or the row has gone; only a
// re-read tells those apart, and they are different answers.
var cur models.Instance
if err := db.Admin("admin_instances").FindOne(ctx,
bson.M{"instance_id": inst.InstanceID, "account_id": inst.AccountID}).Decode(&cur); err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
return
}
if cur.RenamedAt != nil {
until := cur.RenamedAt.Add(models.RenameCooldown)
c.JSON(http.StatusTooManyRequests, gin.H{
"error": fmt.Sprintf("this instance was renamed recently; it can be renamed again after %s UTC", until.Format("2 Jan 2006 15:04")),
"retry_after": until,
})
return
}
// The row is here and its cooldown is spent, yet the claim matched
// nothing: it changed under us. Nothing has been written, so refuse
// rather than guess which way.
log.Printf("renameInstance: cooldown claim on %s matched nothing against an eligible row", inst.InstanceID)
c.JSON(http.StatusInternalServerError, gin.H{"error": "could not rename the instance"})
return
}
ctx := c.Request.Context()
renamed, err := cloudprov.RenameInstance(ctx, inst.InstanceID, name)
// releaseClaim puts renamed_at back to whatever the claim overwrote — the
// previous instant, or absent when there was none. Every failure past the
// claim owes the customer their rename back.
releaseClaim := func(after string) {
undo := bson.M{"$unset": bson.M{"renamed_at": ""}}
if claimed.RenamedAt != nil {
undo = bson.M{"$set": bson.M{"renamed_at": *claimed.RenamedAt}}
}
if _, err := db.Admin("admin_instances").UpdateOne(bgCtx,
bson.M{"instance_id": inst.InstanceID}, undo); err != nil {
log.Printf("renameInstance: releasing the cooldown claim on %s after %s: %v", inst.InstanceID, after, err)
}
}
renamed, prevName, prevSlug, err := cloudprov.RenameInstance(ctx, inst.InstanceID, name)
switch {
case errors.Is(err, provision.ErrSlugTaken):
releaseClaim("a taken slug")
c.JSON(http.StatusConflict, gin.H{"error": "that name is already in use — try another"})
return
case errors.Is(err, provision.ErrNameRejected):
releaseClaim("a rejected name")
c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()})
return
case err != nil:
releaseClaim("a failed control-plane rename")
log.Printf("renameInstance: control plane rename of %s: %v", inst.InstanceID, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "could not rename the instance"})
return
}
if _, err := db.Admin("admin_instances").UpdateOne(ctx,
// A matched count of zero is a silent version of the same failure: the
// control plane moved and admin's row did not.
res, err := db.Admin("admin_instances").UpdateOne(ctx,
bson.M{"instance_id": inst.InstanceID},
bson.M{"$set": bson.M{
"name": renamed.Name,
"slug": renamed.Slug,
"renamed_at": time.Now().UTC(),
}}); err != nil {
if rbErr := cloudprov.RestoreInstanceIdentity(ctx, inst.InstanceID, inst.Name, inst.Slug); rbErr != nil {
bson.M{"$set": bson.M{"name": renamed.Name, "slug": renamed.Slug}})
if err == nil && res.MatchedCount == 0 {
err = errors.New("admin_instances row matched nothing")
}
if err != nil {
// The control plane's own previous values, not admin's copy: admin's may
// be stale, and its slug is omitempty.
if rbErr := cloudprov.RestoreInstanceIdentity(bgCtx, inst.InstanceID, prevName, prevSlug); rbErr != nil {
log.Printf("renameInstance: rollback of %s failed: %v", inst.InstanceID, rbErr)
}
releaseClaim("a failed record write")
log.Printf("renameInstance: record rename of %s: %v", inst.InstanceID, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "could not rename the instance"})
return
}
if renamed.Slug == prevSlug {
// The cooldown exists because a rename moves the DNS host; a cosmetic
// edit that derives to the same slug moves nothing, so it should not
// spend one. The claim is already written by this point — releasing it
// is how that is expressed now the check is atomic.
releaseClaim("a rename that did not move the host")
}
s := auth.Current(c)
audit.Write(ctx, models.AuditEntry{
audit.Write(bgCtx, models.AuditEntry{
Actor: s.Email, Action: "instance.renamed", AccountID: s.AccountID,
Target: inst.InstanceID, Detail: inst.Slug + " -> " + renamed.Slug, IP: c.ClientIP()})
Target: inst.InstanceID, Detail: prevSlug + " -> " + renamed.Slug, IP: c.ClientIP()})
c.JSON(http.StatusOK, gin.H{
"instance_id": inst.InstanceID,
admin/internal/api/staff.go
+36 -8
View File
@@ -1,6 +1,7 @@
package api
import (
"context"
"errors"
"fmt"
"log"
@@ -631,6 +632,11 @@ func staffCreateAccountUser(c *gin.Context) {
// On self-hosted it changes admin's label only. There is no control-plane row to
// write — the install is the customer's — and no slug, because self-hosted has
// no tenant subdomain.
//
// A cloud placeholder is refused outright rather than relabelled: it has no
// control-plane row yet, so a label-only rename here would be a name that the
// instance never gets when provisioning finally derives its slug from the
// checkout's name. The customer endpoint refuses it for the same reason.
func staffRenameInstance(c *gin.Context) {
var body struct {
Name string `json:"name"`
@@ -652,12 +658,27 @@ func staffRenameInstance(c *gin.Context) {
c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
return
}
if inst.Deployment == license.DeploymentCloud && inst.Placeholder {
c.JSON(http.StatusConflict, gin.H{"error": "this instance is not provisioned yet"})
return
}
// The unwind and the audit write must survive the request being cancelled:
// an unwind on a dead context leaves the two databases disagreeing, which is
// the failure the unwind exists for.
bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
set := bson.M{"name": name}
slug := inst.Slug
cloud := inst.Deployment == license.DeploymentCloud
// The control plane's own previous values, not admin's copy: admin's may be
// stale, and its slug is omitempty, so unwinding from it can write an empty
// slug into instances.
prevName, prevSlug := inst.Name, inst.Slug
if inst.Deployment == license.DeploymentCloud && !inst.Placeholder {
renamed, err := cloudprov.RenameInstance(ctx, inst.InstanceID, name)
if cloud {
renamed, pName, pSlug, err := cloudprov.RenameInstance(ctx, inst.InstanceID, name)
switch {
case errors.Is(err, provision.ErrSlugTaken):
c.JSON(http.StatusConflict, gin.H{"error": "that name is already in use"})
@@ -669,14 +690,21 @@ func staffRenameInstance(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
prevName, prevSlug = pName, pSlug
slug = renamed.Slug
set["slug"] = renamed.Slug
}
if _, err := db.Admin("admin_instances").UpdateOne(ctx,
bson.M{"instance_id": inst.InstanceID}, bson.M{"$set": set}); err != nil {
if inst.Deployment == license.DeploymentCloud && !inst.Placeholder {
if rbErr := cloudprov.RestoreInstanceIdentity(ctx, inst.InstanceID, inst.Name, inst.Slug); rbErr != nil {
// A matched count of zero is the same failure quietly: the control plane
// moved and admin's row did not.
res, err := db.Admin("admin_instances").UpdateOne(ctx,
bson.M{"instance_id": inst.InstanceID}, bson.M{"$set": set})
if err == nil && res.MatchedCount == 0 {
err = errors.New("admin_instances row matched nothing")
}
if err != nil {
if cloud {
if rbErr := cloudprov.RestoreInstanceIdentity(bgCtx, inst.InstanceID, prevName, prevSlug); rbErr != nil {
log.Printf("staffRenameInstance: rollback of %s failed: %v", inst.InstanceID, rbErr)
}
}
@@ -684,9 +712,9 @@ func staffRenameInstance(c *gin.Context) {
return
}
audit.Write(ctx, models.AuditEntry{
audit.Write(bgCtx, models.AuditEntry{
Actor: auth.Current(c).Email, Action: "instance.renamed", AccountID: inst.AccountID,
Target: inst.InstanceID, Detail: inst.Slug + " -> " + slug, IP: c.ClientIP()})
Target: inst.InstanceID, Detail: prevSlug + " -> " + slug, IP: c.ClientIP()})
c.JSON(http.StatusOK, gin.H{"instance_id": inst.InstanceID, "name": name, "slug": slug})
}
admin/internal/cloudprov/cloudprov.go
+4 -1
View File
@@ -209,7 +209,10 @@ func ProjectedUsers(ctx context.Context, hqUserID string) ([]sharedmodels.User,
// It writes `instances` and nothing else, so admin's control-plane write
// boundary is unchanged. It issues no licence: a licence binds the instance
// UUID, which a rename never touches.
func RenameInstance(ctx context.Context, instanceID, name string) (*sharedmodels.Instance, error) {
//
// The previous name and slug come back with the result because they are what an
// unwind must restore — admin's own copy can be stale, or slugless.
func RenameInstance(ctx context.Context, instanceID, name string) (inst *sharedmodels.Instance, prevName, prevSlug string, err error) {
return provision.RenameInstance(ctx, db.ControlDB(), instanceID, name)
}
admin/internal/models/models.go
+2 -2
View File
@@ -150,8 +150,8 @@ type Instance struct {
// rename cooldown. It is a pointer because absent means "never renamed"; a
// zero time.Time would read as year 1 — an inert cooldown, but only by
// accident. Staff renames deliberately leave it alone.
RenamedAt *time.Time `bson:"renamed_at,omitempty" json:"renamed_at,omitempty"`
InjectFailedAt *time.Time `bson:"inject_failed_at,omitempty" json:"inject_failed_at,omitempty"`
RenamedAt *time.Time `bson:"renamed_at,omitempty" json:"renamed_at,omitempty"`
InjectFailedAt *time.Time `bson:"inject_failed_at,omitempty" json:"inject_failed_at,omitempty"`
// NoticesSent holds the lifecycle notice keys already emailed for the
// CURRENT term ("expiring", "expired", "delete_7", "delete_1"). Renewal
// clears it, so the next term starts the sequence again. It is what stops a