fix: Fixes to command stream
Chart Release / chart (push) Successful in 11s
Server Deploy / deploy (push) Successful in 2m36s

This commit is contained in:
2026-08-10 14:07:17 +01:00
parent 727bb09eff
commit ef86ef04a1
4 changed files with 119 additions and 22 deletions
+51 -6
View File
@@ -204,8 +204,31 @@ func SetPresence(ctx context.Context, serverID string, ttl time.Duration) error
return rdb.Set(ctx, PresenceKey+serverID, nodeID, ttl).Err()
}
// RenewPresence extends serverID's claim, but only while this node still holds
// it, and reports whether it did.
// RenewResult is the outcome of one presence renewal.
//
// Four outcomes, not two, because the three ways a renewal can fail to extend
// an existing claim call for three different responses. Collapsing them into a
// single false is what made a momentary Redis blip permanent: the renewal loop
// treated "the key is gone" and "Redis did not answer" as "another pod owns
// this agent now" and stopped renewing for the life of a stream that was still
// perfectly healthy, leaving the agent connected and undispatchable.
type RenewResult int
const (
// RenewFailed means Redis could not be reached. Nothing is known about who
// holds the claim, so the only safe response is to try again.
RenewFailed RenewResult = iota
// RenewedOwner means the claim was ours and its TTL was extended.
RenewedOwner
// RenewedClaim means nobody held the claim and this node took it. Normal
// after a Redis restart, failover, eviction or an outage outliving the TTL.
RenewedClaim
// RenewLost means another node holds the claim. This stream is superseded.
RenewLost
)
// RenewPresence extends serverID's claim while this node holds it, and reclaims
// it if nobody does.
//
// A blind SET here is wrong, not merely untidy. When an agent reconnects, its
// previous stream can stay half-open on another pod for the length of a
@@ -213,10 +236,25 @@ func SetPresence(ctx context.Context, serverID string, ttl time.Duration) error
// each other's claim every renewal interval and the key names whichever wrote
// last rather than whichever holds the live stream. A superseded pod must lose
// quietly instead.
func RenewPresence(ctx context.Context, serverID string, ttl time.Duration) bool {
//
// Reclaiming an *absent* key is not that case and is safe: absence means no pod
// is currently advertising the agent, and the caller demonstrably holds a live
// stream to it. Refusing to reclaim is what leaves an agent unreachable until it
// happens to reconnect.
func RenewPresence(ctx context.Context, serverID string, ttl time.Duration) RenewResult {
n, err := renewPresenceIfOwner.Run(ctx, rdb,
[]string{PresenceKey + serverID}, nodeID, int64(ttl/time.Millisecond)).Int64()
return err == nil && n == 1
if err != nil {
return RenewFailed
}
switch n {
case 1:
return RenewedOwner
case 2:
return RenewedClaim
default:
return RenewLost
}
}
// ClearPresence releases serverID, but only if this node still holds it. A
@@ -289,9 +327,16 @@ end
return ""
`)
// 1 = renewed, 2 = reclaimed an unheld key, 0 = held by another node.
var renewPresenceIfOwner = redis.NewScript(`
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
local v = redis.call("GET", KEYS[1])
if v == ARGV[1] then
redis.call("PEXPIRE", KEYS[1], ARGV[2])
return 1
end
if not v then
redis.call("SET", KEYS[1], ARGV[1], "PX", ARGV[2])
return 2
end
return 0
`)