fix(patching): final review fixes
- no dispatch in the last 15 minutes of a window; no-result timeout from dispatch time - per-server output moves to patch_run_outputs (16MB document limit) - reboot proven by a changed boot time; RebootTimeout 45m, ResultGrace 20m - window update and delete are server-scoped against the policies using them - scheduler puts the claim back on an error after it, so the next tick retries - cancelled runs with failures alert; MCP apply_updates audits per server - apply-updates 503 body documented; openapi regenerated - web: cleared numeric fields no longer save as 0; Run now asks for confirmation
This commit is contained in:
@@ -17,6 +17,17 @@
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"api.ApplyUpdatesErrorResponse": {
|
||||
"properties": {
|
||||
"error": {
|
||||
"type": "string"
|
||||
},
|
||||
"run_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"api.ApplyUpdatesResponse": {
|
||||
"properties": {
|
||||
"message": {
|
||||
@@ -1077,6 +1088,10 @@
|
||||
},
|
||||
"models.Inventory": {
|
||||
"properties": {
|
||||
"boot_time": {
|
||||
"description": "BootTime is the host's last reported boot time, stored on every report\nthat carries one so a patch reboot can be proven by a changed boot.",
|
||||
"type": "string"
|
||||
},
|
||||
"cpu": {
|
||||
"$ref": "#/components/schemas/models.CPUInfo"
|
||||
},
|
||||
@@ -7203,11 +7218,11 @@
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/api.ApplyUpdatesResponse"
|
||||
"$ref": "#/components/schemas/api.ApplyUpdatesErrorResponse"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Service Unavailable"
|
||||
"description": "agent offline; the attempt is recorded as run_id"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
|
||||
@@ -756,7 +756,7 @@ func updateAgent(c *gin.Context) {
|
||||
// @Param source query string false "vulnerabilities when started from the vulnerabilities page"
|
||||
// @Success 202 {object} ApplyUpdatesResponse
|
||||
// @Failure 404 {object} ErrorResponse
|
||||
// @Failure 503 {object} ApplyUpdatesResponse
|
||||
// @Failure 503 {object} ApplyUpdatesErrorResponse "agent offline; the attempt is recorded as run_id"
|
||||
// @Security cookieAuth
|
||||
// @Security bearerAuth
|
||||
// @Router /servers/{id}/apply-updates [post]
|
||||
|
||||
@@ -170,6 +170,10 @@ func updateWindow(c *gin.Context) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if err := services.CheckWindowScope(auth.InstanceID(c), c.Param("id"), auth.ServerScope(c)); err != nil {
|
||||
patchError(c, err)
|
||||
return
|
||||
}
|
||||
w, err := services.UpdateWindow(auth.InstanceID(c), c.Param("id"), body)
|
||||
if err != nil {
|
||||
patchError(c, err)
|
||||
@@ -192,6 +196,10 @@ func updateWindow(c *gin.Context) {
|
||||
// @Security bearerAuth
|
||||
// @Router /maintenance-windows/{id} [delete]
|
||||
func deleteWindow(c *gin.Context) {
|
||||
if err := services.CheckWindowScope(auth.InstanceID(c), c.Param("id"), auth.ServerScope(c)); err != nil {
|
||||
patchError(c, err)
|
||||
return
|
||||
}
|
||||
if err := services.DeleteWindow(auth.InstanceID(c), c.Param("id")); err != nil {
|
||||
patchError(c, err)
|
||||
return
|
||||
|
||||
@@ -400,13 +400,16 @@ var serverScopedRoutes = map[string]scopeDecl{
|
||||
"GET /api/mcp": exempt,
|
||||
|
||||
// Maintenance windows are a cron expression, a zone and a duration. They
|
||||
// name no server and return no server data.
|
||||
// name no server and return no server data, so reading and creating one
|
||||
// is exempt. Changing or deleting one moves or stops the patching of
|
||||
// every policy using it, so each is refused when any of those policies
|
||||
// targets servers outside the token's tag restriction.
|
||||
"GET /api/maintenance-windows": exempt,
|
||||
"POST /api/maintenance-windows": exempt,
|
||||
"POST /api/maintenance-windows/preview": exempt,
|
||||
"GET /api/maintenance-windows/:id": exempt,
|
||||
"PUT /api/maintenance-windows/:id": exempt,
|
||||
"DELETE /api/maintenance-windows/:id": exempt,
|
||||
"PUT /api/maintenance-windows/:id": scoped,
|
||||
"DELETE /api/maintenance-windows/:id": scoped,
|
||||
// Reading a policy returns its selector (server IDs and tag pairs) and no
|
||||
// hostname, inventory or state, the same data a workflow's targets carry.
|
||||
"GET /api/patch-policies": exempt,
|
||||
|
||||
@@ -307,6 +307,13 @@ type ApplyUpdatesResponse struct {
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
}
|
||||
|
||||
// ApplyUpdatesErrorResponse is the 503 body of apply-updates: the agent is
|
||||
// offline, and the attempt is still recorded as a run.
|
||||
type ApplyUpdatesErrorResponse struct {
|
||||
Error string `json:"error"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
}
|
||||
|
||||
// WindowPreviewRequest is the body of POST /maintenance-windows/preview.
|
||||
type WindowPreviewRequest struct {
|
||||
Cron string `json:"cron"`
|
||||
|
||||
@@ -162,12 +162,11 @@ type updateBatchResult struct {
|
||||
|
||||
// apply_updates. The REST route (internal/api/handlers.go's applyUpdates) is
|
||||
// per-server: POST /servers/:id/apply-updates resolves one server with
|
||||
// services.GetServerScoped and calls services.DispatchApplyUpdates(serverID).
|
||||
// There is no fleet-wide variant of that service call to invoke once, so this
|
||||
// tool resolves the requested targets through ResolveTargetsScoped exactly as
|
||||
// the brief describes, then calls the same DispatchApplyUpdates the REST route
|
||||
// calls, once per resolved server - the identical dispatch, just looped
|
||||
// instead of hardcoded to one server_id from the URL.
|
||||
// services.GetServerScoped and calls services.StartManualRun. There is no
|
||||
// fleet-wide variant of that service call to invoke once, so this tool
|
||||
// resolves the requested targets through ResolveTargetsScoped, then calls the
|
||||
// same StartManualRun once per resolved server and writes the same
|
||||
// updates.applied audit event per server with its run ID.
|
||||
func init() {
|
||||
All().Register(Tool{
|
||||
Name: "apply_updates",
|
||||
@@ -201,6 +200,8 @@ func init() {
|
||||
result.Failed[srv.ServerID] = err.Error()
|
||||
continue
|
||||
}
|
||||
services.LogEvent(c.InstanceID, "updates.applied", "mcp:"+c.TokenName, srv.ServerID, "",
|
||||
fmt.Sprintf("package update run %s started on %s", run.RunID, srv.Hostname))
|
||||
result.Succeeded = append(result.Succeeded, srv.ServerID)
|
||||
if result.RunIDs == nil {
|
||||
result.RunIDs = map[string]string{}
|
||||
|
||||
@@ -105,8 +105,12 @@ type PatchServerRun struct {
|
||||
PendingAfter *int `bson:"pending_after,omitempty" json:"pending_after,omitempty"`
|
||||
RebootedAt *time.Time `bson:"rebooted_at,omitempty" json:"rebooted_at,omitempty"`
|
||||
VerifiedAt *time.Time `bson:"verified_at,omitempty" json:"verified_at,omitempty"`
|
||||
Output string `bson:"output,omitempty" json:"output,omitempty"`
|
||||
Error string `bson:"error,omitempty" json:"error,omitempty"`
|
||||
StartedAt *time.Time `bson:"started_at,omitempty" json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `bson:"finished_at,omitempty" json:"finished_at,omitempty"`
|
||||
// BootTimeBefore is the host's reported boot time when the reboot was
|
||||
// announced. A later report with a different boot time proves the
|
||||
// restart without comparing the host clock to the server clock.
|
||||
BootTimeBefore *time.Time `bson:"boot_time_before,omitempty" json:"-"`
|
||||
Output string `bson:"output,omitempty" json:"output,omitempty"`
|
||||
Error string `bson:"error,omitempty" json:"error,omitempty"`
|
||||
StartedAt *time.Time `bson:"started_at,omitempty" json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `bson:"finished_at,omitempty" json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
@@ -42,6 +42,9 @@ type Inventory struct {
|
||||
RebootRequired bool `bson:"reboot_required,omitempty" json:"reboot_required,omitempty"`
|
||||
MetricsAt *time.Time `bson:"metrics_at,omitempty" json:"metrics_at,omitempty"`
|
||||
StaticAt *time.Time `bson:"static_at,omitempty" json:"static_at,omitempty"`
|
||||
// BootTime is the host's last reported boot time, stored on every report
|
||||
// that carries one so a patch reboot can be proven by a changed boot.
|
||||
BootTime *time.Time `bson:"boot_time,omitempty" json:"boot_time,omitempty"`
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
|
||||
@@ -20,9 +20,19 @@ import (
|
||||
const MinAgentVersion = "1.4.0"
|
||||
|
||||
const (
|
||||
ResultGrace = 10 * time.Minute // after the window deadline, before "no result"
|
||||
ManualTimeout = 2 * time.Hour // the agent's own cap when no deadline is sent
|
||||
RebootTimeout = 20 * time.Minute // for a post-boot inventory report
|
||||
// ResultGrace covers what the agent does after the upgrade and before it
|
||||
// answers: a pending-update re-check of up to 10 minutes (Windows Update
|
||||
// search) plus a 2 minute reboot check, with room to spare.
|
||||
ResultGrace = 20 * time.Minute
|
||||
// ManualTimeout is the agent's backstop for one started upgrade, counted
|
||||
// from its own start. The window end never stops a running upgrade.
|
||||
ManualTimeout = 2 * time.Hour
|
||||
// RebootTimeout is how long a rebooting server has to send a post-boot
|
||||
// inventory report. Windows cumulative updates routinely take over 20.
|
||||
RebootTimeout = 45 * time.Minute
|
||||
// LatestStartBeforeEnd is the tail of a window in which no server starts
|
||||
// patching: a late start would run long past the window end.
|
||||
LatestStartBeforeEnd = 15 * time.Minute
|
||||
)
|
||||
|
||||
// AgentSupportsPatchResults compares major.minor.patch. Empty, "dev" and
|
||||
@@ -88,6 +98,9 @@ func Advance(run models.PatchRun, now time.Time, connected map[string]bool) []Tr
|
||||
return nil
|
||||
}
|
||||
windowOpen := run.WindowEnd == nil || now.Before(*run.WindowEnd)
|
||||
// In the last LatestStartBeforeEnd of a window nothing new starts: queued
|
||||
// and waiting servers simply wait, and close at WindowEnd as usual.
|
||||
mayStart := run.WindowEnd == nil || now.Before(run.WindowEnd.Add(-LatestStartBeforeEnd))
|
||||
|
||||
inFlight := 0
|
||||
for _, s := range run.Servers {
|
||||
@@ -109,6 +122,8 @@ func Advance(run models.PatchRun, now time.Time, connected map[string]bool) []Tr
|
||||
to = models.PatchSrvMissedOffline
|
||||
}
|
||||
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: to})
|
||||
case !mayStart:
|
||||
// The window tail: no dispatch, no transition.
|
||||
case run.MaxConcurrent > 0 && inFlight >= run.MaxConcurrent:
|
||||
// No slot this tick.
|
||||
case !connected[s.ServerID]:
|
||||
@@ -125,17 +140,17 @@ func Advance(run models.PatchRun, now time.Time, connected map[string]bool) []Tr
|
||||
}
|
||||
case models.PatchSrvRebooting:
|
||||
if s.RebootedAt != nil && now.After(s.RebootedAt.Add(RebootTimeout)) {
|
||||
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvFailed, Error: "did not come back within 20 minutes"})
|
||||
out = append(out, Transition{ServerID: s.ServerID, From: s.Status, To: models.PatchSrvFailed, Error: fmt.Sprintf("did not come back within %d minutes", int(RebootTimeout.Minutes()))})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// resultDeadline is the same for windowed and manual runs: the agent lets a
|
||||
// started upgrade finish past the window end, so the window end says nothing
|
||||
// about when a result is due. The base is the server's dispatch time.
|
||||
func resultDeadline(run models.PatchRun, s models.PatchServerRun) time.Time {
|
||||
if run.WindowEnd != nil {
|
||||
return run.WindowEnd.Add(ResultGrace)
|
||||
}
|
||||
start := run.StartedAt
|
||||
if s.StartedAt != nil {
|
||||
start = *s.StartedAt
|
||||
@@ -176,11 +191,21 @@ func ApplyResult(s models.PatchServerRun, r *pb.PatchResult, now time.Time) (mod
|
||||
return s, true
|
||||
}
|
||||
|
||||
// VerifyReboot settles a rebooting server from a static inventory report. Only
|
||||
// a boot time later than the reboot command proves the host restarted: a
|
||||
// snapshot sent during the one-minute grace period must not count.
|
||||
// VerifyReboot settles a rebooting server from a static inventory report.
|
||||
// When the boot time reported before the reboot is known, a later boot time
|
||||
// is the proof: both come from the host clock, so skew against the server
|
||||
// clock does not matter. Otherwise only a boot time later than the reboot
|
||||
// command counts. Either way a snapshot sent during the one-minute grace
|
||||
// period, before the host went down, does not.
|
||||
func VerifyReboot(s models.PatchServerRun, bootTime time.Time, rebootRequired bool, now time.Time) (models.PatchServerRun, bool) {
|
||||
if s.Status != models.PatchSrvRebooting || s.RebootedAt == nil || !bootTime.After(*s.RebootedAt) {
|
||||
if s.Status != models.PatchSrvRebooting || s.RebootedAt == nil {
|
||||
return s, false
|
||||
}
|
||||
proven := bootTime.After(*s.RebootedAt)
|
||||
if s.BootTimeBefore != nil {
|
||||
proven = bootTime.After(*s.BootTimeBefore)
|
||||
}
|
||||
if !proven {
|
||||
return s, false
|
||||
}
|
||||
if rebootRequired {
|
||||
|
||||
@@ -93,36 +93,90 @@ func TestAdvanceWindowCloses(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvanceNoResultTimeout(t *testing.T) {
|
||||
// A windowed run times out from the server's own dispatch time, not from the
|
||||
// window end: a server dispatched late in the window may finish past it.
|
||||
func TestAdvanceNoResultTimeoutFromDispatch(t *testing.T) {
|
||||
run := windowRun(0, srv("a", models.PatchSrvPatching))
|
||||
if ts := Advance(run, t0.Add(2*time.Hour+9*time.Minute), nil); len(ts) != 0 {
|
||||
t.Fatalf("inside grace: %+v", ts)
|
||||
run.Servers[0].StartedAt = tp(t0.Add(90 * time.Minute))
|
||||
deadline := t0.Add(90*time.Minute + ManualTimeout + ResultGrace)
|
||||
if ts := Advance(run, run.WindowEnd.Add(ResultGrace+time.Minute), nil); len(ts) != 0 {
|
||||
t.Fatalf("past WindowEnd+grace but inside the dispatch timeout: %+v", ts)
|
||||
}
|
||||
ts := Advance(run, t0.Add(2*time.Hour+11*time.Minute), nil)
|
||||
if ts := Advance(run, deadline.Add(-time.Minute), nil); len(ts) != 0 {
|
||||
t.Fatalf("inside the dispatch timeout: %+v", ts)
|
||||
}
|
||||
ts := Advance(run, deadline.Add(time.Minute), nil)
|
||||
if len(ts) != 1 || ts[0].To != models.PatchSrvFailed || ts[0].Error == "" {
|
||||
t.Fatalf("past grace: %+v", ts)
|
||||
t.Fatalf("past the dispatch timeout: %+v", ts)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a server StartedAt the run's own start is the base.
|
||||
func TestAdvanceNoResultTimeoutFallsBackToRunStart(t *testing.T) {
|
||||
run := windowRun(0, srv("a", models.PatchSrvPatching))
|
||||
if ts := Advance(run, t0.Add(2*time.Hour+19*time.Minute), nil); len(ts) != 0 {
|
||||
t.Fatalf("inside timeout: %+v", ts)
|
||||
}
|
||||
if ts := Advance(run, t0.Add(2*time.Hour+21*time.Minute), nil); len(ts) != 1 || ts[0].To != models.PatchSrvFailed {
|
||||
t.Fatalf("past timeout: %+v", ts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvanceManualRunTimeout(t *testing.T) {
|
||||
run := models.PatchRun{Status: models.PatchRunRunning, StartedAt: t0, Servers: []models.PatchServerRun{srv("a", models.PatchSrvPatching)}}
|
||||
run.Servers[0].StartedAt = tp(t0)
|
||||
if ts := Advance(run, t0.Add(2*time.Hour+9*time.Minute), nil); len(ts) != 0 {
|
||||
if ts := Advance(run, t0.Add(2*time.Hour+19*time.Minute), nil); len(ts) != 0 {
|
||||
t.Fatalf("manual inside timeout: %+v", ts)
|
||||
}
|
||||
if ts := Advance(run, t0.Add(2*time.Hour+11*time.Minute), nil); len(ts) != 1 || ts[0].To != models.PatchSrvFailed {
|
||||
if ts := Advance(run, t0.Add(2*time.Hour+21*time.Minute), nil); len(ts) != 1 || ts[0].To != models.PatchSrvFailed {
|
||||
t.Fatalf("manual past timeout: %+v", ts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimingConstants(t *testing.T) {
|
||||
if ResultGrace != 20*time.Minute || RebootTimeout != 45*time.Minute || LatestStartBeforeEnd != 15*time.Minute {
|
||||
t.Fatalf("ResultGrace=%v RebootTimeout=%v LatestStartBeforeEnd=%v", ResultGrace, RebootTimeout, LatestStartBeforeEnd)
|
||||
}
|
||||
}
|
||||
|
||||
// No server starts patching in the last 15 minutes of a window: it would
|
||||
// either be cut short or run long past the window end.
|
||||
func TestAdvanceNoDispatchInWindowTail(t *testing.T) {
|
||||
run := windowRun(0, srv("q", models.PatchSrvQueued), srv("w", models.PatchSrvWaitingOffline), srv("o", models.PatchSrvQueued))
|
||||
cutoff := run.WindowEnd.Add(-LatestStartBeforeEnd)
|
||||
online := map[string]bool{"q": true, "w": true}
|
||||
for _, at := range []time.Time{cutoff, cutoff.Add(time.Minute), run.WindowEnd.Add(-time.Second)} {
|
||||
if ts := Advance(run, at, online); len(ts) != 0 {
|
||||
t.Fatalf("at %v: nothing may change in the window tail, got %+v", at.Sub(t0), ts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvanceDispatchJustBeforeWindowTail(t *testing.T) {
|
||||
run := windowRun(0, srv("q", models.PatchSrvQueued))
|
||||
at := run.WindowEnd.Add(-LatestStartBeforeEnd - time.Second)
|
||||
ts := Advance(run, at, map[string]bool{"q": true})
|
||||
if len(ts) != 1 || !ts[0].Dispatch {
|
||||
t.Fatalf("dispatch must be allowed just before the tail: %+v", ts)
|
||||
}
|
||||
}
|
||||
|
||||
// A manual run has no window and no tail.
|
||||
func TestAdvanceManualRunHasNoTail(t *testing.T) {
|
||||
run := models.PatchRun{Status: models.PatchRunRunning, StartedAt: t0, Servers: []models.PatchServerRun{srv("a", models.PatchSrvQueued)}}
|
||||
if ts := Advance(run, t0.Add(10*time.Hour), map[string]bool{"a": true}); len(ts) != 1 || !ts[0].Dispatch {
|
||||
t.Fatalf("manual run must dispatch: %+v", ts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvanceRebootTimeout(t *testing.T) {
|
||||
run := windowRun(0, srv("a", models.PatchSrvRebooting))
|
||||
run.Servers[0].RebootedAt = tp(t0)
|
||||
if ts := Advance(run, t0.Add(19*time.Minute), nil); len(ts) != 0 {
|
||||
if ts := Advance(run, t0.Add(44*time.Minute), nil); len(ts) != 0 {
|
||||
t.Fatalf("inside reboot timeout: %+v", ts)
|
||||
}
|
||||
ts := Advance(run, t0.Add(21*time.Minute), nil)
|
||||
if len(ts) != 1 || ts[0].To != models.PatchSrvFailed {
|
||||
ts := Advance(run, t0.Add(46*time.Minute), nil)
|
||||
if len(ts) != 1 || ts[0].To != models.PatchSrvFailed || ts[0].Error != "did not come back within 45 minutes" {
|
||||
t.Fatalf("past reboot timeout: %+v", ts)
|
||||
}
|
||||
}
|
||||
@@ -202,6 +256,28 @@ func TestVerifyReboot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// With the boot time recorded before the reboot, a changed boot time is the
|
||||
// proof, whatever the skew between the host clock and the server clock.
|
||||
func TestVerifyRebootChangedBoot(t *testing.T) {
|
||||
s := srv("a", models.PatchSrvRebooting)
|
||||
s.RebootedAt = tp(t0)
|
||||
s.BootTimeBefore = tp(t0.Add(-10 * 24 * time.Hour))
|
||||
now := t0.Add(5 * time.Minute)
|
||||
// The host clock runs 10 minutes slow: its new boot time reads earlier
|
||||
// than the server's RebootedAt, yet the boot did change.
|
||||
got, ok := VerifyReboot(s, t0.Add(-8*time.Minute), false, now)
|
||||
if !ok || got.Status != models.PatchSrvSucceeded {
|
||||
t.Fatalf("changed boot behind a slow clock must be proven: %+v ok=%v", got, ok)
|
||||
}
|
||||
// A fast host clock with an unchanged boot is not proof.
|
||||
if _, ok := VerifyReboot(s, *s.BootTimeBefore, false, now); ok {
|
||||
t.Error("an unchanged boot time is not proof")
|
||||
}
|
||||
if _, ok := VerifyReboot(s, s.BootTimeBefore.Add(-time.Minute), false, now); ok {
|
||||
t.Error("an earlier boot time is not proof")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalize(t *testing.T) {
|
||||
mk := func(statuses ...string) models.PatchRun {
|
||||
r := windowRun(0)
|
||||
|
||||
@@ -96,18 +96,18 @@ func process(ctx context.Context, deps Deps, p models.PatchPolicy, now time.Time
|
||||
|
||||
n, err := deps.CountTargets(p)
|
||||
if err != nil {
|
||||
recordSkip(ctx, deps, p, "error: "+err.Error(), due, now)
|
||||
retryLater(ctx, deps, p, err, due, next, now)
|
||||
return
|
||||
}
|
||||
active, err := hasActiveRun(ctx, p)
|
||||
if err != nil {
|
||||
recordSkip(ctx, deps, p, "error: "+err.Error(), due, now)
|
||||
retryLater(ctx, deps, p, err, due, next, now)
|
||||
return
|
||||
}
|
||||
switch d := Decide(due, end, now, active, n); d {
|
||||
case Fire:
|
||||
if err := deps.StartPolicyRun(p, end); err != nil {
|
||||
recordSkip(ctx, deps, p, "error: "+err.Error(), due, now)
|
||||
retryLater(ctx, deps, p, err, due, next, now)
|
||||
return
|
||||
}
|
||||
_, _ = db.Col("patch_policies").UpdateOne(ctx, bson.M{"policy_id": p.PolicyID},
|
||||
@@ -130,6 +130,19 @@ func hasActiveRun(ctx context.Context, p models.PatchPolicy) (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// retryLater handles an error after the claim. The claim is put back, guarded
|
||||
// on the value just written so a concurrent edit to the policy is not undone,
|
||||
// and the next tick retries the same occurrence. Decide's missed rule bounds
|
||||
// the retries: once the occurrence is too late it is skipped as missed.
|
||||
func retryLater(ctx context.Context, deps Deps, p models.PatchPolicy, cause error, due, next, now time.Time) {
|
||||
if _, err := db.Col("patch_policies").UpdateOne(ctx,
|
||||
bson.M{"policy_id": p.PolicyID, "next_run_at": next},
|
||||
bson.M{"$set": bson.M{"next_run_at": due}}); err != nil {
|
||||
log.Printf("patchsched: policy %s: put back claim: %v", p.PolicyID, err)
|
||||
}
|
||||
recordSkip(ctx, deps, p, "error: "+cause.Error(), due, now)
|
||||
}
|
||||
|
||||
func recordSkip(ctx context.Context, deps Deps, p models.PatchPolicy, reason string, due, at time.Time) {
|
||||
_, _ = db.Col("patch_policies").UpdateOne(ctx, bson.M{"policy_id": p.PolicyID},
|
||||
bson.M{"$set": bson.M{"last_skipped": models.Skip{Reason: reason, Due: due, At: at}}})
|
||||
|
||||
@@ -23,6 +23,11 @@ func StoreInventory(serverID string, r *pb.InventoryReport) error {
|
||||
set["inventory.memory.used_bytes"] = r.Memory.UsedBytes
|
||||
}
|
||||
set["inventory.swap_used_bytes"] = r.SwapUsed
|
||||
// Kept current so a patch reboot can be proven by a changed boot time,
|
||||
// independent of clock skew between the host and the control plane.
|
||||
if r.BootTimeUnix > 0 {
|
||||
set["inventory.boot_time"] = time.Unix(r.BootTimeUnix, 0).UTC()
|
||||
}
|
||||
|
||||
if r.IncludeStatic {
|
||||
set["inventory.static_at"] = now
|
||||
|
||||
@@ -51,6 +51,7 @@ var ScopedCollections = []string{
|
||||
"maintenance_windows",
|
||||
"patch_policies",
|
||||
"patch_runs",
|
||||
"patch_run_outputs",
|
||||
}
|
||||
|
||||
// collectionRenames maps the two collections whose names change. Ordered so the
|
||||
|
||||
@@ -29,6 +29,12 @@ func EnsurePatchIndexes() error {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := db.Col("patch_run_outputs").Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "run_id", Value: 1}, {Key: "server_id", Value: 1}},
|
||||
Options: options.Index().SetUnique(true),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := db.Col("patch_runs").Indexes().CreateMany(ctx, []mongo.IndexModel{
|
||||
{Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "run_id", Value: 1}}, Options: options.Index().SetUnique(true)},
|
||||
{Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "policy_id", Value: 1}, {Key: "started_at", Value: -1}}},
|
||||
|
||||
@@ -194,6 +194,37 @@ func CountPolicyTargets(p models.PatchPolicy) (int, error) {
|
||||
|
||||
var ErrPatchRunActive = errors.New("a run of this policy is already in progress")
|
||||
|
||||
// PoliciesForWindow lists every policy that uses a window, enabled or not.
|
||||
func PoliciesForWindow(instanceID, windowID string) ([]models.PatchPolicy, error) {
|
||||
ctx, cancel := patchCtx()
|
||||
defer cancel()
|
||||
cur, err := db.Col("patch_policies").Find(ctx, bson.M{"instance_id": instanceID, "window_id": windowID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []models.PatchPolicy{}
|
||||
return out, cur.All(ctx, &out)
|
||||
}
|
||||
|
||||
// CheckWindowScope refuses a tag-restricted token changing a window when any
|
||||
// policy using it targets servers outside its restriction: moving or
|
||||
// deleting the window moves or stops those servers' patching.
|
||||
func CheckWindowScope(instanceID, windowID string, tokenScope map[string]string) error {
|
||||
if len(tokenScope) == 0 {
|
||||
return nil
|
||||
}
|
||||
ps, err := PoliciesForWindow(instanceID, windowID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range ps {
|
||||
if err := CheckPolicyScope(instanceID, p, tokenScope); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckPolicyScope refuses a tag-restricted token acting on a policy whose
|
||||
// targets reach outside its restriction, with the workflow rule unchanged.
|
||||
func CheckPolicyScope(instanceID string, p models.PatchPolicy, tokenScope map[string]string) error {
|
||||
|
||||
@@ -27,6 +27,61 @@ var (
|
||||
|
||||
const patchRunsCol = "patch_runs"
|
||||
|
||||
// patchRunOutputsCol holds each server's output tail, one document per
|
||||
// (run_id, server_id). Kept out of the run document because a large run with
|
||||
// up to 64KB of output per server would pass MongoDB's 16MB document limit.
|
||||
const patchRunOutputsCol = "patch_run_outputs"
|
||||
|
||||
// storePatchOutput upserts one server's output tail for a run.
|
||||
func storePatchOutput(ctx context.Context, instanceID, runID, serverID, output string) error {
|
||||
_, err := db.Col(patchRunOutputsCol).UpdateOne(ctx,
|
||||
bson.M{"instance_id": instanceID, "run_id": runID, "server_id": serverID},
|
||||
bson.M{"$set": bson.M{"output": output, "updated_at": time.Now()}},
|
||||
options.UpdateOne().SetUpsert(true))
|
||||
return err
|
||||
}
|
||||
|
||||
// fillPatchOutputs puts each server's stored output back on the run, so the
|
||||
// API response carries it as before. A run written before outputs moved out
|
||||
// keeps the output it has inline.
|
||||
func fillPatchOutputs(ctx context.Context, run *models.PatchRun) error {
|
||||
cur, err := db.Col(patchRunOutputsCol).Find(ctx, bson.M{"instance_id": run.InstanceID, "run_id": run.RunID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var outs []struct {
|
||||
ServerID string `bson:"server_id"`
|
||||
Output string `bson:"output"`
|
||||
}
|
||||
if err := cur.All(ctx, &outs); err != nil {
|
||||
return err
|
||||
}
|
||||
byServer := make(map[string]string, len(outs))
|
||||
for _, o := range outs {
|
||||
byServer[o.ServerID] = o.Output
|
||||
}
|
||||
for i := range run.Servers {
|
||||
if out, ok := byServer[run.Servers[i].ServerID]; ok {
|
||||
run.Servers[i].Output = out
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// serverBootTime is the boot time the server last reported, or nil.
|
||||
func serverBootTime(ctx context.Context, instanceID, serverID string) *time.Time {
|
||||
var doc struct {
|
||||
Inventory struct {
|
||||
BootTime *time.Time `bson:"boot_time"`
|
||||
} `bson:"inventory"`
|
||||
}
|
||||
if err := db.Col("servers").FindOne(ctx, bson.M{"instance_id": instanceID, "server_id": serverID},
|
||||
options.FindOne().SetProjection(bson.M{"inventory.boot_time": 1})).Decode(&doc); err != nil {
|
||||
return nil
|
||||
}
|
||||
return doc.Inventory.BootTime
|
||||
}
|
||||
|
||||
func newServerRun(s models.Server, now time.Time) models.PatchServerRun {
|
||||
r := models.PatchServerRun{ServerID: s.ServerID, Hostname: s.Hostname, Status: models.PatchSrvQueued, PendingBefore: len(s.AvailableUpdates)}
|
||||
if !patchrun.AgentSupportsPatchResults(s.AgentVersion) {
|
||||
@@ -185,9 +240,11 @@ func claimServerForDispatch(ctx context.Context, runID, serverID, from, commandI
|
||||
return res.MatchedCount > 0, nil
|
||||
}
|
||||
|
||||
// loadRun reads a run for the tick and result paths, which never need output,
|
||||
// so any inline output left on an old document is not read either.
|
||||
func loadRun(ctx context.Context, filter bson.M) (*models.PatchRun, error) {
|
||||
var run models.PatchRun
|
||||
err := db.Col(patchRunsCol).FindOne(ctx, filter).Decode(&run)
|
||||
err := db.Col(patchRunsCol).FindOne(ctx, filter, options.FindOne().SetProjection(bson.M{"servers.output": 0})).Decode(&run)
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, ErrPatchRunNotFound
|
||||
}
|
||||
@@ -299,11 +356,25 @@ func finalizeRun(ctx context.Context, runID string) {
|
||||
}
|
||||
LogEvent(run.InstanceID, "patch.run_finished", run.TriggeredBy, "", "",
|
||||
fmt.Sprintf("patch run %s (%s) %s: %s", runID, name, status, patchrun.Summary(*run)))
|
||||
if status == models.PatchRunPartial || status == models.PatchRunFailed {
|
||||
if status == models.PatchRunPartial || status == models.PatchRunFailed ||
|
||||
(status == models.PatchRunCancelled && hasBadOutcome(*run)) {
|
||||
notifyPatchRun(*run)
|
||||
}
|
||||
}
|
||||
|
||||
// hasBadOutcome reports whether any server failed or was not reached. A
|
||||
// cancelled run with such a result still alerts: cancelling does not make a
|
||||
// failure somebody else's problem.
|
||||
func hasBadOutcome(run models.PatchRun) bool {
|
||||
for _, s := range run.Servers {
|
||||
switch s.Status {
|
||||
case models.PatchSrvFailed, models.PatchSrvMissedOffline, models.PatchSrvWindowClosed:
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func notifyPatchRun(run models.PatchRun) {
|
||||
if run.PolicyID == "" {
|
||||
return // a manual run was watched by the person who clicked
|
||||
@@ -351,18 +422,36 @@ func RecordPatchResult(instanceID, serverID string, r *pb.PatchResult) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
set := bson.M{"status": updated.Status, "output": updated.Output, "error": updated.Error}
|
||||
// The output goes to patch_run_outputs, not the run document.
|
||||
set := bson.M{"status": updated.Status, "error": updated.Error}
|
||||
if updated.PendingAfter != nil {
|
||||
set["pending_after"] = *updated.PendingAfter
|
||||
}
|
||||
if updated.RebootedAt != nil {
|
||||
set["rebooted_at"] = *updated.RebootedAt
|
||||
}
|
||||
if updated.Status == models.PatchSrvRebooting {
|
||||
// The boot time before the reboot: a later report with a
|
||||
// different one proves the restart.
|
||||
if bt := serverBootTime(ctx, instanceID, serverID); bt != nil {
|
||||
set["boot_time_before"] = *bt
|
||||
}
|
||||
}
|
||||
if updated.FinishedAt != nil {
|
||||
set["finished_at"] = *updated.FinishedAt
|
||||
}
|
||||
if ok, _ := setServerForCommand(ctx, run.RunID, serverID, r.CommandId, models.PatchSrvPatching, set); ok && updated.Status == models.PatchSrvRebooting {
|
||||
LogEvent(instanceID, "patch.reboot", "schedule", serverID, "",
|
||||
ok, err := setServerForCommand(ctx, run.RunID, serverID, r.CommandId, models.PatchSrvPatching, set)
|
||||
if err != nil {
|
||||
log.Printf("patch run %s: server %s: record result: %v", run.RunID, serverID, err)
|
||||
}
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := storePatchOutput(ctx, instanceID, run.RunID, serverID, updated.Output); err != nil {
|
||||
log.Printf("patch run %s: server %s: store output: %v", run.RunID, serverID, err)
|
||||
}
|
||||
if updated.Status == models.PatchSrvRebooting {
|
||||
LogEvent(instanceID, "patch.reboot", run.TriggeredBy, serverID, "",
|
||||
fmt.Sprintf("%s rebooting for patch policy %s", s.Hostname, run.PolicyName))
|
||||
}
|
||||
}
|
||||
@@ -427,7 +516,18 @@ func CancelPatchRun(instanceID, runID string) error {
|
||||
func GetPatchRun(instanceID, runID string) (*models.PatchRun, error) {
|
||||
ctx, cancel := patchCtx()
|
||||
defer cancel()
|
||||
return loadRun(ctx, bson.M{"instance_id": instanceID, "run_id": runID})
|
||||
var run models.PatchRun
|
||||
err := db.Col(patchRunsCol).FindOne(ctx, bson.M{"instance_id": instanceID, "run_id": runID}).Decode(&run)
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, ErrPatchRunNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := fillPatchOutputs(ctx, &run); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &run, nil
|
||||
}
|
||||
|
||||
// ListPatchRuns omits output: a list of fifty runs would otherwise carry up
|
||||
@@ -516,6 +616,8 @@ func sweepPatchRuns() {
|
||||
if days <= 0 || !r.FinishedAt.Before(now.AddDate(0, 0, -days)) {
|
||||
continue
|
||||
}
|
||||
_, _ = db.Col(patchRunsCol).DeleteOne(ctx, bson.M{"run_id": r.RunID})
|
||||
if _, err := db.Col(patchRunsCol).DeleteOne(ctx, bson.M{"run_id": r.RunID}); err == nil {
|
||||
_, _ = db.Col(patchRunOutputsCol).DeleteMany(ctx, bson.M{"instance_id": r.InstanceID, "run_id": r.RunID})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import "testing"
|
||||
// A tenant-scoped collection missing from ScopedCollections outlives its
|
||||
// instance when the instance is purged.
|
||||
func TestPatchCollectionsAreScoped(t *testing.T) {
|
||||
for _, name := range []string{"maintenance_windows", "patch_policies", "patch_runs"} {
|
||||
for _, name := range []string{"maintenance_windows", "patch_policies", "patch_runs", "patch_run_outputs"} {
|
||||
found := false
|
||||
for _, got := range ScopedCollections {
|
||||
if got == name {
|
||||
|
||||
Reference in New Issue
Block a user