fix(patching): final review fixes

- no dispatch in the last 15 minutes of a window; no-result timeout from dispatch time
- per-server output moves to patch_run_outputs (16MB document limit)
- reboot proven by a changed boot time; RebootTimeout 45m, ResultGrace 20m
- window update and delete are server-scoped against the policies using them
- scheduler puts the claim back on an error after it, so the next tick retries
- cancelled runs with failures alert; MCP apply_updates audits per server
- apply-updates 503 body documented; openapi regenerated
- web: cleared numeric fields no longer save as 0; Run now asks for confirmation

server/internal/services/inventory.go

@@ -23,6 +23,11 @@ func StoreInventory(serverID string, r *pb.InventoryReport) error {
 		set["inventory.memory.used_bytes"] = r.Memory.UsedBytes
 	}
 	set["inventory.swap_used_bytes"] = r.SwapUsed
+	// Kept current so a patch reboot can be proven by a changed boot time,
+	// independent of clock skew between the host and the control plane.
+	if r.BootTimeUnix > 0 {
+		set["inventory.boot_time"] = time.Unix(r.BootTimeUnix, 0).UTC()
+	}
 
 	if r.IncludeStatic {
 		set["inventory.static_at"] = now

server/internal/services/migrate_instance.go

@@ -51,6 +51,7 @@ var ScopedCollections = []string{
 	"maintenance_windows",
 	"patch_policies",
 	"patch_runs",
+	"patch_run_outputs",
 }
 
 // collectionRenames maps the two collections whose names change. Ordered so the

server/internal/services/patch_indexes.go

@@ -29,6 +29,12 @@ func EnsurePatchIndexes() error {
 	}); err != nil {
 		return err
 	}
+	if _, err := db.Col("patch_run_outputs").Indexes().CreateOne(ctx, mongo.IndexModel{
+		Keys:    bson.D{{Key: "instance_id", Value: 1}, {Key: "run_id", Value: 1}, {Key: "server_id", Value: 1}},
+		Options: options.Index().SetUnique(true),
+	}); err != nil {
+		return err
+	}
 	_, err := db.Col("patch_runs").Indexes().CreateMany(ctx, []mongo.IndexModel{
 		{Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "run_id", Value: 1}}, Options: options.Index().SetUnique(true)},
 		{Keys: bson.D{{Key: "instance_id", Value: 1}, {Key: "policy_id", Value: 1}, {Key: "started_at", Value: -1}}},

server/internal/services/patch_policies.go

@@ -194,6 +194,37 @@ func CountPolicyTargets(p models.PatchPolicy) (int, error) {
 
 var ErrPatchRunActive = errors.New("a run of this policy is already in progress")
 
+// PoliciesForWindow lists every policy that uses a window, enabled or not.
+func PoliciesForWindow(instanceID, windowID string) ([]models.PatchPolicy, error) {
+	ctx, cancel := patchCtx()
+	defer cancel()
+	cur, err := db.Col("patch_policies").Find(ctx, bson.M{"instance_id": instanceID, "window_id": windowID})
+	if err != nil {
+		return nil, err
+	}
+	out := []models.PatchPolicy{}
+	return out, cur.All(ctx, &out)
+}
+
+// CheckWindowScope refuses a tag-restricted token changing a window when any
+// policy using it targets servers outside its restriction: moving or
+// deleting the window moves or stops those servers' patching.
+func CheckWindowScope(instanceID, windowID string, tokenScope map[string]string) error {
+	if len(tokenScope) == 0 {
+		return nil
+	}
+	ps, err := PoliciesForWindow(instanceID, windowID)
+	if err != nil {
+		return err
+	}
+	for _, p := range ps {
+		if err := CheckPolicyScope(instanceID, p, tokenScope); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // CheckPolicyScope refuses a tag-restricted token acting on a policy whose
 // targets reach outside its restriction, with the workflow rule unchanged.
 func CheckPolicyScope(instanceID string, p models.PatchPolicy, tokenScope map[string]string) error {

server/internal/services/patch_runs.go

@@ -27,6 +27,61 @@ var (
 
 const patchRunsCol = "patch_runs"
 
+// patchRunOutputsCol holds each server's output tail, one document per
+// (run_id, server_id). Kept out of the run document because a large run with
+// up to 64KB of output per server would pass MongoDB's 16MB document limit.
+const patchRunOutputsCol = "patch_run_outputs"
+
+// storePatchOutput upserts one server's output tail for a run.
+func storePatchOutput(ctx context.Context, instanceID, runID, serverID, output string) error {
+	_, err := db.Col(patchRunOutputsCol).UpdateOne(ctx,
+		bson.M{"instance_id": instanceID, "run_id": runID, "server_id": serverID},
+		bson.M{"$set": bson.M{"output": output, "updated_at": time.Now()}},
+		options.UpdateOne().SetUpsert(true))
+	return err
+}
+
+// fillPatchOutputs puts each server's stored output back on the run, so the
+// API response carries it as before. A run written before outputs moved out
+// keeps the output it has inline.
+func fillPatchOutputs(ctx context.Context, run *models.PatchRun) error {
+	cur, err := db.Col(patchRunOutputsCol).Find(ctx, bson.M{"instance_id": run.InstanceID, "run_id": run.RunID})
+	if err != nil {
+		return err
+	}
+	var outs []struct {
+		ServerID string `bson:"server_id"`
+		Output   string `bson:"output"`
+	}
+	if err := cur.All(ctx, &outs); err != nil {
+		return err
+	}
+	byServer := make(map[string]string, len(outs))
+	for _, o := range outs {
+		byServer[o.ServerID] = o.Output
+	}
+	for i := range run.Servers {
+		if out, ok := byServer[run.Servers[i].ServerID]; ok {
+			run.Servers[i].Output = out
+		}
+	}
+	return nil
+}
+
+// serverBootTime is the boot time the server last reported, or nil.
+func serverBootTime(ctx context.Context, instanceID, serverID string) *time.Time {
+	var doc struct {
+		Inventory struct {
+			BootTime *time.Time `bson:"boot_time"`
+		} `bson:"inventory"`
+	}
+	if err := db.Col("servers").FindOne(ctx, bson.M{"instance_id": instanceID, "server_id": serverID},
+		options.FindOne().SetProjection(bson.M{"inventory.boot_time": 1})).Decode(&doc); err != nil {
+		return nil
+	}
+	return doc.Inventory.BootTime
+}
+
 func newServerRun(s models.Server, now time.Time) models.PatchServerRun {
 	r := models.PatchServerRun{ServerID: s.ServerID, Hostname: s.Hostname, Status: models.PatchSrvQueued, PendingBefore: len(s.AvailableUpdates)}
 	if !patchrun.AgentSupportsPatchResults(s.AgentVersion) {
@@ -185,9 +240,11 @@ func claimServerForDispatch(ctx context.Context, runID, serverID, from, commandI
 	return res.MatchedCount > 0, nil
 }
 
+// loadRun reads a run for the tick and result paths, which never need output,
+// so any inline output left on an old document is not read either.
 func loadRun(ctx context.Context, filter bson.M) (*models.PatchRun, error) {
 	var run models.PatchRun
-	err := db.Col(patchRunsCol).FindOne(ctx, filter).Decode(&run)
+	err := db.Col(patchRunsCol).FindOne(ctx, filter, options.FindOne().SetProjection(bson.M{"servers.output": 0})).Decode(&run)
 	if errors.Is(err, mongo.ErrNoDocuments) {
 		return nil, ErrPatchRunNotFound
 	}
@@ -299,11 +356,25 @@ func finalizeRun(ctx context.Context, runID string) {
 	}
 	LogEvent(run.InstanceID, "patch.run_finished", run.TriggeredBy, "", "",
 		fmt.Sprintf("patch run %s (%s) %s: %s", runID, name, status, patchrun.Summary(*run)))
-	if status == models.PatchRunPartial || status == models.PatchRunFailed {
+	if status == models.PatchRunPartial || status == models.PatchRunFailed ||
+		(status == models.PatchRunCancelled && hasBadOutcome(*run)) {
 		notifyPatchRun(*run)
 	}
 }
 
+// hasBadOutcome reports whether any server failed or was not reached. A
+// cancelled run with such a result still alerts: cancelling does not make a
+// failure somebody else's problem.
+func hasBadOutcome(run models.PatchRun) bool {
+	for _, s := range run.Servers {
+		switch s.Status {
+		case models.PatchSrvFailed, models.PatchSrvMissedOffline, models.PatchSrvWindowClosed:
+			return true
+		}
+	}
+	return false
+}
+
 func notifyPatchRun(run models.PatchRun) {
 	if run.PolicyID == "" {
 		return // a manual run was watched by the person who clicked
@@ -351,18 +422,36 @@ func RecordPatchResult(instanceID, serverID string, r *pb.PatchResult) {
 		if !ok {
 			return
 		}
-		set := bson.M{"status": updated.Status, "output": updated.Output, "error": updated.Error}
+		// The output goes to patch_run_outputs, not the run document.
+		set := bson.M{"status": updated.Status, "error": updated.Error}
 		if updated.PendingAfter != nil {
 			set["pending_after"] = *updated.PendingAfter
 		}
 		if updated.RebootedAt != nil {
 			set["rebooted_at"] = *updated.RebootedAt
 		}
+		if updated.Status == models.PatchSrvRebooting {
+			// The boot time before the reboot: a later report with a
+			// different one proves the restart.
+			if bt := serverBootTime(ctx, instanceID, serverID); bt != nil {
+				set["boot_time_before"] = *bt
+			}
+		}
 		if updated.FinishedAt != nil {
 			set["finished_at"] = *updated.FinishedAt
 		}
-		if ok, _ := setServerForCommand(ctx, run.RunID, serverID, r.CommandId, models.PatchSrvPatching, set); ok && updated.Status == models.PatchSrvRebooting {
-			LogEvent(instanceID, "patch.reboot", "schedule", serverID, "",
+		ok, err := setServerForCommand(ctx, run.RunID, serverID, r.CommandId, models.PatchSrvPatching, set)
+		if err != nil {
+			log.Printf("patch run %s: server %s: record result: %v", run.RunID, serverID, err)
+		}
+		if !ok {
+			return
+		}
+		if err := storePatchOutput(ctx, instanceID, run.RunID, serverID, updated.Output); err != nil {
+			log.Printf("patch run %s: server %s: store output: %v", run.RunID, serverID, err)
+		}
+		if updated.Status == models.PatchSrvRebooting {
+			LogEvent(instanceID, "patch.reboot", run.TriggeredBy, serverID, "",
 				fmt.Sprintf("%s rebooting for patch policy %s", s.Hostname, run.PolicyName))
 		}
 	}
@@ -427,7 +516,18 @@ func CancelPatchRun(instanceID, runID string) error {
 func GetPatchRun(instanceID, runID string) (*models.PatchRun, error) {
 	ctx, cancel := patchCtx()
 	defer cancel()
-	return loadRun(ctx, bson.M{"instance_id": instanceID, "run_id": runID})
+	var run models.PatchRun
+	err := db.Col(patchRunsCol).FindOne(ctx, bson.M{"instance_id": instanceID, "run_id": runID}).Decode(&run)
+	if errors.Is(err, mongo.ErrNoDocuments) {
+		return nil, ErrPatchRunNotFound
+	}
+	if err != nil {
+		return nil, err
+	}
+	if err := fillPatchOutputs(ctx, &run); err != nil {
+		return nil, err
+	}
+	return &run, nil
 }
 
 // ListPatchRuns omits output: a list of fifty runs would otherwise carry up
@@ -516,6 +616,8 @@ func sweepPatchRuns() {
 		if days <= 0 || !r.FinishedAt.Before(now.AddDate(0, 0, -days)) {
 			continue
 		}
-		_, _ = db.Col(patchRunsCol).DeleteOne(ctx, bson.M{"run_id": r.RunID})
+		if _, err := db.Col(patchRunsCol).DeleteOne(ctx, bson.M{"run_id": r.RunID}); err == nil {
+			_, _ = db.Col(patchRunOutputsCol).DeleteMany(ctx, bson.M{"instance_id": r.InstanceID, "run_id": r.RunID})
+		}
 	}
 }

server/internal/services/patch_scoped_test.go

@@ -5,7 +5,7 @@ import "testing"
 // A tenant-scoped collection missing from ScopedCollections outlives its
 // instance when the instance is purged.
 func TestPatchCollectionsAreScoped(t *testing.T) {
-	for _, name := range []string{"maintenance_windows", "patch_policies", "patch_runs"} {
+	for _, name := range []string{"maintenance_windows", "patch_policies", "patch_runs", "patch_run_outputs"} {
 		found := false
 		for _, got := range ScopedCollections {
 			if got == name {