Compare commits
8
Commits
b777ffcf58
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f552006318 | ||
|
|
b5b9775d2b | ||
|
|
ecd703d502 | ||
|
|
db75d7dcbd | ||
|
|
a4745834de | ||
|
|
51b58fef21 | ||
|
|
5f5e19e23e | ||
|
|
3aa24bb938 |
@@ -93,8 +93,13 @@ too.
|
||||
|
||||
## What the agent will not do
|
||||
|
||||
- **It never reboots a host.** `ApplyUpdatesCmd` installs and stops there;
|
||||
`inventory.reboot_required` reports that one is owed.
|
||||
- **It reboots a host only when told to and only when owed.** An
|
||||
`ApplyUpdatesCmd` with `reboot_if_required` set, on a host whose OS reports a
|
||||
reboot is owed, with at least 5 minutes left before `deadline_unix`, reboots
|
||||
after a one-minute grace period (`shutdown -r +1`, `shutdown /r /t 60`), and
|
||||
only once the `PatchResult` announcing it has been sent. Anything else
|
||||
installs and stops there, and `inventory.reboot_required` reports what is
|
||||
owed.
|
||||
- **It decides what it will not touch.** The protected workload set is computed
|
||||
and enforced agent-side - `vantage-agent.service`, `VantageAgent` on Windows,
|
||||
and its own container ID from `/proc/self/cgroup`. The control plane may name
|
||||
@@ -124,6 +129,34 @@ need a PowerShell Gallery install on every host and fails on an air-gapped
|
||||
fleet. `CurrentVersion` is empty on Windows and `NewVersion` carries the KB
|
||||
article ID: a Windows update is not a version bump of a named package.
|
||||
|
||||
## Patching
|
||||
|
||||
`updates.Apply` takes a scope and a deadline and answers with the tail of the
|
||||
package manager's output. Security-only uses `--security` on dnf/yum,
|
||||
`zypper patch --category security`, the Security and Critical classifications
|
||||
on Windows, and for apt a temporary `SourceParts` directory holding only the
|
||||
`-security` suites (with `APT::Get::List-Cleanup=0`, or the reduced update
|
||||
deletes every other list file). apk and pacman have no security metadata and
|
||||
report `unsupported`; security-only never falls back to installing
|
||||
everything. One run at a time: a second command answers `busy`.
|
||||
|
||||
The deadline (`deadline_unix`, the window end) only gates the **start** of each
|
||||
phase: the apt index refresh, the upgrade command, the Windows install script.
|
||||
A phase that has not started by then is refused with "the maintenance window
|
||||
ended before <phase> could start" (`canStart` in `internal/updates/phase.go`).
|
||||
A started upgrade is never killed by the window: it runs under a 2 hour
|
||||
backstop from its own start (`defaultApplyCap`), which on Linux sends SIGTERM
|
||||
and waits 5 minutes before a kill. Interrupting a package manager mid-transaction
|
||||
is worse than letting it finish late.
|
||||
|
||||
A `PatchResult` whose send fails (the command stream reconnected while the
|
||||
patch ran) is kept in a bounded queue (32, oldest dropped) and flushed on the
|
||||
next stream right after `AgentReady`. A queued result that announced a reboot
|
||||
has the reboot decision taken again at flush time, and the host still reboots
|
||||
only once the result is delivered. The startup static inventory report is
|
||||
retried every 30 seconds, up to 10 attempts: it carries the boot time the
|
||||
control plane uses to prove a patch reboot.
|
||||
|
||||
## Two constants that mirror the control plane
|
||||
|
||||
Neither can be shared - this is a separate module and the control plane's are
|
||||
|
||||
@@ -9,7 +9,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.1
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.6.0
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260908043556-f8649ddbbfe6 // indirect
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.1 h1:rPzXSRwU+4+F2pdkmDrIxKsIzqz3S6feJEWalGmKqfU=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.2.1/go.mod h1:dWjeOFLltQ8sv9Pnn1xRxGfWGgqa2fkG0esuaJLoPXQ=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.6.0 h1:EtojZ1d3cN9foHpc/CAI3KzBewYGn4sKWdkWs2MV78Q=
|
||||
gitea.hostxtra.co.uk/vantage/vantage-shared v0.6.0/go.mod h1:Zo66XhqF8No3dveIowLCepvMxVg8KnhsNMz0k0Xpuck=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// parseBtime reads the boot time, in Unix seconds, from /proc/stat. The
|
||||
// control plane compares it with the moment it asked for a reboot: a report
|
||||
// is only proof the host restarted if its boot time is later.
|
||||
func parseBtime(stat string) int64 {
|
||||
for _, line := range strings.Split(stat, "\n") {
|
||||
if rest, ok := strings.CutPrefix(line, "btime "); ok {
|
||||
v, _ := strconv.ParseInt(strings.TrimSpace(rest), 10, 64)
|
||||
return v
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package inventory
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseBtime(t *testing.T) {
|
||||
stat := "cpu 1 2 3 4\ncpu0 1 2 3 4\nintr 1\nctxt 99\nbtime 1757800000\nprocesses 5\n"
|
||||
if got := parseBtime(stat); got != 1757800000 {
|
||||
t.Fatalf("got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBtimeMissing(t *testing.T) {
|
||||
if got := parseBtime("cpu 1 2 3\n"); got != 0 {
|
||||
t.Fatalf("got %d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
)
|
||||
|
||||
func collect(r *pb.InventoryReport, includeStatic bool) {
|
||||
r.BootTimeUnix = parseBtime(readProc("/proc/stat"))
|
||||
r.CPU.UsagePct = cpuUsage()
|
||||
r.CPU.Load1 = load1()
|
||||
memTotal, memAvail, swapTotal, swapFree := meminfo()
|
||||
|
||||
@@ -20,6 +20,8 @@ var (
|
||||
)
|
||||
|
||||
func collect(r *pb.InventoryReport, includeStatic bool) {
|
||||
// DurationSinceBoot returns milliseconds since boot and does not wrap.
|
||||
r.BootTimeUnix = time.Now().Add(-windows.DurationSinceBoot()).Unix()
|
||||
r.CPU.UsagePct = cpuUsage()
|
||||
// Windows has no load average. Left at zero; the UI already treats it as
|
||||
// optional because it is omitempty on the wire.
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package agentsync
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gitea.hostxtra.co.uk/vantage/vantage-agent/internal/config"
|
||||
grpcclient "gitea.hostxtra.co.uk/vantage/vantage-agent/internal/grpc"
|
||||
"gitea.hostxtra.co.uk/vantage/vantage-agent/internal/updates"
|
||||
"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
|
||||
)
|
||||
|
||||
// minRebootLeeway is the least time that must remain in the window for the
|
||||
// agent to start a reboot. A reboot that lands after the window closes is the
|
||||
// outage the window existed to prevent.
|
||||
const minRebootLeeway = 5 * time.Minute
|
||||
|
||||
// shouldReboot is the whole reboot decision. The agent reboots a host only
|
||||
// when the command asked, the OS reports a reboot is owed, and the window
|
||||
// still has room for it. A command with no deadline is a manual run, which
|
||||
// never reboots.
|
||||
func shouldReboot(requested, owed bool, now, deadline time.Time) bool {
|
||||
if !requested || !owed || deadline.IsZero() {
|
||||
return false
|
||||
}
|
||||
return deadline.Sub(now) >= minRebootLeeway
|
||||
}
|
||||
|
||||
func handleApplyUpdates(send func(*pb.AgentMessage) error, cfg *config.Config, cmd *pb.ServerCommand) {
|
||||
c := cmd.ApplyUpdates
|
||||
var deadline time.Time
|
||||
if c.DeadlineUnix > 0 {
|
||||
deadline = time.Unix(c.DeadlineUnix, 0)
|
||||
}
|
||||
log.Printf("applying OS updates (cmd=%s scope=%q reboot=%v)", cmd.CommandId, c.Scope, c.RebootIfRequired)
|
||||
|
||||
res, err := updates.Apply(updates.ApplyOptions{SecurityOnly: c.Scope == pb.PatchScopeSecurity, Deadline: deadline})
|
||||
pr := &pb.PatchResult{CommandId: cmd.CommandId, OutputTail: res.Output, PendingAfter: -1}
|
||||
switch {
|
||||
case errors.Is(err, updates.ErrBusy):
|
||||
pr.Status, pr.Message = pb.PatchStatusBusy, err.Error()
|
||||
case err != nil:
|
||||
pr.Status, pr.Message = pb.PatchStatusFailed, err.Error()
|
||||
case res.Unsupported:
|
||||
pr.Status, pr.Message = pb.PatchStatusUnsupported, res.Reason
|
||||
default:
|
||||
pr.Status = pb.PatchStatusOK
|
||||
}
|
||||
|
||||
// Refresh the pending list whether the run succeeded or not, so the counts
|
||||
// the operator sees are this host's real state. A busy refusal changed
|
||||
// nothing, and the run in progress will report for itself.
|
||||
if pr.Status != pb.PatchStatusBusy {
|
||||
pr.PendingAfter = int32(reportPendingUpdates(cfg))
|
||||
}
|
||||
pr.RebootRequired = updates.RebootRequired()
|
||||
pr.Rebooting = pr.Status == pb.PatchStatusOK && shouldReboot(c.RebootIfRequired, pr.RebootRequired, time.Now(), deadline)
|
||||
|
||||
msg := &pb.AgentMessage{ServerId: cfg.ServerID, AgentToken: cfg.AgentToken, PatchResult: pr}
|
||||
if err := send(msg); err != nil {
|
||||
log.Printf("send patch result (cmd=%s): %v; keeping it for the next command stream", cmd.CommandId, err)
|
||||
// A reboot nobody was told about looks like a crash. Without a
|
||||
// delivered result, do not reboot: the queued result retakes the
|
||||
// decision when it is finally sent.
|
||||
if pendingResults.push(pendingResult{msg: msg, requested: c.RebootIfRequired, deadline: deadline}) {
|
||||
log.Printf("patch result queue full: dropped the oldest undelivered result")
|
||||
}
|
||||
return
|
||||
}
|
||||
log.Printf("patch result sent (cmd=%s status=%s pending_after=%d rebooting=%v)", cmd.CommandId, pr.Status, pr.PendingAfter, pr.Rebooting)
|
||||
if pr.Rebooting {
|
||||
if err := updates.ScheduleReboot(); err != nil {
|
||||
log.Printf("schedule reboot (cmd=%s): %v", cmd.CommandId, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pendingResults holds PatchResults whose send failed, typically because the
|
||||
// command stream reconnected while a patch ran. The server guards result
|
||||
// writes on status and command id, so a late result is safe to deliver.
|
||||
var pendingResults = &resultQueue{max: maxPendingResults}
|
||||
|
||||
// flushPendingResults delivers queued results on a newly established stream.
|
||||
// A queued result that announced a reboot has that decision taken again now:
|
||||
// time has passed, so the window may be too close to its end, or the reboot
|
||||
// may no longer be owed. Rebooting is cleared before sending when it no
|
||||
// longer holds, and the host reboots only once the result is delivered.
|
||||
func flushPendingResults(send func(*pb.AgentMessage) error) {
|
||||
pendingResults.flush(func(p pendingResult) error {
|
||||
pr := p.msg.PatchResult
|
||||
if pr.Rebooting {
|
||||
owed := updates.RebootRequired()
|
||||
pr.RebootRequired = owed
|
||||
if !shouldReboot(p.requested, owed, time.Now(), p.deadline) {
|
||||
pr.Rebooting = false
|
||||
}
|
||||
}
|
||||
if err := send(p.msg); err != nil {
|
||||
log.Printf("send queued patch result (cmd=%s): %v", pr.CommandId, err)
|
||||
return err
|
||||
}
|
||||
log.Printf("queued patch result sent (cmd=%s status=%s rebooting=%v)", pr.CommandId, pr.Status, pr.Rebooting)
|
||||
if pr.Rebooting {
|
||||
if err := updates.ScheduleReboot(); err != nil {
|
||||
log.Printf("schedule reboot (cmd=%s): %v", pr.CommandId, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// reportPendingUpdates re-checks pending updates and reports them, returning
|
||||
// the count, or -1 if either step failed.
|
||||
func reportPendingUpdates(cfg *config.Config) int {
|
||||
pkgs, err := updates.CheckAvailable()
|
||||
if err != nil {
|
||||
log.Printf("post-apply update check: %v", err)
|
||||
return -1
|
||||
}
|
||||
list := make([]pb.PackageUpdate, len(pkgs))
|
||||
for i, p := range pkgs {
|
||||
list[i] = pb.PackageUpdate{Name: p.Name, CurrentVersion: p.CurrentVersion, NewVersion: p.NewVersion, Phased: p.Phased}
|
||||
}
|
||||
client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
|
||||
if err != nil {
|
||||
log.Printf("post-apply report dial: %v", err)
|
||||
return updates.CountInstallable(pkgs)
|
||||
}
|
||||
defer client.Close()
|
||||
if err := client.ReportUpdates(cfg.ServerID, cfg.AgentToken, list); err != nil {
|
||||
log.Printf("post-apply ReportUpdates: %v", err)
|
||||
}
|
||||
return len(pkgs)
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package agentsync
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestShouldReboot(t *testing.T) {
|
||||
now := time.Date(2026, 9, 20, 2, 30, 0, 0, time.UTC)
|
||||
cases := []struct {
|
||||
name string
|
||||
requested, owed bool
|
||||
deadline time.Time
|
||||
want bool
|
||||
}{
|
||||
{"asked, owed, plenty of time", true, true, now.Add(time.Hour), true},
|
||||
{"not asked", false, true, now.Add(time.Hour), false},
|
||||
{"nothing owed", true, false, now.Add(time.Hour), false},
|
||||
{"exactly five minutes left", true, true, now.Add(5 * time.Minute), true},
|
||||
{"under five minutes left", true, true, now.Add(4*time.Minute + 59*time.Second), false},
|
||||
{"no deadline (manual run)", true, true, time.Time{}, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := shouldReboot(c.requested, c.owed, now, c.deadline); got != c.want {
|
||||
t.Errorf("%s: got %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package agentsync
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
|
||||
)
|
||||
|
||||
// maxPendingResults bounds the results kept for a later stream. A host that
|
||||
// cannot reach the control plane for a very long time drops its oldest
|
||||
// results rather than growing without limit.
|
||||
const maxPendingResults = 32
|
||||
|
||||
// pendingResult is a PatchResult that could not be sent, with what the reboot
|
||||
// decision needs to be taken again when it finally can be.
|
||||
type pendingResult struct {
|
||||
msg *pb.AgentMessage
|
||||
requested bool // the command's RebootIfRequired
|
||||
deadline time.Time // the command's deadline, zero for a manual run
|
||||
}
|
||||
|
||||
// resultQueue holds undelivered PatchResults until the next command stream.
|
||||
type resultQueue struct {
|
||||
mu sync.Mutex
|
||||
items []pendingResult
|
||||
max int
|
||||
}
|
||||
|
||||
// push appends p, dropping the oldest entry when the queue is full. It
|
||||
// reports whether an entry was dropped.
|
||||
func (q *resultQueue) push(p pendingResult) bool {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
q.items = append(q.items, p)
|
||||
if len(q.items) > q.max {
|
||||
q.items = q.items[len(q.items)-q.max:]
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// flush sends the queued results oldest first, removing each once sent. It
|
||||
// stops at the first failure and keeps that entry and everything after it
|
||||
// for the next stream. The lock is held throughout so two streams never
|
||||
// deliver the same result.
|
||||
func (q *resultQueue) flush(send func(pendingResult) error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
for len(q.items) > 0 {
|
||||
if err := send(q.items[0]); err != nil {
|
||||
return
|
||||
}
|
||||
q.items = q.items[1:]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package agentsync
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
|
||||
)
|
||||
|
||||
func pending(id string) pendingResult {
|
||||
return pendingResult{msg: &pb.AgentMessage{PatchResult: &pb.PatchResult{CommandId: id}}}
|
||||
}
|
||||
|
||||
func ids(q *resultQueue) []string {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
var out []string
|
||||
for _, p := range q.items {
|
||||
out = append(out, p.msg.PatchResult.CommandId)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestResultQueueEnqueue(t *testing.T) {
|
||||
q := &resultQueue{max: 3}
|
||||
q.push(pending("a"))
|
||||
q.push(pending("b"))
|
||||
if got := ids(q); len(got) != 2 || got[0] != "a" || got[1] != "b" {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultQueueBoundDropsOldest(t *testing.T) {
|
||||
q := &resultQueue{max: 2}
|
||||
q.push(pending("a"))
|
||||
q.push(pending("b"))
|
||||
if dropped := q.push(pending("c")); !dropped {
|
||||
t.Fatal("push over the bound must report a drop")
|
||||
}
|
||||
if got := ids(q); len(got) != 2 || got[0] != "b" || got[1] != "c" {
|
||||
t.Fatalf("got %v, want [b c]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultQueueDefaultBound(t *testing.T) {
|
||||
if maxPendingResults != 32 {
|
||||
t.Fatalf("maxPendingResults = %d, want 32", maxPendingResults)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultQueueFlushOrderAndRemoval(t *testing.T) {
|
||||
q := &resultQueue{max: 8}
|
||||
for _, id := range []string{"a", "b", "c"} {
|
||||
q.push(pending(id))
|
||||
}
|
||||
var sent []string
|
||||
q.flush(func(p pendingResult) error {
|
||||
sent = append(sent, p.msg.PatchResult.CommandId)
|
||||
return nil
|
||||
})
|
||||
if len(sent) != 3 || sent[0] != "a" || sent[1] != "b" || sent[2] != "c" {
|
||||
t.Fatalf("flush order %v, want [a b c]", sent)
|
||||
}
|
||||
if got := ids(q); len(got) != 0 {
|
||||
t.Fatalf("sent entries must be removed, left %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultQueueFlushRetainsOnFailure(t *testing.T) {
|
||||
q := &resultQueue{max: 8}
|
||||
for _, id := range []string{"a", "b", "c"} {
|
||||
q.push(pending(id))
|
||||
}
|
||||
var sent []string
|
||||
q.flush(func(p pendingResult) error {
|
||||
if p.msg.PatchResult.CommandId == "b" {
|
||||
return errors.New("stream gone")
|
||||
}
|
||||
sent = append(sent, p.msg.PatchResult.CommandId)
|
||||
return nil
|
||||
})
|
||||
if len(sent) != 1 || sent[0] != "a" {
|
||||
t.Fatalf("sent %v, want [a]", sent)
|
||||
}
|
||||
if got := ids(q); len(got) != 2 || got[0] != "b" || got[1] != "c" {
|
||||
t.Fatalf("retained %v, want [b c]", got)
|
||||
}
|
||||
}
|
||||
+23
-19
@@ -243,6 +243,11 @@ func connectAndHandleStream(ctx context.Context, cfg *config.Config) error {
|
||||
return stream.Send(msg)
|
||||
}
|
||||
|
||||
// Results that could not be sent on an earlier stream go out first. In
|
||||
// its own goroutine: a queued reboot re-check can take a while on
|
||||
// Windows, and the receive loop below must start promptly.
|
||||
go flushPendingResults(send)
|
||||
|
||||
// Stream liveness, tracked here rather than left to gRPC keepalive.
|
||||
//
|
||||
// Keepalive operates on the transport, and behind an L7 proxy the transport
|
||||
@@ -341,7 +346,7 @@ func connectAndHandleStream(ctx context.Context, cfg *config.Config) error {
|
||||
go handleUpdateAgent(cmd)
|
||||
}
|
||||
if cmd.ApplyUpdates != nil {
|
||||
go handleApplyUpdates(cfg, cmd)
|
||||
go handleApplyUpdates(send, cfg, cmd)
|
||||
}
|
||||
if cmd.CleanupWorkspace != nil {
|
||||
go handleCleanupWorkspace(cmd)
|
||||
@@ -401,6 +406,7 @@ func runUpdateCheck(ctx context.Context, cfg *config.Config) {
|
||||
Name: p.Name,
|
||||
CurrentVersion: p.CurrentVersion,
|
||||
NewVersion: p.NewVersion,
|
||||
Phased: p.Phased,
|
||||
}
|
||||
}
|
||||
client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
|
||||
@@ -446,7 +452,7 @@ func runInventory(ctx context.Context, cfg *config.Config) {
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
report := func(static bool) {
|
||||
report := func(static bool) error {
|
||||
r := inventory.Collect(static)
|
||||
r.ServerId = cfg.ServerID
|
||||
r.AgentToken = cfg.AgentToken
|
||||
@@ -462,10 +468,18 @@ func runInventory(ctx context.Context, cfg *config.Config) {
|
||||
}
|
||||
if err := client.ReportInventory(r); err != nil {
|
||||
log.Printf("report inventory: %v", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
report(true)
|
||||
// The startup static report carries the boot time the server uses to
|
||||
// prove a patch reboot happened, so a failure is retried on the next
|
||||
// ticks (every 30 seconds, up to startupStaticAttempts in total) instead
|
||||
// of waiting a quarter of an hour for the next static snapshot.
|
||||
const startupStaticAttempts = 10
|
||||
attempts := 1
|
||||
startupPending := report(true) != nil
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
tick := 0
|
||||
@@ -475,27 +489,17 @@ func runInventory(ctx context.Context, cfg *config.Config) {
|
||||
return
|
||||
case <-ticker.C:
|
||||
tick++
|
||||
if startupPending && attempts < startupStaticAttempts {
|
||||
attempts++
|
||||
startupPending = report(true) != nil
|
||||
continue
|
||||
}
|
||||
startupPending = false
|
||||
report(tick%30 == 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleApplyUpdates(cfg *config.Config, cmd *pb.ServerCommand) {
|
||||
log.Printf("applying OS updates (cmd=%s)…", cmd.CommandId)
|
||||
if err := updates.ApplyAll(); err != nil {
|
||||
log.Printf("OS upgrade failed (cmd=%s): %v", cmd.CommandId, err)
|
||||
return
|
||||
}
|
||||
log.Printf("OS updates applied successfully (cmd=%s)", cmd.CommandId)
|
||||
|
||||
client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
_ = client.ReportUpdates(cfg.ServerID, cfg.AgentToken, nil)
|
||||
}
|
||||
|
||||
func handleCleanupWorkspace(cmd *pb.ServerCommand) {
|
||||
id := cmd.CleanupWorkspace.WorkspaceId
|
||||
dir := agentexec.WorkspacePath(id)
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Two package managers running at once corrupt each other's locks. The second
|
||||
// caller must be told, not queued.
|
||||
func TestApplyRefusesWhileBusy(t *testing.T) {
|
||||
applyMu.Lock()
|
||||
defer applyMu.Unlock()
|
||||
_, err := Apply(ApplyOptions{Deadline: time.Now().Add(time.Minute)})
|
||||
if !errors.Is(err, ErrBusy) {
|
||||
t.Fatalf("err = %v, want ErrBusy", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Ubuntu phases some -updates releases: a host is selected at random for each
|
||||
// update, and until it is, apt lists the package as upgradable while an
|
||||
// upgrade defers it. Reporting those as pending made a freshly patched host
|
||||
// look unpatched, so they are flagged and left out of pending counts. The
|
||||
// -security pocket is never phased.
|
||||
|
||||
// aptPhasedDeferred asks apt which upgrades it would defer, by simulating an
|
||||
// upgrade. The simulation takes no lock and changes nothing. LC_ALL=C keeps
|
||||
// the heading parsePhasedDeferred looks for in English. Any failure reports no
|
||||
// phased packages: the update list is then exactly what it was before this
|
||||
// existed, never shorter.
|
||||
func aptPhasedDeferred() map[string]bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "apt-get", "-s", "-o", "Debug::NoLocking=1", "upgrade")
|
||||
cmd.Env = append(os.Environ(), "LC_ALL=C", "DEBIAN_FRONTEND=noninteractive")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return parsePhasedDeferred(string(out))
|
||||
}
|
||||
|
||||
// parsePhasedDeferred reads the package names apt lists under "The following
|
||||
// upgrades have been deferred due to phasing:". The names follow on indented
|
||||
// lines; the block ends at the first line that is not indented.
|
||||
func parsePhasedDeferred(out string) map[string]bool {
|
||||
phased := map[string]bool{}
|
||||
in := false
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.Contains(line, "deferred due to phasing") {
|
||||
in = true
|
||||
continue
|
||||
}
|
||||
if !in {
|
||||
continue
|
||||
}
|
||||
if line == "" || (line[0] != ' ' && line[0] != '\t') {
|
||||
in = false
|
||||
continue
|
||||
}
|
||||
for _, name := range strings.Fields(line) {
|
||||
phased[name] = true
|
||||
}
|
||||
}
|
||||
return phased
|
||||
}
|
||||
|
||||
// markPhased flags the updates apt would defer.
|
||||
func markPhased(ups []PackageUpdate, phased map[string]bool) {
|
||||
for i := range ups {
|
||||
if phased[ups[i].Name] {
|
||||
ups[i].Phased = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CountInstallable is the number of updates an upgrade would actually install
|
||||
// now: phased updates are pending but not yet installable.
|
||||
func CountInstallable(ups []PackageUpdate) int {
|
||||
n := 0
|
||||
for _, u := range ups {
|
||||
if !u.Phased {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const simulatedUpgrade = `Reading package lists...
|
||||
Building dependency tree...
|
||||
Reading state information...
|
||||
Calculating upgrade...
|
||||
The following upgrades have been deferred due to phasing:
|
||||
libgssapi-krb5-2 libk5crypto3 libkrb5-3 libkrb5support0 libnetplan1
|
||||
netplan-generator netplan.io python3-netplan
|
||||
The following packages will be upgraded:
|
||||
curl libcurl4
|
||||
2 upgraded, 0 newly installed, 0 to remove and 8 not upgraded.
|
||||
Inst curl [8.5.0-2ubuntu10.5] (8.5.0-2ubuntu10.6 Ubuntu:24.04/noble-updates [amd64])
|
||||
`
|
||||
|
||||
func TestParsePhasedDeferred(t *testing.T) {
|
||||
got := parsePhasedDeferred(simulatedUpgrade)
|
||||
want := map[string]bool{
|
||||
"libgssapi-krb5-2": true, "libk5crypto3": true, "libkrb5-3": true, "libkrb5support0": true,
|
||||
"libnetplan1": true, "netplan-generator": true, "netplan.io": true, "python3-netplan": true,
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v\nwant %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePhasedDeferredNone(t *testing.T) {
|
||||
out := "Calculating upgrade...\nThe following packages will be upgraded:\n curl\n1 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.\n"
|
||||
if got := parsePhasedDeferred(out); len(got) != 0 {
|
||||
t.Fatalf("got %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Packages kept back for other reasons are not phased and must not be flagged.
|
||||
func TestParsePhasedDeferredIgnoresKeptBack(t *testing.T) {
|
||||
out := "The following packages have been kept back:\n linux-generic\nThe following upgrades have been deferred due to phasing:\n netplan.io\n0 upgraded\n"
|
||||
got := parsePhasedDeferred(out)
|
||||
if !reflect.DeepEqual(got, map[string]bool{"netplan.io": true}) {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkPhased(t *testing.T) {
|
||||
ups := []PackageUpdate{{Name: "curl"}, {Name: "netplan.io"}}
|
||||
markPhased(ups, map[string]bool{"netplan.io": true})
|
||||
if ups[0].Phased || !ups[1].Phased {
|
||||
t.Fatalf("got %+v", ups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountInstallable(t *testing.T) {
|
||||
ups := []PackageUpdate{{Name: "a"}, {Name: "b", Phased: true}, {Name: "c"}}
|
||||
if got := CountInstallable(ups); got != 2 {
|
||||
t.Fatalf("got %d, want 2", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// isSecuritySuite reports whether an apt suite carries security fixes. Debian
|
||||
// 11+ and every supported Ubuntu name them "<codename>-security".
|
||||
func isSecuritySuite(s string) bool { return strings.HasSuffix(s, "-security") }
|
||||
|
||||
// securitySources reduces a host's apt source files to only the entries that
|
||||
// point at a security suite, so an upgrade run against them installs security
|
||||
// fixes and nothing else.
|
||||
//
|
||||
// It is a pure function of file contents so it is tested on any platform. The
|
||||
// caller writes list to a *.list file and deb822 to a *.sources file in a
|
||||
// temporary SourceParts directory: keeping deb822 paragraphs as deb822 means
|
||||
// an inline Signed-By key block survives verbatim, which a conversion to
|
||||
// one-line format could not carry.
|
||||
//
|
||||
// ok is false when no security suite exists at all. The caller must then
|
||||
// report unsupported, never fall back to installing everything.
|
||||
func securitySources(files map[string]string) (list string, deb822 string, ok bool) {
|
||||
paths := make([]string, 0, len(files))
|
||||
for p := range files {
|
||||
paths = append(paths, p)
|
||||
}
|
||||
sort.Strings(paths) // deterministic output
|
||||
|
||||
var lb, db strings.Builder
|
||||
for _, p := range paths {
|
||||
if strings.HasSuffix(p, ".sources") {
|
||||
db.WriteString(filterDeb822(files[p]))
|
||||
} else {
|
||||
lb.WriteString(filterOneLine(files[p]))
|
||||
}
|
||||
}
|
||||
list, deb822 = lb.String(), db.String()
|
||||
return list, deb822, list != "" || deb822 != ""
|
||||
}
|
||||
|
||||
func filterOneLine(content string) string {
|
||||
var b strings.Builder
|
||||
for _, raw := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 3 || fields[0] != "deb" {
|
||||
continue
|
||||
}
|
||||
i := 1
|
||||
if strings.HasPrefix(fields[i], "[") {
|
||||
// Options run until the token that closes the bracket.
|
||||
for i < len(fields) && !strings.HasSuffix(fields[i], "]") {
|
||||
i++
|
||||
}
|
||||
i++
|
||||
}
|
||||
// fields[i] is the URI, fields[i+1] the suite.
|
||||
if i+1 < len(fields) && isSecuritySuite(fields[i+1]) {
|
||||
b.WriteString(line)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// deb822Fields groups a paragraph's lines into fields. A line that starts
|
||||
// with a space or tab continues the field above it (a folded or multi-line
|
||||
// value, such as a long Suites list or an inline Signed-By key block).
|
||||
func deb822Fields(para string) [][]string {
|
||||
var fields [][]string
|
||||
for _, l := range strings.Split(strings.Trim(para, "\n"), "\n") {
|
||||
if (strings.HasPrefix(l, " ") || strings.HasPrefix(l, "\t")) && len(fields) > 0 {
|
||||
fields[len(fields)-1] = append(fields[len(fields)-1], l)
|
||||
continue
|
||||
}
|
||||
fields = append(fields, []string{l})
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func filterDeb822(content string) string {
|
||||
var b strings.Builder
|
||||
for _, para := range strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n\n") {
|
||||
var out []string
|
||||
isDeb, enabled, kept := false, true, false
|
||||
for _, field := range deb822Fields(para) {
|
||||
key, val, found := strings.Cut(field[0], ":")
|
||||
k := strings.ToLower(strings.TrimSpace(key))
|
||||
v := strings.TrimSpace(val)
|
||||
switch {
|
||||
case found && k == "types":
|
||||
for _, t := range strings.Fields(v) {
|
||||
if t == "deb" {
|
||||
isDeb = true
|
||||
}
|
||||
}
|
||||
case found && k == "enabled":
|
||||
enabled = strings.ToLower(v) != "no"
|
||||
case found && k == "suites":
|
||||
// The value runs across every continuation line. The filtered
|
||||
// result is written back as one line and the continuation
|
||||
// lines are dropped with the rest of the original field.
|
||||
all := strings.Fields(strings.Join(append([]string{v}, field[1:]...), " "))
|
||||
var sec []string
|
||||
for _, s := range all {
|
||||
if isSecuritySuite(s) {
|
||||
sec = append(sec, s)
|
||||
}
|
||||
}
|
||||
if len(sec) == 0 {
|
||||
continue // drop the field; the paragraph is dropped below
|
||||
}
|
||||
kept = true
|
||||
out = append(out, "Suites: "+strings.Join(sec, " "))
|
||||
continue
|
||||
}
|
||||
// Every other field, continuation lines included, stays verbatim.
|
||||
out = append(out, field...)
|
||||
}
|
||||
if isDeb && enabled && kept {
|
||||
b.WriteString(strings.Join(out, "\n"))
|
||||
b.WriteString("\n\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSecuritySourcesOneLine(t *testing.T) {
|
||||
files := map[string]string{
|
||||
"/etc/apt/sources.list": `# comment
|
||||
deb http://deb.debian.org/debian bookworm main
|
||||
deb http://deb.debian.org/debian bookworm-updates main
|
||||
deb http://security.debian.org/debian-security bookworm-security main contrib
|
||||
deb [arch=amd64 signed-by=/usr/share/keyrings/x.gpg] http://archive.ubuntu.com/ubuntu jammy-security main
|
||||
deb-src http://security.debian.org/debian-security bookworm-security main
|
||||
`,
|
||||
}
|
||||
list, d822, ok := securitySources(files)
|
||||
if !ok {
|
||||
t.Fatal("ok = false, want true")
|
||||
}
|
||||
if d822 != "" {
|
||||
t.Fatalf("deb822 = %q, want empty", d822)
|
||||
}
|
||||
want := "deb http://security.debian.org/debian-security bookworm-security main contrib\n" +
|
||||
"deb [arch=amd64 signed-by=/usr/share/keyrings/x.gpg] http://archive.ubuntu.com/ubuntu jammy-security main\n"
|
||||
if list != want {
|
||||
t.Fatalf("list =\n%s\nwant\n%s", list, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecuritySourcesDeb822(t *testing.T) {
|
||||
files := map[string]string{
|
||||
"/etc/apt/sources.list.d/ubuntu.sources": `Types: deb
|
||||
URIs: http://archive.ubuntu.com/ubuntu/
|
||||
Suites: noble noble-updates noble-backports
|
||||
Components: main restricted
|
||||
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
||||
|
||||
Types: deb
|
||||
URIs: http://security.ubuntu.com/ubuntu/
|
||||
Suites: noble-security
|
||||
Components: main restricted
|
||||
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
||||
`,
|
||||
}
|
||||
list, d822, ok := securitySources(files)
|
||||
if !ok || list != "" {
|
||||
t.Fatalf("ok=%v list=%q", ok, list)
|
||||
}
|
||||
if !strings.Contains(d822, "Suites: noble-security") || strings.Contains(d822, "noble-updates") {
|
||||
t.Fatalf("deb822 wrong:\n%s", d822)
|
||||
}
|
||||
if !strings.Contains(d822, "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg") {
|
||||
t.Fatalf("Signed-By must be kept verbatim:\n%s", d822)
|
||||
}
|
||||
}
|
||||
|
||||
// A paragraph listing several suites keeps only the security ones.
|
||||
func TestSecuritySourcesDeb822MixedSuites(t *testing.T) {
|
||||
files := map[string]string{"/x.sources": "Types: deb\nURIs: http://a/\nSuites: noble noble-security\nComponents: main\n"}
|
||||
_, d822, ok := securitySources(files)
|
||||
if !ok || !strings.Contains(d822, "Suites: noble-security\n") || strings.Contains(d822, "Suites: noble noble") {
|
||||
t.Fatalf("got ok=%v\n%s", ok, d822)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecuritySourcesDisabledParagraphIgnored(t *testing.T) {
|
||||
files := map[string]string{"/x.sources": "Types: deb\nURIs: http://a/\nSuites: noble-security\nComponents: main\nEnabled: no\n"}
|
||||
if _, _, ok := securitySources(files); ok {
|
||||
t.Fatal("a disabled paragraph must not count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecuritySourcesNone(t *testing.T) {
|
||||
files := map[string]string{"/etc/apt/sources.list": "deb http://mirror/debian bookworm main\n"}
|
||||
if _, _, ok := securitySources(files); ok {
|
||||
t.Fatal("ok = true with no security suite, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// A folded Suites field continues on lines that start with whitespace. Those
|
||||
// lines belong to Suites and must be filtered with it, not copied verbatim.
|
||||
func TestSecuritySourcesDeb822FoldedSuites(t *testing.T) {
|
||||
files := map[string]string{"/x.sources": "Types: deb\nURIs: http://a/\nSuites: noble\n noble-security\nComponents: main\n"}
|
||||
_, d822, ok := securitySources(files)
|
||||
if !ok || !strings.Contains(d822, "Suites: noble-security\n") {
|
||||
t.Fatalf("got ok=%v\n%s", ok, d822)
|
||||
}
|
||||
if strings.Contains(d822, "\n noble") {
|
||||
t.Fatalf("the folded continuation line must be dropped:\n%s", d822)
|
||||
}
|
||||
}
|
||||
|
||||
// A folded Suites field with no security suite on any line drops the paragraph.
|
||||
func TestSecuritySourcesDeb822FoldedSuitesNoSecurity(t *testing.T) {
|
||||
files := map[string]string{"/x.sources": "Types: deb\nURIs: http://a/\nSuites: noble\n\tnoble-updates\nComponents: main\n"}
|
||||
if _, _, ok := securitySources(files); ok {
|
||||
t.Fatal("no security suite across the folded lines, want ok=false")
|
||||
}
|
||||
}
|
||||
|
||||
// An inline Signed-By key block is a multi-line field of its own. Its
|
||||
// continuation lines stay verbatim, including the "." blank-line marker.
|
||||
func TestSecuritySourcesDeb822InlineSignedBy(t *testing.T) {
|
||||
key := "Signed-By: -----BEGIN PGP PUBLIC KEY BLOCK-----\n .\n mQINBGRkZXYBEAC\n -----END PGP PUBLIC KEY BLOCK-----\n"
|
||||
files := map[string]string{"/x.sources": "Types: deb\nURIs: http://a/\nSuites: noble noble-security\nComponents: main\n" + key}
|
||||
_, d822, ok := securitySources(files)
|
||||
if !ok || !strings.Contains(d822, "Suites: noble-security\n") {
|
||||
t.Fatalf("got ok=%v\n%s", ok, d822)
|
||||
}
|
||||
if !strings.Contains(d822, key) {
|
||||
t.Fatalf("inline key block must be kept verbatim:\n%s", d822)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// canStart is the whole "may this phase begin" decision. The maintenance
|
||||
// window deadline only gates the start of a phase: a package manager that is
|
||||
// already running is never interrupted by it, because killing apt, dnf or
|
||||
// Windows Update partway through a transaction is worse than letting it
|
||||
// finish late. A zero deadline is a manual run, which always may start.
|
||||
func canStart(now, deadline time.Time) bool {
|
||||
return deadline.IsZero() || now.Before(deadline)
|
||||
}
|
||||
|
||||
// startGate returns the error reported when a phase is refused.
|
||||
func startGate(deadline time.Time, phase string) error {
|
||||
if canStart(time.Now(), deadline) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("the maintenance window ended before %s could start", phase)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCanStart(t *testing.T) {
|
||||
now := time.Date(2026, 9, 20, 2, 30, 0, 0, time.UTC)
|
||||
cases := []struct {
|
||||
name string
|
||||
deadline time.Time
|
||||
want bool
|
||||
}{
|
||||
{"no deadline (manual run)", time.Time{}, true},
|
||||
{"deadline ahead", now.Add(time.Second), true},
|
||||
{"deadline exactly now", now, false},
|
||||
{"deadline passed", now.Add(-time.Minute), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := canStart(now, c.deadline); got != c.want {
|
||||
t.Errorf("%s: got %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartGate(t *testing.T) {
|
||||
if err := startGate(time.Time{}, "the upgrade"); err != nil {
|
||||
t.Fatalf("zero deadline: %v", err)
|
||||
}
|
||||
err := startGate(time.Now().Add(-time.Minute), "the upgrade")
|
||||
if err == nil || !strings.Contains(err.Error(), "the maintenance window ended before the upgrade could start") {
|
||||
t.Fatalf("passed deadline: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package updates
|
||||
|
||||
import "sync"
|
||||
|
||||
// outputTailMax bounds the package-manager output carried back in a
|
||||
// PatchResult. The end of the output is where apt and dnf say what failed, so
|
||||
// the newest bytes are the ones kept.
|
||||
const outputTailMax = 64 << 10
|
||||
|
||||
// tailBuffer is an io.Writer that retains only the last max bytes written.
|
||||
// Stdout and stderr are both pointed at one, so it is safe for concurrent use.
|
||||
type tailBuffer struct {
|
||||
mu sync.Mutex
|
||||
max int
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func newTailBuffer(max int) *tailBuffer { return &tailBuffer{max: max} }
|
||||
|
||||
func (t *tailBuffer) Write(p []byte) (int, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.buf = append(t.buf, p...)
|
||||
if over := len(t.buf) - t.max; over > 0 {
|
||||
t.buf = append([]byte(nil), t.buf[over:]...)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (t *tailBuffer) String() string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return string(t.buf)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTailBufferKeepsNewestBytes(t *testing.T) {
|
||||
b := newTailBuffer(10)
|
||||
_, _ = b.Write([]byte("0123456789"))
|
||||
_, _ = b.Write([]byte("abcde"))
|
||||
if got := b.String(); got != "56789abcde" {
|
||||
t.Fatalf("got %q, want %q", got, "56789abcde")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTailBufferSingleWriteLargerThanMax(t *testing.T) {
|
||||
b := newTailBuffer(4)
|
||||
n, err := b.Write([]byte("abcdefgh"))
|
||||
if err != nil || n != 8 {
|
||||
t.Fatalf("Write must report the full length consumed, got %d, %v", n, err)
|
||||
}
|
||||
if got := b.String(); got != "efgh" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTailBufferDefaultCap(t *testing.T) {
|
||||
b := newTailBuffer(outputTailMax)
|
||||
_, _ = b.Write([]byte(strings.Repeat("x", outputTailMax+100)))
|
||||
if len(b.String()) != outputTailMax {
|
||||
t.Fatalf("len %d, want %d", len(b.String()), outputTailMax)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,11 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PackageUpdate is one pending update. On Linux it is a package with a version
|
||||
// on each side. On Windows CurrentVersion is empty and NewVersion carries the
|
||||
// KB article ID: a Windows update is not a version bump of a named package,
|
||||
@@ -9,16 +15,57 @@ type PackageUpdate struct {
|
||||
Name string
|
||||
CurrentVersion string
|
||||
NewVersion string
|
||||
Phased bool // Ubuntu phased update this host is not yet selected for
|
||||
}
|
||||
|
||||
// CheckAvailable lists pending OS updates.
|
||||
func CheckAvailable() ([]PackageUpdate, error) { return checkAvailable() }
|
||||
|
||||
// ApplyAll installs every pending update. It never reboots: a control plane
|
||||
// silently restarting a production server is unrecoverable from the UI, so the
|
||||
// reboot stays a decision a person or a workflow makes. RebootRequired reports
|
||||
// when one is owed.
|
||||
func ApplyAll() error { return applyAll() }
|
||||
// ApplyOptions selects what an Apply run installs and when it must stop.
|
||||
type ApplyOptions struct {
|
||||
// SecurityOnly installs security fixes only. A host with no security
|
||||
// metadata reports Unsupported and installs nothing: it never falls back
|
||||
// to installing everything.
|
||||
SecurityOnly bool
|
||||
// Deadline is the end of the maintenance window. It only gates the start
|
||||
// of each phase (index refresh, upgrade, Windows install): a phase that
|
||||
// has not started by then is not started, and one already running is
|
||||
// allowed to finish, bounded by defaultApplyCap from its own start. Zero
|
||||
// means a manual run with no window.
|
||||
Deadline time.Time
|
||||
}
|
||||
|
||||
// Result is what one Apply run did. Output is the tail of the package
|
||||
// manager's combined output, for the operator to read when something failed.
|
||||
type Result struct {
|
||||
Output string
|
||||
Unsupported bool
|
||||
Reason string // why Unsupported, in words for the run page
|
||||
}
|
||||
|
||||
// ErrBusy means another Apply is already running on this host.
|
||||
var ErrBusy = errors.New("an update run is already in progress on this host")
|
||||
|
||||
// defaultApplyCap is the backstop for one started upgrade command, counted
|
||||
// from that command's own start. It exists for a package manager that hangs,
|
||||
// not to enforce the window.
|
||||
const defaultApplyCap = 2 * time.Hour
|
||||
|
||||
var applyMu sync.Mutex
|
||||
|
||||
// Apply installs pending updates. It never reboots: ScheduleReboot is a
|
||||
// separate decision taken by the caller, and only when the command asked.
|
||||
func Apply(opts ApplyOptions) (Result, error) {
|
||||
if !applyMu.TryLock() {
|
||||
return Result{}, ErrBusy
|
||||
}
|
||||
defer applyMu.Unlock()
|
||||
return apply(opts.SecurityOnly, opts.Deadline)
|
||||
}
|
||||
|
||||
// ScheduleReboot restarts the host after a short grace period, so a result
|
||||
// sent just before it has time to leave.
|
||||
func ScheduleReboot() error { return scheduleReboot() }
|
||||
|
||||
// RebootRequired reports whether this host is waiting on a restart.
|
||||
func RebootRequired() bool { return rebootRequired() }
|
||||
|
||||
@@ -4,9 +4,14 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -22,12 +27,14 @@ func detectPM() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
|
||||
|
||||
func checkAvailable() ([]PackageUpdate, error) {
|
||||
switch detectPM() {
|
||||
case "apt":
|
||||
return checkApt()
|
||||
ups, err := checkApt()
|
||||
if err == nil && len(ups) > 0 {
|
||||
markPhased(ups, aptPhasedDeferred())
|
||||
}
|
||||
return ups, err
|
||||
case "dnf":
|
||||
return checkDnfYum("dnf")
|
||||
case "yum":
|
||||
@@ -43,30 +50,160 @@ func checkAvailable() ([]PackageUpdate, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// aptRefreshTimeout bounds the index refresh alone. The upgrade itself runs
|
||||
// under defaultApplyCap: one shared five-minute limit used to kill large
|
||||
// upgrades partway through.
|
||||
const aptRefreshTimeout = 5 * time.Minute
|
||||
|
||||
func applyAll() error {
|
||||
switch detectPM() {
|
||||
case "apt":
|
||||
// termGrace is how long a command has to exit after SIGTERM before it is
|
||||
// killed. Package managers finish or roll back their current step on TERM.
|
||||
const termGrace = 5 * time.Minute
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
if err := exec.CommandContext(ctx, "apt-get", "update", "-qq").Run(); err != nil {
|
||||
return err
|
||||
}
|
||||
return exec.CommandContext(ctx, "apt-get", "upgrade", "-y").Run()
|
||||
case "dnf":
|
||||
return exec.Command("dnf", "upgrade", "-y").Run()
|
||||
case "yum":
|
||||
return exec.Command("yum", "upgrade", "-y").Run()
|
||||
case "pacman":
|
||||
return exec.Command("pacman", "-Syu", "--noconfirm").Run()
|
||||
case "zypper":
|
||||
return exec.Command("zypper", "update", "-y").Run()
|
||||
case "apk":
|
||||
return exec.Command("apk", "upgrade").Run()
|
||||
default:
|
||||
return nil
|
||||
// phase is one package manager command, started only if the window deadline
|
||||
// has not passed and then bounded by its own limit from its own start.
|
||||
type phase struct {
|
||||
deadline time.Time
|
||||
out io.Writer
|
||||
env []string
|
||||
}
|
||||
|
||||
func (p phase) run(name string, limit time.Duration, args ...string) error {
|
||||
return p.runNamed("the upgrade", name, limit, args...)
|
||||
}
|
||||
|
||||
func (p phase) runNamed(label, name string, limit time.Duration, args ...string) error {
|
||||
if err := startGate(p.deadline, label); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), limit)
|
||||
defer cancel()
|
||||
fmt.Fprintf(p.out, "$ %s %s\n", name, strings.Join(args, " "))
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Stdout, cmd.Stderr = p.out, p.out
|
||||
cmd.Env = append(os.Environ(), p.env...)
|
||||
// On the backstop, ask the package manager to stop rather than killing it
|
||||
// outright, and give it time to leave its database consistent.
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }
|
||||
cmd.WaitDelay = termGrace
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func apply(securityOnly bool, deadline time.Time) (Result, error) {
|
||||
out := newTailBuffer(outputTailMax)
|
||||
ph := phase{deadline: deadline, out: out}
|
||||
|
||||
var err error
|
||||
switch pm := detectPM(); pm {
|
||||
case "apt":
|
||||
var res Result
|
||||
res, err = applyApt(deadline, out, securityOnly)
|
||||
if res.Unsupported {
|
||||
res.Output = out.String()
|
||||
return res, nil
|
||||
}
|
||||
case "dnf", "yum":
|
||||
args := []string{"upgrade", "-y"}
|
||||
if securityOnly {
|
||||
args = append(args, "--security")
|
||||
}
|
||||
err = ph.run(pm, defaultApplyCap, args...)
|
||||
case "zypper":
|
||||
if securityOnly {
|
||||
err = ph.run("zypper", defaultApplyCap, "--non-interactive", "patch", "--category", "security")
|
||||
} else {
|
||||
err = ph.run("zypper", defaultApplyCap, "--non-interactive", "update")
|
||||
}
|
||||
// 102 and 103 mean "installed, and a reboot or restart is now needed".
|
||||
// That is success; RebootRequired reports the rest.
|
||||
var ee *exec.ExitError
|
||||
if errors.As(err, &ee) && (ee.ExitCode() == 102 || ee.ExitCode() == 103) {
|
||||
err = nil
|
||||
}
|
||||
case "pacman":
|
||||
if securityOnly {
|
||||
return Result{Unsupported: true, Reason: "pacman publishes no security metadata"}, nil
|
||||
}
|
||||
err = ph.run("pacman", defaultApplyCap, "-Syu", "--noconfirm")
|
||||
case "apk":
|
||||
if securityOnly {
|
||||
return Result{Unsupported: true, Reason: "apk publishes no security metadata"}, nil
|
||||
}
|
||||
if err = ph.runNamed("the apk index refresh", "apk", aptRefreshTimeout, "update"); err == nil {
|
||||
err = ph.run("apk", defaultApplyCap, "upgrade")
|
||||
}
|
||||
default:
|
||||
return Result{Unsupported: true, Reason: "no supported package manager found"}, nil
|
||||
}
|
||||
return Result{Output: out.String()}, err
|
||||
}
|
||||
|
||||
func applyApt(deadline time.Time, out io.Writer, securityOnly bool) (Result, error) {
|
||||
ph := phase{deadline: deadline, out: out, env: []string{"DEBIAN_FRONTEND=noninteractive"}}
|
||||
var srcOpts []string
|
||||
if securityOnly {
|
||||
dir, res, err := writeSecuritySourceParts()
|
||||
if err != nil || res.Unsupported {
|
||||
return res, err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
srcOpts = []string{
|
||||
"-o", "Dir::Etc::SourceList=/dev/null",
|
||||
"-o", "Dir::Etc::SourceParts=" + dir,
|
||||
// Without this, an update against the reduced source set deletes
|
||||
// every other list file and the next normal apt call sees nothing.
|
||||
"-o", "APT::Get::List-Cleanup=0",
|
||||
}
|
||||
}
|
||||
if err := ph.runNamed("the apt index refresh", "apt-get", aptRefreshTimeout, append([]string{"update", "-q"}, srcOpts...)...); err != nil {
|
||||
return Result{}, fmt.Errorf("apt-get update: %w", err)
|
||||
}
|
||||
args := []string{"upgrade", "-y", "-q",
|
||||
"-o", "Dpkg::Options::=--force-confdef",
|
||||
"-o", "Dpkg::Options::=--force-confold"}
|
||||
return Result{}, ph.runNamed("the upgrade", "apt-get", defaultApplyCap, append(args, srcOpts...)...)
|
||||
}
|
||||
|
||||
// writeSecuritySourceParts writes the security-only sources to a temporary
|
||||
// directory for Dir::Etc::SourceParts. The caller removes the directory.
|
||||
func writeSecuritySourceParts() (string, Result, error) {
|
||||
files := map[string]string{}
|
||||
paths := []string{"/etc/apt/sources.list"}
|
||||
for _, pat := range []string{"/etc/apt/sources.list.d/*.list", "/etc/apt/sources.list.d/*.sources"} {
|
||||
m, _ := filepath.Glob(pat)
|
||||
paths = append(paths, m...)
|
||||
}
|
||||
for _, p := range paths {
|
||||
if b, err := os.ReadFile(p); err == nil {
|
||||
files[p] = string(b)
|
||||
}
|
||||
}
|
||||
list, d822, ok := securitySources(files)
|
||||
if !ok {
|
||||
return "", Result{Unsupported: true, Reason: "no security suites found in apt sources"}, nil
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "vantage-apt-security-")
|
||||
if err != nil {
|
||||
return "", Result{}, err
|
||||
}
|
||||
if list != "" {
|
||||
if err := os.WriteFile(filepath.Join(dir, "security.list"), []byte(list), 0o644); err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return "", Result{}, err
|
||||
}
|
||||
}
|
||||
if d822 != "" {
|
||||
if err := os.WriteFile(filepath.Join(dir, "security.sources"), []byte(d822), 0o644); err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return "", Result{}, err
|
||||
}
|
||||
}
|
||||
return dir, Result{}, nil
|
||||
}
|
||||
|
||||
// scheduleReboot gives the host one minute, so the PatchResult announcing the
|
||||
// reboot is on the wire before the network goes down.
|
||||
func scheduleReboot() error {
|
||||
return exec.Command("shutdown", "-r", "+1", "Vantage patch policy").Run()
|
||||
}
|
||||
|
||||
// rebootRequired reads what the distributions themselves record. Debian and
|
||||
|
||||
@@ -4,6 +4,17 @@
|
||||
// without it this file compiles on Linux too and collides with updates_linux.go.
|
||||
package updates
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
func checkAvailable() ([]PackageUpdate, error) { return nil, nil }
|
||||
func applyAll() error { return nil }
|
||||
func rebootRequired() bool { return false }
|
||||
|
||||
func apply(securityOnly bool, deadline time.Time) (Result, error) {
|
||||
return Result{Unsupported: true, Reason: "OS updates are not supported on this platform"}, nil
|
||||
}
|
||||
|
||||
func scheduleReboot() error { return errors.New("reboot is not supported on this platform") }
|
||||
|
||||
func rebootRequired() bool { return false }
|
||||
|
||||
@@ -3,6 +3,7 @@ package updates
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -14,10 +15,6 @@ const (
|
||||
// routinely slow. Ten minutes is not generous, it is realistic.
|
||||
searchTimeout = 10 * time.Minute
|
||||
|
||||
// A patch-Tuesday cumulative genuinely takes this long to download and
|
||||
// install on a modest server.
|
||||
applyTimeout = 60 * time.Minute
|
||||
|
||||
rebootTimeout = 2 * time.Minute
|
||||
)
|
||||
|
||||
@@ -39,37 +36,6 @@ foreach ($u in $result.Updates) {
|
||||
ConvertTo-Json -InputObject @($rows) -Depth 3 -Compress
|
||||
`
|
||||
|
||||
const applyScript = `
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$session = New-Object -ComObject Microsoft.Update.Session
|
||||
$result = $session.CreateUpdateSearcher().Search("IsInstalled=0 and Type='Software' and IsHidden=0")
|
||||
|
||||
$batch = New-Object -ComObject Microsoft.Update.UpdateColl
|
||||
foreach ($u in $result.Updates) {
|
||||
if ($u.InstallationBehavior.CanRequestUserInput) { continue }
|
||||
if (-not $u.EulaAccepted) {
|
||||
try { $u.AcceptEula() } catch { continue }
|
||||
}
|
||||
$null = $batch.Add($u)
|
||||
}
|
||||
|
||||
if ($batch.Count -eq 0) { Write-Output 'nothing-to-install'; exit 0 }
|
||||
|
||||
$downloader = $session.CreateUpdateDownloader()
|
||||
$downloader.Updates = $batch
|
||||
$null = $downloader.Download()
|
||||
|
||||
$installer = $session.CreateUpdateInstaller()
|
||||
$installer.Updates = $batch
|
||||
$r = $installer.Install()
|
||||
|
||||
Write-Output ('resultcode=' + $r.ResultCode)
|
||||
# 2 = succeeded, 3 = succeeded with errors. Anything else failed, and this
|
||||
# process must exit non-zero so the agent logs a failure rather than an ack.
|
||||
if ($r.ResultCode -ne 2 -and $r.ResultCode -ne 3) { exit 1 }
|
||||
exit 0
|
||||
`
|
||||
|
||||
const rebootScript = `
|
||||
$ErrorActionPreference = 'SilentlyContinue'
|
||||
$si = New-Object -ComObject Microsoft.Update.SystemInfo
|
||||
@@ -95,14 +61,29 @@ func checkAvailable() ([]PackageUpdate, error) {
|
||||
return parseUpdateSearch(out)
|
||||
}
|
||||
|
||||
func applyAll() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), applyTimeout)
|
||||
func apply(securityOnly bool, deadline time.Time) (Result, error) {
|
||||
if err := startGate(deadline, "the Windows Update install"); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
// The window deadline only gates the start. Once running, the install is
|
||||
// bounded by defaultApplyCap from its own start, with the default kill:
|
||||
// Windows has no SIGTERM to offer PowerShell.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultApplyCap)
|
||||
defer cancel()
|
||||
|
||||
if _, err := winexec.Run(ctx, applyScript); err != nil {
|
||||
return fmt.Errorf("windows update install: %w", err)
|
||||
out, err := winexec.Run(ctx, applyScriptFor(securityOnly))
|
||||
tail := newTailBuffer(outputTailMax)
|
||||
_, _ = tail.Write([]byte(out))
|
||||
if err != nil {
|
||||
return Result{Output: tail.String()}, fmt.Errorf("windows update install: %w", err)
|
||||
}
|
||||
return nil
|
||||
return Result{Output: tail.String()}, nil
|
||||
}
|
||||
|
||||
// scheduleReboot gives the host sixty seconds, so the PatchResult announcing
|
||||
// the reboot is sent before the service stops.
|
||||
func scheduleReboot() error {
|
||||
return exec.Command("shutdown", "/r", "/t", "60", "/c", "Vantage patch policy").Run()
|
||||
}
|
||||
|
||||
func rebootRequired() bool {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package updates
|
||||
|
||||
// applyScriptFor builds the Windows Update install script. It lives in a file
|
||||
// with no build tag so its content is tested on Linux: this module has no
|
||||
// Windows CI.
|
||||
//
|
||||
// Security-only keeps updates in the Security Updates or Critical Updates
|
||||
// classifications. Those two GUIDs are fixed by Microsoft and identical on
|
||||
// every Windows Update and WSUS server.
|
||||
func applyScriptFor(securityOnly bool) string {
|
||||
flag := "$false"
|
||||
if securityOnly {
|
||||
flag = "$true"
|
||||
}
|
||||
return "$SecurityOnly = " + flag + "\n" + applyScriptBody
|
||||
}
|
||||
|
||||
const applyScriptBody = `
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$securityCats = @('0FA1201D-4330-4FA8-8AE9-B877473B6441', 'E6CF1350-C01B-414D-A61F-263D14D133B4')
|
||||
$session = New-Object -ComObject Microsoft.Update.Session
|
||||
$result = $session.CreateUpdateSearcher().Search("IsInstalled=0 and Type='Software' and IsHidden=0")
|
||||
|
||||
$batch = New-Object -ComObject Microsoft.Update.UpdateColl
|
||||
foreach ($u in $result.Updates) {
|
||||
if ($u.InstallationBehavior.CanRequestUserInput) { continue }
|
||||
if ($SecurityOnly) {
|
||||
$isSec = $false
|
||||
foreach ($c in $u.Categories) { if ($securityCats -contains $c.CategoryID.ToUpper()) { $isSec = $true } }
|
||||
if (-not $isSec) { continue }
|
||||
}
|
||||
if (-not $u.EulaAccepted) {
|
||||
try { $u.AcceptEula() } catch { continue }
|
||||
}
|
||||
Write-Output ('selected: ' + $u.Title)
|
||||
$null = $batch.Add($u)
|
||||
}
|
||||
|
||||
if ($batch.Count -eq 0) { Write-Output 'nothing-to-install'; exit 0 }
|
||||
|
||||
$downloader = $session.CreateUpdateDownloader()
|
||||
$downloader.Updates = $batch
|
||||
$null = $downloader.Download()
|
||||
|
||||
$installer = $session.CreateUpdateInstaller()
|
||||
$installer.Updates = $batch
|
||||
$r = $installer.Install()
|
||||
|
||||
Write-Output ('resultcode=' + $r.ResultCode)
|
||||
# 2 = succeeded, 3 = succeeded with errors. Anything else failed, and this
|
||||
# process must exit non-zero so the agent reports a failure rather than an ack.
|
||||
if ($r.ResultCode -ne 2 -and $r.ResultCode -ne 3) { exit 1 }
|
||||
exit 0
|
||||
`
|
||||
@@ -0,0 +1,29 @@
|
||||
package updates
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
catSecurity = "0FA1201D-4330-4FA8-8AE9-B877473B6441"
|
||||
catCritical = "E6CF1350-C01B-414D-A61F-263D14D133B4"
|
||||
)
|
||||
|
||||
func TestApplyScriptSecurityOnly(t *testing.T) {
|
||||
s := applyScriptFor(true)
|
||||
if !strings.HasPrefix(strings.TrimSpace(s), "$SecurityOnly = $true") {
|
||||
t.Fatalf("script must open with the flag set:\n%s", s)
|
||||
}
|
||||
for _, id := range []string{catSecurity, catCritical} {
|
||||
if !strings.Contains(s, id) {
|
||||
t.Errorf("script missing category %s", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyScriptAll(t *testing.T) {
|
||||
if !strings.HasPrefix(strings.TrimSpace(applyScriptFor(false)), "$SecurityOnly = $false") {
|
||||
t.Fatal("script must open with the flag cleared")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user