vantage-agent/internal/sync/sync.go

package agentsync

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/config"
	agentexec "gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/exec"
	grpcclient "gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/grpc"
	"gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/grpc/pb"
	"gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/inventory"
	"gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/keys"
	"gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/monitors"
	agentproxy "gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/proxy"
	"gitea.hostxtra.co.uk/mrhid6/vantage/agent/internal/updates"
)

func Run(ctx context.Context, cfg *config.Config, version string) error {
	client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
	if err != nil {
		return fmt.Errorf("dial grpc: %w", err)
	}
	defer client.Close()

	if cfg.PreRegToken != "" {
		log.Println("registering with server...")
		hostname, _ := os.Hostname()
		ipAddress := localIP()
		osInfo := fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH)

		agentToken, err := client.Register(cfg.ServerID, cfg.PreRegToken, hostname, ipAddress, osInfo)
		if err != nil {
			return fmt.Errorf("registration failed: %w", err)
		}

		cfg.AgentToken = agentToken
		cfg.PreRegToken = ""
		if err := config.Save(cfg); err != nil {
			return fmt.Errorf("save config: %w", err)
		}
		log.Println("registration successful")

		client.Close()
		client, err = grpcclient.New(cfg.ServerURL, cfg.TLS)
		if err != nil {
			return fmt.Errorf("reconnect: %w", err)
		}
	}

	if cfg.AgentToken == "" {
		return fmt.Errorf("no agent token available registration required")
	}

	go runCommandStream(ctx, cfg)

	go runUpdateCheck(ctx, cfg)

	go runInventory(ctx, cfg)

	go monitors.Run(ctx, cfg)

	ticker := time.NewTicker(cfg.PollInterval)
	defer ticker.Stop()

	if err := poll(client, cfg, version); err != nil {
		log.Printf("poll error: %v", err)
	}

	for {
		select {
		case <-ctx.Done():
			return nil
		case <-ticker.C:
			if err := poll(client, cfg, version); err != nil {
				log.Printf("poll error: %v", err)
			}
		}
	}
}

func poll(client *grpcclient.Client, cfg *config.Config, version string) error {
	resp, err := client.SyncKeys(cfg.ServerID, cfg.AgentToken, version)
	if err != nil {
		return fmt.Errorf("SyncKeys: %w", err)
	}

	// Stored atomically: the hourly package loop reads this from another
	// goroutine. Absent on the wire decodes as false, so an older server leaves
	// collection off rather than on.
	collectPackagesFlag.Store(resp.CollectPackages)

	desired := resp.PublicKeys

	if runtime.GOOS != "linux" {
		return nil
	}

	current, err := keys.ReadAuthorizedKeys()
	if err != nil {
		return fmt.Errorf("read authorized_keys: %w", err)
	}

	if !keys.StateChanged(current, desired) {
		log.Println("authorized_keys unchanged, skipping write")
		return nil
	}

	if err := keys.WriteAuthorizedKeys(desired); err != nil {
		return fmt.Errorf("write authorized_keys: %w", err)
	}
	log.Printf("authorized_keys updated (%d keys)", len(desired))
	return nil
}

// How long a command stream must survive before it counts as having worked.
// Past this, the next drop is treated as a fresh incident rather than as the
// continuation of a run of failures.
const streamHealthyAfter = time.Minute

// Stream staleness. The server beats every 20s, so 70s tolerates three missed
// beats before the stream is written off — high enough that a slow network or a
// briefly busy server does not cost a reconnect, low enough that an agent is
// not uncommandable for minutes after a control-plane restart.
const (
	streamStaleAfter = 70 * time.Second
	streamStaleCheck = 10 * time.Second

	// How often a healthy stream reports itself. Also the interval at which an
	// agent talking to a control plane too old to send heartbeats says so —
	// that agent is running without a watchdog, and the journal should not be
	// silent about it.
	pingSummaryInterval = 5 * time.Minute
)

func runCommandStream(ctx context.Context, cfg *config.Config) {
	backoff := time.Second

	// Two minutes was the old ceiling, and it was reached far too easily. The
	// command stream is what makes this agent controllable at all: while it is
	// down, workflows and console sessions fail as "agent offline" even though
	// SyncKeys keeps polling happily and the fleet list still shows the server
	// active. A shorter ceiling costs a few reconnect attempts; the old one cost
	// two minutes of an agent that looks fine and answers nothing.
	const maxBackoff = 30 * time.Second

	for {
		select {
		case <-ctx.Done():
			return
		default:
		}

		started := time.Now()
		err := connectAndHandleStream(ctx, cfg)
		if ctx.Err() != nil {
			return
		}

		// A stream that stayed up is evidence the control plane is reachable,
		// whatever ended it. Without this the backoff only ever climbed:
		// connectAndHandleStream returns an error on *every* stream end,
		// including a healthy one dropped by a routine deploy, so an agent
		// pinned itself at the ceiling after a handful of ordinary restarts and
		// stayed there for the rest of its life.
		if time.Since(started) >= streamHealthyAfter {
			backoff = time.Second
		}

		// The uptime is in the line because it is what distinguishes a stream
		// that never worked from one that ran for hours and was dropped by a
		// deploy — and it is the same measure that decides whether the backoff
		// resets, so a reader can see why the delay is what it is.
		up := time.Since(started).Truncate(time.Second)
		if err != nil {
			log.Printf("command stream error after %s: %v, reconnecting in %s", up, err, backoff)
		} else {
			log.Printf("command stream closed after %s, reconnecting in %s", up, backoff)
		}

		select {
		case <-ctx.Done():
			return
		case <-time.After(backoff):
		}

		if backoff < maxBackoff {
			backoff *= 2
			if backoff > maxBackoff {
				backoff = maxBackoff
			}
		}
	}
}

func connectAndHandleStream(ctx context.Context, cfg *config.Config) error {
	client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
	if err != nil {
		return fmt.Errorf("dial: %w", err)
	}
	defer client.Close()

	// Cancelling this context is what unblocks Recv when the stream has gone
	// quiet. Without it the watchdog below would have no way to interrupt a
	// read that is never going to return.
	streamCtx, abandon := context.WithCancel(ctx)
	defer abandon()

	stream, err := client.CommandStream(streamCtx)
	if err != nil {
		return fmt.Errorf("open stream: %w", err)
	}

	if err := stream.Send(&pb.AgentMessage{
		ServerId:   cfg.ServerID,
		AgentToken: cfg.AgentToken,
		Ready:      &pb.AgentReady{},
	}); err != nil {
		return fmt.Errorf("send auth: %w", err)
	}

	log.Printf("command stream connected to %s", cfg.ServerURL)

	var sendMu sync.Mutex
	send := func(msg *pb.AgentMessage) error {
		sendMu.Lock()
		defer sendMu.Unlock()
		return stream.Send(msg)
	}

	// Stream liveness, tracked here rather than left to gRPC keepalive.
	//
	// Keepalive operates on the transport, and behind an L7 proxy the transport
	// ends at the proxy: it answers pings whether or not the server behind it
	// is still running. A control-plane pod that dies therefore leaves this
	// agent blocked in Recv on a stream that will never deliver another message
	// and never error, while the control plane dispatches commands into it and
	// the operator watches nothing happen.
	//
	// The watchdog only arms once a ping has actually been seen. A server too
	// old to send them must not be treated as dead — that would put the agent
	// in a reconnect loop against a control plane that is working perfectly.
	var (
		lastMu   sync.Mutex
		lastRecv = time.Now()
		pinged   bool
		beats    int
	)
	markRecv := func(isPing bool) {
		lastMu.Lock()
		lastRecv = time.Now()
		if isPing {
			beats++
			// Logged once per stream, because it is the moment the agent starts
			// holding the control plane to account: before this the watchdog is
			// disarmed and a dead stream would go unnoticed indefinitely.
			if !pinged {
				pinged = true
				log.Printf("command stream heartbeat detected, watchdog armed (%s threshold)", streamStaleAfter)
			}
		}
		lastMu.Unlock()
	}

	go func() {
		t := time.NewTicker(streamStaleCheck)
		defer t.Stop()

		// Reported periodically rather than per beat: at one every 20s the
		// journal would be nothing else. The count is what makes a partial
		// failure visible — beats arriving but fewer than expected is a
		// different problem from beats stopping altogether.
		summary := time.NewTicker(pingSummaryInterval)
		defer summary.Stop()

		for {
			select {
			case <-streamCtx.Done():
				return
			case <-summary.C:
				lastMu.Lock()
				n, armed := beats, pinged
				beats = 0
				lastMu.Unlock()
				if armed {
					log.Printf("command stream healthy, %d heartbeats in the last %s", n, pingSummaryInterval)
				} else {
					log.Printf("command stream up but sending no heartbeats; "+
						"control plane predates them, watchdog stays disarmed (last message %s ago)",
						time.Since(lastRecv).Truncate(time.Second))
				}
			case <-t.C:
				lastMu.Lock()
				idle, armed := time.Since(lastRecv), pinged
				lastMu.Unlock()
				if armed && idle > streamStaleAfter {
					log.Printf("command stream silent for %s (threshold %s), assuming it is dead and reconnecting",
						idle.Truncate(time.Second), streamStaleAfter)
					abandon()
					return
				}
			}
		}
	}()

	for {
		cmd, err := stream.Recv()
		if err != nil {
			return fmt.Errorf("recv: %w", err)
		}
		markRecv(cmd.Ping != nil)

		// Pings carry nothing and are not acknowledged; being received is their
		// whole purpose.
		if cmd.Ping != nil {
			continue
		}

		if cmd.GenerateKey != nil {
			go handleGenerateKey(cfg, cmd)
		}
		if cmd.DeleteKey != nil {
			go handleDeleteKey(cmd)
		}
		if cmd.UpdateAgent != nil {
			go handleUpdateAgent(cmd)
		}
		if cmd.ApplyUpdates != nil {
			go handleApplyUpdates(cfg, cmd)
		}
		if cmd.CleanupWorkspace != nil {
			go handleCleanupWorkspace(cmd)
		}
		if cmd.OpenProxy != nil {
			go handleOpenProxy(ctx, cfg, cmd.OpenProxy)
		}
		if cmd.RunStep != nil {
			go func(rc *pb.RunStepCmd, cid string) {
				emit := func(seq uint64, data []byte) {
					_ = send(&pb.AgentMessage{
						ServerId:   cfg.ServerID,
						AgentToken: cfg.AgentToken,
						StepOutput: &pb.StepOutputChunk{CommandId: cid, Seq: seq, Data: data},
					})
				}
				res := agentexec.RunStep(rc, emit)
				res.CommandId = cid

				_ = send(&pb.AgentMessage{
					ServerId:   cfg.ServerID,
					AgentToken: cfg.AgentToken,
					StepOutput: &pb.StepOutputChunk{CommandId: cid, Eof: true},
				})
				_ = send(&pb.AgentMessage{
					ServerId:   cfg.ServerID,
					AgentToken: cfg.AgentToken,
					StepResult: res,
				})
			}(cmd.RunStep, cmd.CommandId)
			continue
		}
	}
}

func runUpdateCheck(ctx context.Context, cfg *config.Config) {
	const interval = time.Hour

	doCheck := func() {
		pkgs, err := updates.CheckAvailable()
		if err != nil {
			log.Printf("update check error: %v", err)
			return
		}
		pbUpdates := make([]pb.PackageUpdate, len(pkgs))
		for i, p := range pkgs {
			pbUpdates[i] = pb.PackageUpdate{
				Name:           p.Name,
				CurrentVersion: p.CurrentVersion,
				NewVersion:     p.NewVersion,
			}
		}
		client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
		if err != nil {
			log.Printf("update report dial error: %v", err)
			return
		}
		defer client.Close()
		if err := client.ReportUpdates(cfg.ServerID, cfg.AgentToken, pbUpdates); err != nil {
			log.Printf("ReportUpdates error: %v", err)
			return
		}
		log.Printf("reported %d available OS updates", len(pkgs))

		// Same hourly cadence, same connection. A package set changes on
		// roughly the schedule available updates do, so this needs no timer of
		// its own.
		reportPackages(client, cfg)
	}

	doCheck()
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			doCheck()
		}
	}
}

func runInventory(ctx context.Context, cfg *config.Config) {
	client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
	if err != nil {
		log.Printf("inventory dial error: %v", err)
		return
	}
	defer client.Close()

	report := func(static bool) {
		r := inventory.Collect(static)
		r.ServerId = cfg.ServerID
		r.AgentToken = cfg.AgentToken
		if err := client.ReportInventory(r); err != nil {
			log.Printf("report inventory: %v", err)
		}
	}

	report(true)
	ticker := time.NewTicker(30 * time.Second)
	defer ticker.Stop()
	tick := 0
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			tick++
			report(tick%30 == 0)
		}
	}
}

func handleApplyUpdates(cfg *config.Config, cmd *pb.ServerCommand) {
	log.Printf("applying OS updates (cmd=%s)…", cmd.CommandId)
	if err := updates.ApplyAll(); err != nil {
		log.Printf("OS upgrade failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}
	log.Printf("OS updates applied successfully (cmd=%s)", cmd.CommandId)

	client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
	if err != nil {
		return
	}
	defer client.Close()
	_ = client.ReportUpdates(cfg.ServerID, cfg.AgentToken, nil)
}

func handleCleanupWorkspace(cmd *pb.ServerCommand) {
	id := cmd.CleanupWorkspace.WorkspaceId
	dir := agentexec.WorkspacePath(id)
	if err := os.RemoveAll(dir); err != nil {
		log.Printf("cleanup workspace %s failed (cmd=%s): %v", dir, cmd.CommandId, err)
		return
	}
	log.Printf("removed run workspace %s (cmd=%s)", dir, cmd.CommandId)
}

// handleOpenProxy relays one console connection. It uses its own gRPC
// connection so console traffic never shares a stream with commands, key sync
// or workflow output.
func handleOpenProxy(ctx context.Context, cfg *config.Config, cmd *pb.OpenProxyCmd) {
	client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
	if err != nil {
		log.Printf("proxy %s: dial control plane: %v", cmd.ProxyId, err)
		return
	}
	defer client.Close()

	stream, err := client.ProxyStream(ctx)
	if err != nil {
		log.Printf("proxy %s: open stream: %v", cmd.ProxyId, err)
		return
	}

	log.Printf("proxy %s: relaying 127.0.0.1:%d", cmd.ProxyId, cmd.Port)
	if err := agentproxy.Open(stream, cfg.ServerID, cfg.AgentToken, cmd.ProxyId, cmd.Port); err != nil {
		log.Printf("proxy %s: %v", cmd.ProxyId, err)
	}
}

func handleDeleteKey(cmd *pb.ServerCommand) {
	label := cmd.DeleteKey.Label
	keyPath := fmt.Sprintf("/root/.ssh/vantage_%s", strings.ReplaceAll(label, " ", "_"))

	if err := keys.RemoveSSHIdentity(keyPath); err != nil {
		log.Printf("remove ssh identity failed (cmd=%s): %v", cmd.CommandId, err)
	}

	for _, path := range []string{keyPath, keyPath + ".pub"} {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			log.Printf("delete key file %s (cmd=%s): %v", path, cmd.CommandId, err)
		}
	}
	log.Printf("deleted local key files for %q (cmd=%s)", label, cmd.CommandId)
}

func handleUpdateAgent(cmd *pb.ServerCommand) {
	if runtime.GOOS == "windows" {
		handleUpdateAgentWindows(cmd)
		return
	}

	u := cmd.UpdateAgent
	arch := runtime.GOARCH
	tag := "agent%2Fv" + u.Version
	binaryURL := fmt.Sprintf("%s/mrhid6/vantage/releases/download/%s/vantage-agent-linux-%s", u.GiteaBaseURL, tag, arch)
	checksumURL := fmt.Sprintf("%s/mrhid6/vantage/releases/download/%s/checksums.txt", u.GiteaBaseURL, tag)

	log.Printf("updating agent to v%s from %s (cmd=%s)", u.Version, u.GiteaBaseURL, cmd.CommandId)

	tmpBin := "/tmp/vantage-agent-update"
	if err := downloadFile(binaryURL, tmpBin); err != nil {
		log.Printf("update download failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}

	checksumData, err := httpGetBytes(checksumURL)
	if err != nil {
		log.Printf("update checksum fetch failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}
	if err := verifyChecksum(tmpBin, fmt.Sprintf("vantage-agent-linux-%s", arch), checksumData); err != nil {
		log.Printf("update checksum mismatch (cmd=%s): %v", cmd.CommandId, err)
		os.Remove(tmpBin)
		return
	}

	if err := os.Chmod(tmpBin, 0755); err != nil {
		log.Printf("update chmod failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}
	if err := os.Rename(tmpBin, "/usr/local/bin/vantage-agent"); err != nil {
		log.Printf("update replace binary failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}

	log.Printf("agent binary replaced, restarting service (cmd=%s)", cmd.CommandId)
	exec.Command("systemctl", "restart", "vantage-agent").Run()
}

func handleUpdateAgentWindows(cmd *pb.ServerCommand) {
	u := cmd.UpdateAgent
	tag := "agent%2Fv" + u.Version
	msiURL := fmt.Sprintf("%s/mrhid6/vantage/releases/download/%s/vantage-agent.msi", u.GiteaBaseURL, tag)
	checksumURL := fmt.Sprintf("%s/mrhid6/vantage/releases/download/%s/checksums-msi.txt", u.GiteaBaseURL, tag)

	log.Printf("updating agent to v%s from %s (cmd=%s)", u.Version, u.GiteaBaseURL, cmd.CommandId)

	msiPath := filepath.Join(os.TempDir(), "vantage-agent-update.msi")
	if err := downloadFile(msiURL, msiPath); err != nil {
		log.Printf("update download failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}

	checksumData, err := httpGetBytes(checksumURL)
	if err != nil {
		log.Printf("update checksum fetch failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}
	if err := verifyChecksum(msiPath, "vantage-agent.msi", checksumData); err != nil {
		log.Printf("update checksum mismatch (cmd=%s): %v", cmd.CommandId, err)
		os.Remove(msiPath)
		return
	}

	logPath := filepath.Join(os.TempDir(), "vantage-agent-msi.log")

	// The MSI stops the vantage-agent service as part of the upgrade. Anything
	// descended from this process is killed with it, so msiexec must not be a
	// child: run it from a scheduled task, which is parented to the Task
	// Scheduler service instead.
	if err := launchDetachedUpdate(msiPath, logPath, cmd.CommandId); err != nil {
		log.Printf("failed to launch msiexec (cmd=%s): %v", cmd.CommandId, err)
		return
	}
	log.Printf("scheduled msiexec for upgrade to v%s (cmd=%s)", u.Version, cmd.CommandId)
}

const updateTaskName = "VantageAgentUpdate"

func launchDetachedUpdate(msiPath, logPath, commandID string) error {
	scriptPath := filepath.Join(os.TempDir(), "vantage-agent-update.cmd")
	script := fmt.Sprintf("@echo off\r\n"+
		"timeout /t 5 /nobreak >nul\r\n"+
		"msiexec /i \"%s\" /qn /norestart /l*v \"%s\"\r\n"+
		"schtasks /delete /tn %s /f >nul 2>&1\r\n"+
		"del /f /q \"%s\" >nul 2>&1\r\n"+
		"(goto) 2>nul & del /f /q \"%%~f0\"\r\n",
		msiPath, logPath, updateTaskName, msiPath)
	if err := os.WriteFile(scriptPath, []byte(script), 0o600); err != nil {
		return fmt.Errorf("write update script: %w", err)
	}

	// Stale task from a previous attempt would make /create fail even with /f
	// if it is still running, so tear it down first and ignore the result.
	exec.Command("schtasks", "/end", "/tn", updateTaskName).Run()
	exec.Command("schtasks", "/delete", "/tn", updateTaskName, "/f").Run()

	create := exec.Command("schtasks", "/create",
		"/tn", updateTaskName,
		"/tr", `"`+scriptPath+`"`,
		"/sc", "once",
		// Already in the past: the task never fires on its own, only via /run.
		"/st", "00:00",
		"/ru", "SYSTEM",
		"/rl", "HIGHEST",
		"/f")
	if out, err := create.CombinedOutput(); err != nil {
		return fmt.Errorf("schtasks create: %v: %s", err, strings.TrimSpace(string(out)))
	}

	if out, err := exec.Command("schtasks", "/run", "/tn", updateTaskName).CombinedOutput(); err != nil {
		return fmt.Errorf("schtasks run: %v: %s", err, strings.TrimSpace(string(out)))
	}
	return nil
}

func downloadFile(url, dest string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("HTTP %d from %s", resp.StatusCode, url)
	}
	f, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(f, resp.Body)
	return err
}

func httpGetBytes(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP %d from %s", resp.StatusCode, url)
	}
	return io.ReadAll(resp.Body)
}

func verifyChecksum(filePath, filename string, checksumData []byte) error {
	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return err
	}
	actual := hex.EncodeToString(h.Sum(nil))

	for _, line := range strings.Split(string(checksumData), "\n") {
		fields := strings.Fields(line)
		if len(fields) == 2 && fields[1] == filename {
			if fields[0] != actual {
				return fmt.Errorf("expected %s got %s", fields[0], actual)
			}
			return nil
		}
	}
	return fmt.Errorf("no checksum entry found for %s", filename)
}

func handleGenerateKey(cfg *config.Config, cmd *pb.ServerCommand) {
	g := cmd.GenerateKey
	label := g.Label
	keyPath := fmt.Sprintf("/root/.ssh/vantage_%s", strings.ReplaceAll(label, " ", "_"))

	opts := keys.KeyGenOptions{
		KeyType:    g.KeyType,
		KeySize:    g.KeySize,
		Passphrase: g.Passphrase,
		Comment:    g.Comment,
	}
	pubKey, err := keys.GenerateKeyPair(keyPath, opts)
	if err != nil {
		log.Printf("key generation failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}

	privKeyData, err := os.ReadFile(keyPath)
	if err != nil {
		log.Printf("read private key failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}

	client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
	if err != nil {
		log.Printf("dial for key upload failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}
	defer client.Close()

	keyID, err := client.UploadGeneratedKey(cfg.ServerID, cfg.AgentToken, pubKey, string(privKeyData), label)
	if err != nil {
		log.Printf("key upload failed (cmd=%s): %v", cmd.CommandId, err)
		return
	}

	if err := keys.AddSSHIdentity(keyPath); err != nil {
		log.Printf("add ssh identity failed (cmd=%s): %v", cmd.CommandId, err)
	}
	log.Printf("generated and uploaded key %q (key_id=%s, cmd=%s)", label, keyID, cmd.CommandId)
}

func localIP() string {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return ""
	}
	for _, addr := range addrs {
		if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() {
			if ipNet.IP.To4() != nil {
				return ipNet.IP.String()
			}
		}
	}
	return ""
}

func GenerateAndUpload(cfg *config.Config, label string) error {
	client, err := grpcclient.New(cfg.ServerURL, cfg.TLS)
	if err != nil {
		return err
	}
	defer client.Close()

	keyPath := fmt.Sprintf("/root/.ssh/vantage_%s", strings.ReplaceAll(label, " ", "_"))
	pubKey, err := keys.GenerateKeyPair(keyPath, keys.KeyGenOptions{Comment: label})
	if err != nil {
		return err
	}

	privKeyData, err := os.ReadFile(keyPath)
	if err != nil {
		return fmt.Errorf("read private key: %w", err)
	}

	keyID, err := client.UploadGeneratedKey(cfg.ServerID, cfg.AgentToken, pubKey, string(privKeyData), label)
	if err != nil {
		return err
	}

	if err := keys.AddSSHIdentity(keyPath); err != nil {
		log.Printf("add ssh identity: %v", err)
	}
	log.Printf("uploaded generated key %s (key_id=%s)", label, keyID)
	return nil
}