From 1c6d9e84955f8d23a762993ff20bae1cae60f6c6 Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Tue, 8 Sep 2026 08:44:47 +0000 Subject: [PATCH] refactor: move proto/ to vantage-shared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vantage.proto documents the hand-written types in shared/grpc/pb, and nothing compiles it. Keeping it in a different repository from the Go types it describes meant the one rule holding them together — add the message to both in the same commit — could not be followed at all. server's rebuild trigger loses proto/, which it only carried as insurance against exactly that split. --- .gitea/workflows/server-deploy.yml | 10 +- CLAUDE.md | 38 +-- proto/vantage/v1/vantage.proto | 392 ----------------------------- 3 files changed, 25 insertions(+), 415 deletions(-) delete mode 100644 proto/vantage/v1/vantage.proto diff --git a/.gitea/workflows/server-deploy.yml b/.gitea/workflows/server-deploy.yml index 0cedc91..e1599ab 100644 --- a/.gitea/workflows/server-deploy.yml +++ b/.gitea/workflows/server-deploy.yml @@ -80,11 +80,11 @@ jobs: # to three images, and no longer a way to ship a service # against a shared/ it was never built with. # - # proto/ stays in server's list as insurance: the hand-written - # pb now lives in vantage-shared, but a proto change made here - # in the same push should not depend on the pin being bumped - # in that same push to trigger a rebuild. - flag server '^(server/|proto/|default_steps/|go\.work)' + # proto/ is gone too. It documents the wire types, and it + # moved to sit beside the hand-written pb it describes, so + # that a message added to one is added to the other in the + # same commit. Nothing here reads it. + flag server '^(server/|default_steps/|go\.work)' # web/ uses its own directory as the build context, so # nothing outside it can affect it. It is the only front end diff --git a/CLAUDE.md b/CLAUDE.md index d2742b2..26ad1e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,7 +68,6 @@ vantage/ │ ├── app/login, app/setup # unauthed routes │ ├── components/ # ui/, workflows/, monitors/, Sidebar │ └── lib/ # api client, guac console, query client -├── proto/vantage/v1/vantage.proto ├── installer/ # Windows: setup.ps1, nssm.exe, WiX .wxs ├── deploy/ # docker-compose.yml, agent.service └── .gitea/workflows/ # agent-release.yml, server-deploy.yml @@ -101,7 +100,8 @@ The rest lives in that repository's own CLAUDE.md. email system: transport plus templates), `license/` (payload, sign, verify, trusted keys, plans), `models/` (Instance, User, Settings), `provision/`, `backup/`, `cryptobox/`, `indexes/`, `grpc/pb` + `grpc/codec`, and -`cmd/lkctl/`. Three modules here depend on it — `server`, `agent` and +`cmd/lkctl/`, and `proto/vantage/v1/vantage.proto`, which documents `grpc/pb` +and moved there to sit beside it. Three modules here depend on it — `server`, `agent` and `vantagectl` — each pinning a version in its own `go.mod`, as do `vantage-admin` and `vantage-site`. It was a directory in this repository until it was extracted with its history; the @@ -470,25 +470,27 @@ container's stdout is arbitrary and cannot be masked the way a workflow's can. is in `ScopedCollections` (which `scopedCollectionsForPurge` derives from). There is no history: a workload list is state, not a record. -**`proto/vantage/v1/vantage.proto` is documentation, not a generator input.** -`shared/grpc/pb` is hand-written JSON-tagged structs over the custom codec in -`shared/grpc/codec`, and it is **one** package shared by both sides — a message -added to it must be added to the `.proto` in the same commit, but there is no -longer a second Go copy to keep in step. There used to be two +**The wire contract is not in this repository at all.** `shared/grpc/pb` is +hand-written JSON-tagged structs over the custom codec in `shared/grpc/codec`, +and `proto/vantage/v1/vantage.proto` is documentation of them rather than a +generator input — nothing compiles it. Both live in `vantage-shared`, together, +because that co-location is the only thing making "add the message to both in +the same commit" possible. + +It is **one** `pb` package serving both sides. There used to be two (`agent/internal/grpc/pb` and `server/internal/grpc/pb`) and they had already drifted: the agent's `UnimplementedVantageServer` was three methods stale and carried no `ReportWorkloads` at all. The agent links the server half as dead code, which the linker drops. -This makes `agent` the **fifth** consumer of `shared/`, and the wire contract now -lives outside this repository entirely: a message added to `vantage-shared` is -not a message either side has until its pin is bumped. What that buys is that -the mismatch is a compile error rather than two copies that both compiled and -disagreed on the wire. What it costs is ordering — a wire change needs a -`vantage-shared` release, then a pin bump in `server/` (live at the next push to -main) and a pin bump in `agent/` (live at the next `agent/v*` tag). The -server-ahead-of-fleet gap existed before too; it is now explicit in two `go.mod` -files instead of implicit in a shared directory. +A message added to `vantage-shared` is not a message either side has until its +pin is bumped. What that buys is a mismatch that is a compile error rather than +two copies that both compiled and disagreed on the wire. What it costs is +ordering — **a wire change is three steps**: release `vantage-shared`, bump the +pin in `server/` (live at the next push to main), bump the pin in `agent/` (live +only at the next `agent/v*` tag). The control plane runs ahead of the fleet in +between, which was true before too; it is now explicit in two `go.mod` files +rather than implicit in a shared directory. ### Status pages @@ -943,7 +945,7 @@ keeps succeeding and the fleet list still shows the server `active`. The agent's watchdog arms only **after** it has seen a first ping, so an older server that sends none is treated as working rather than put into a reconnect loop. -Key-state polling stays on the 30s `SyncKeys` interval. Full message definitions live in `proto/vantage/v1/vantage.proto`. +Key-state polling stays on the 30s `SyncKeys` interval. Full message definitions live in `vantage-shared`, in `proto/vantage/v1/vantage.proto` beside the `grpc/pb` types it describes. --- @@ -1341,7 +1343,7 @@ cd /opt/vantage && docker compose -f deploy/docker/docker-compose.yml pull && \ | Image | Rebuilds when | | ---------------------------- | -------------------------------- | -| `server` | `server/`, `proto/`, `go.work` | +| `server` | `server/`, `go.work` | | `web` | `web/` only | **No path in this table names `shared/` any more**, and no fan-out rule replaces diff --git a/proto/vantage/v1/vantage.proto b/proto/vantage/v1/vantage.proto deleted file mode 100644 index c0a2822..0000000 --- a/proto/vantage/v1/vantage.proto +++ /dev/null @@ -1,392 +0,0 @@ -syntax = "proto3"; - -package vantage.v1; - -option go_package="gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/grpc/pb"; - -service Vantage { - rpc Register(RegisterRequest) returns (RegisterResponse); - rpc SyncKeys(SyncRequest) returns (SyncResponse); - rpc UploadGeneratedKey(UploadKeyRequest) returns (UploadKeyResponse); - rpc ReportUpdates(ReportUpdatesRequest) returns (ReportUpdatesResponse); - rpc ReportPackages(ReportPackagesRequest) returns (ReportPackagesResponse); - rpc ReportWorkloads(ReportWorkloadsRequest) returns (ReportWorkloadsResponse); - rpc ReportInventory(InventoryReport) returns (InventoryReportResponse); - rpc SyncMonitors(SyncMonitorsRequest) returns (SyncMonitorsResponse); - rpc ReportChecks(ReportChecksRequest) returns (ReportChecksResponse); - // Bidirectional stream: agent sends auth once, server pushes commands. - rpc CommandStream(stream AgentMessage) returns (stream ServerCommand); - rpc ProxyStream(stream ProxyClientMsg) returns (stream ProxyServerMsg); -} - -message RegisterRequest { - string server_id = 1; - string pre_reg_token = 2; - string hostname = 3; - string ip_address = 4; - string os_info = 5; -} - -message RegisterResponse { - string agent_token = 1; -} - -message SyncRequest { - string server_id = 1; - string agent_token = 2; - string agent_version = 3; -} - -message SyncResponse { - repeated string public_keys = 1; - - // collect_packages tells the agent whether this instance's licence grants - // vulnerability scanning. False means do not collect at all: no gRPC body, - // no document, no storage. The server re-checks on ReportPackages — this - // flag is the optimisation, the server check is the boundary. - // - // Absent reads as false, which is the safe direction: an old server that - // does not send it leaves agents collecting nothing. - bool collect_packages = 2; -} - -// ReportPackages carries a server's installed package set. -// -// The agent calls twice at most. The first call sends only the hash; if the -// server already holds that hash it answers need_full = false and the ~150KB -// body is never sent. A machine's package set changes rarely, so almost every -// hour costs one small message. -message ReportPackagesRequest { - string server_id = 1; - string agent_token = 2; - string hash = 3; - OSRelease os = 4; - repeated InstalledPackage packages = 5; // empty on the offer call -} - -message ReportPackagesResponse { - bool need_full = 1; -} - -message OSRelease { - string family = 1; - // version_id is not optional: Ubuntu 22.04 and 24.04 publish different fixed - // versions for the same CVE, so a scan without it is guesswork. - string version_id = 2; - string arch = 3; -} - -message InstalledPackage { - string name = 1; - string version = 2; - int32 epoch = 3; - string arch = 4; - // source_name is what the Debian and Ubuntu feeds are keyed on: one advisory - // against "openssl" covers libssl3, openssl and libssl-dev. - string source_name = 5; -} - -message UploadKeyRequest { - string server_id = 1; - string agent_token = 2; - string public_key = 3; - string label = 4; - string private_key = 5; -} - -message UploadKeyResponse { - string key_id = 1; -} - -// CommandStream messages - -message AgentMessage { - string server_id = 1; - string agent_token = 2; - oneof payload { - AgentReady ready = 3; - CommandResult result = 4; - StepResult step_result = 5; - StepOutputChunk step_output = 6; - WorkloadLogsResult workload_logs_result = 7; - } -} - -message AgentReady { - -} - -message CommandResult { - string command_id = 1; - bool success = 2; - string message = 3; -} - -message PackageUpdate { - string name = 1; - string current_version = 2; - string new_version = 3; -} - -message ReportUpdatesRequest { - string server_id = 1; - string agent_token = 2; - repeated PackageUpdate updates = 3; -} - -message ReportUpdatesResponse { - -} - -message CPUReport { - string model = 1; - int32 cores = 2; - double usage_pct = 3; - double load1 = 4; -} - -message MemReport { - uint64 total_bytes = 1; - uint64 used_bytes = 2; -} - -message PartitionReport { - string device = 1; - string mountpoint = 2; - string fstype = 3; - uint64 total_bytes = 4; - uint64 used_bytes = 5; -} - -message InventoryReport { - string server_id = 1; - string agent_token = 2; - bool include_static = 3; - CPUReport cpu = 4; - MemReport memory = 5; - uint64 swap_total = 6; - uint64 swap_used = 7; - repeated PartitionReport partitions = 8; - string kernel = 9; - // Set on static snapshots only. The agent never reboots; it reports that one - // is owed and leaves the decision to a person or a workflow. - bool reboot_required = 10; -} - -message InventoryReportResponse { - -} - -message MonitorSpec { - string monitor_id = 1; - string type = 2; - string url = 3; - string host = 4; - int32 port = 5; - string method = 6; - int32 expected_status = 7; - string keyword = 8; - int32 tls_warn_days = 9; - int32 interval_sec = 10; - int32 retries = 11; - bool insecure = 12; -} - -message SyncMonitorsRequest { - string server_id = 1; - string agent_token = 2; -} - -message SyncMonitorsResponse { - repeated MonitorSpec monitors = 1; -} - -message CheckResult { - string monitor_id = 1; - bool up = 2; - int32 latency_ms = 3; - string message = 4; - int64 cert_expiry_unix = 5; -} - -message ReportChecksRequest { - string server_id = 1; - string agent_token = 2; - repeated CheckResult results = 3; -} - -message ReportChecksResponse { - -} - -message ApplyUpdatesCmd { - -} - -message ServerCommand { - string command_id = 1; - oneof command { - GenerateKeyCmd generate_key = 2; - DeleteKeyCmd delete_key = 3; - UpdateAgentCmd update_agent = 4; - ApplyUpdatesCmd apply_updates = 5; - RunStepCmd run_step = 6; - CleanupWorkspaceCmd cleanup_workspace = 7; - OpenProxyCmd open_proxy = 8; - PingCmd ping = 9; - RefreshWorkloadsCmd refresh_workloads = 10; - ControlWorkloadCmd control_workload = 11; - WorkloadLogsCmd workload_logs = 12; - } -} - -// PingCmd is a liveness beat, carrying nothing and requiring no reply. -// -// It exists because gRPC keepalive cannot prove what the agent needs to know. -// Behind an L7 proxy the agent's HTTP/2 connection terminates at the proxy, so -// keepalive pings are answered by the proxy whether or not the server behind it -// is still there. A pod that dies leaves the agent blocked in Recv on a stream -// that will never produce another message and never error — commands are -// dispatched into it and silently lost. Only traffic that originates at the -// server itself distinguishes a live stream from an orphaned one. -message PingCmd { -} - -// CleanupWorkspaceCmd tells the agent to recursively remove the run's working -// directory once all steps on that server have finished. -message CleanupWorkspaceCmd { - string workspace_id = 1; -} - -message DeleteKeyCmd { - string label = 1; -} - -message UpdateAgentCmd { - string version = 1; // e.g. "1.2.3" - string gitea_base_url = 2; // e.g. "https://gitea.example.com" -} - -message GenerateKeyCmd { - string label = 1; - string key_type = 2; // ed25519 | rsa | ecdsa (default: ed25519) - int32 key_size = 3; // bits; used for rsa and ecdsa - string passphrase = 4; // empty = no passphrase - string comment = 5; // embedded in public key -} - -message RunStepCmd { - string interpreter = 1; // "bash" | "powershell" - string script = 2; - map env = 3; - int32 timeout_seconds = 4; - string workspace_id = 5; -} - -message StepResult { - string command_id = 1; - int32 exit_code = 2; - string stdout = 3; - string stderr = 4; - map output_env = 5; -} - -message StepOutputChunk { - string command_id = 1; - uint64 seq = 2; - bytes data = 3; - bool eof = 4; -} - -// OpenProxyCmd tells the agent to dial 127.0.0.1:port locally and relay that -// connection back over a fresh ProxyStream identified by proxy_id. -message OpenProxyCmd { - string proxy_id = 1; - uint32 port = 2; -} - -message ProxyOpen { - string server_id = 1; - string agent_token = 2; - string proxy_id = 3; -} - -message ProxyClose { string reason = 1; } - -message ProxyClientMsg { - oneof payload { - ProxyOpen open = 1; // first message only - bytes data = 2; - ProxyClose close = 3; - } -} - -message ProxyServerMsg { - oneof payload { - bytes data = 1; - ProxyClose close = 2; - } -} - -// --------------------------------------------------------------------------- -// Workload registry - -// ReportWorkloads carries what a server is running. -// -// Offer-then-send, the same handshake as ReportPackages: the agent calls once -// with workloads empty, and resends with the body only if need_full is set. -message ReportWorkloadsRequest { - string server_id = 1; - string agent_token = 2; - string hash = 3; - bool docker_ok = 4; - string docker_error = 5; - bool systemd_ok = 6; - string systemd_error = 7; - repeated Workload workloads = 8; // empty on the offer call - // full marks the second call. It is not inferred from an empty workloads - // list: a host running nothing sends an empty list as its full report. - bool full = 9; -} - -message ReportWorkloadsResponse { - bool need_full = 1; -} - -message Workload { - string kind = 1; // "container" | "unit" - string id = 2; - string name = 3; - string state = 4; - string health = 5; - string image = 6; - string stack = 7; - repeated string ports = 8; - int32 restarts = 9; - string started_at = 10; // RFC3339, empty when not running - bool protected = 11; -} - -// RefreshWorkloadsCmd carries no payload back. It makes the agent report -// immediately through ReportWorkloads, so there is exactly one writer for the -// server_workloads collection rather than two arriving by different routes. -message RefreshWorkloadsCmd {} - -message ControlWorkloadCmd { - string kind = 1; - string id = 2; - string action = 3; // "start" | "stop" | "restart" -} - -message WorkloadLogsCmd { - string kind = 1; - string id = 2; - int32 tail = 3; -} - -message WorkloadLogsResult { - string command_id = 1; - string text = 2; - bool truncated = 3; - string error = 4; -}