From 89cc524f3f633419c683aa442c2850eb3bab529f Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Mon, 7 Sep 2026 14:29:03 +0000 Subject: [PATCH] docs: Document backup and restore The page leads with KEY_ENCRYPTION_KEY rather than mentioning it in a note, because holding a good database dump and no key is the way this goes wrong. --- CLAUDE.md | 70 ++++++- docsite/docs/operations/backup-and-restore.md | 193 ++++++++++++++++++ docsite/sidebars.ts | 2 +- 3 files changed, 262 insertions(+), 3 deletions(-) create mode 100644 docsite/docs/operations/backup-and-restore.md diff --git a/CLAUDE.md b/CLAUDE.md index 4e92fec..776bfe6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -563,6 +563,65 @@ inserted in front. The same setting also decides the address recorded in `UpdateAgentCmd` carries a target version and Gitea base URL; the agent downloads and replaces itself. +### Backup and restore + +`vantagectl` is a standalone Go module (`vantagectl/`), not a subcommand of +`server`. It needs its own module rather than living inside `server`'s for the +same reason `admin` and `sitesvc` already do: `server` imports the rest of +`server`'s dependency graph, and `spf13/cobra` has no business in a process +that also terminates gRPC streams and serves the REST API. More to the point, +`vantagectl` has to run when the control plane **does not** — a backup or +restore against a database with no server container alive at all — so it +cannot be a mode of the binary whose crash is the reason you need it. + +The actual logic lives in `shared/backup` (dump, restore, verify, manifest, +fingerprint), not in `vantagectl/internal/cmd`, which holds only argument +parsing and operator-facing output. That split is what lets `server` import +`shared/backup` later — a scheduled in-process backup, say — without a second +implementation to keep in sync. `shared/cryptobox` is the same move one layer +down: it is now the **single** AES-256-GCM implementation, and +`server/internal/services/crypto.go` delegates to it rather than keeping its +own copy that `shared/backup` would otherwise have had to duplicate to decrypt +a probe value during `verify`. + +**The archive stores a SHA-256 fingerprint of `KEY_ENCRYPTION_KEY`, never the +key.** `backup` refuses to run without the key set in the environment unless +`--allow-no-key` is passed, because an archive with no fingerprint at all +cannot later tell a restore that the wrong key is in hand — it can only find +that out when the data comes back as noise. The fingerprint is what turns that +failure into a refusal at `restore` time instead. + +**Collections are enumerated live** — `shared/backup` lists what the database +actually holds rather than reading `services.ScopedCollections`, the opposite +choice from the one instance-deletion purge makes. Purge must never miss a +tenant-scoped collection, so it keeps one hand-maintained registry; a backup +must never miss **any** collection, tenant-scoped or not (`migrations`, +`vulndb_meta`), so a static list is the wrong shape twice over — once for the +collections it would still owe `instance_id` deletion but not a backup, and +once for the two singleton collections that carry neither `instance_id` nor a +release note. + +**Restore refuses a non-empty target database and has no merge semantics.** +There is no code path that upserts an archive's documents over existing ones: +merging two control planes' data reconciles nothing about which SSH keys are +still valid or which users still exist, and an upsert would resurrect a +revoked key or a deleted member from the older side. `--force` drops each +collection in the archive first, and is gated behind a typed confirmation +(the target database's name, typed back) on a terminal, or `--confirm-db NAME` +matching the target exactly with none. Naming the target in the command itself +means a copied command carries its intended target with it and cannot destroy +a different one by accident. + +**`vantagectl/Dockerfile`'s runtime stage is `scratch`, and needs the same +explicit `/tmp` as `server/Dockerfile`.** `restore` extracts an archive to a +temporary directory before verifying its checksums, and a scratch image has no +`/tmp` for `os.MkdirTemp` to find — the same failure mode `vulnsched` hits on +`server`, but here it would break every restore rather than only vulnerability +scanning. + +**`shared/` now fans out to four Go images** in `server-deploy.yml`: +`server`, `sitesvc`, `admin` and `vantagectl` — see the CI section below. + ### API tokens and OpenAPI A token is `vt_` plus 32 random bytes hex, shown once at creation and stored @@ -1221,7 +1280,7 @@ GOOS=linux GOARCH=amd64 go build \ ### `server-deploy.yml` — triggered on every push to `main` -Builds and pushes seven images to the Gitea container registry: `server`, `web`, `site`, `sitesvc`, `admin`, `adminsite` and `docsite`. +Builds and pushes eight images to the Gitea container registry: `server`, `web`, `site`, `sitesvc`, `admin`, `adminsite`, `docsite` and `vantagectl`. Note that despite the name, **this workflow does not deploy** — it only builds and pushes. There is no SSH step. Rolling images out is a separate manual step on the host: @@ -1237,9 +1296,16 @@ cd /opt/vantage && docker compose -f docker-compose.yml -f docker-compose.site.y | `server` | `server/`, `shared/`, `proto/`, `go.work` | | `admin` | `admin/`, `shared/`, `go.work` | | `sitesvc` | `sitesvc/`, `shared/`, `go.work` | +| `vantagectl` | `vantagectl/`, `shared/`, `go.work` | | `web` · `site` · `adminsite` · `docsite` | their own directory only | -`shared/` fans out to all three Go images because each of their Dockerfiles copies `shared/` from a root context — **if a fourth service ever imports `shared/`, add it to that list or it will ship stale**. A change to the workflow file rebuilds everything, since a build arg is baked into the image. So does anything that leaves no trustworthy base commit: a manual `workflow_dispatch`, a new branch, or a force-push whose old head is gone. +`shared/` fans out to **four** Go images (`server`, `sitesvc`, `admin`, +`vantagectl`) because each of their Dockerfiles copies `shared/` from a root +context — **if a fifth service ever imports `shared/`, add it to that list or +it will ship stale**. A change to the workflow file rebuilds everything, since +a build arg is baked into the image. So does anything that leaves no +trustworthy base commit: a manual `workflow_dispatch`, a new branch, or a +force-push whose old head is gone. The gap this leaves: **changing a repo variable pushes no commit, so nothing rebuilds.** After editing `ADMIN_API_URL`, `HQ_URL` or `ADMIN_ENV`, run the workflow manually — that is what `workflow_dispatch` is there for. Base images also stop being refreshed on a service nobody touches; a periodic manual run covers that. diff --git a/docsite/docs/operations/backup-and-restore.md b/docsite/docs/operations/backup-and-restore.md new file mode 100644 index 0000000..b319155 --- /dev/null +++ b/docsite/docs/operations/backup-and-restore.md @@ -0,0 +1,193 @@ +--- +id: backup-and-restore +title: Backup and restore +sidebar_label: Backup and restore +--- + +`vantagectl` is a separate command-line tool that backs up and restores the +MongoDB database behind a Vantage control plane. It talks to MongoDB directly, +never to the Vantage API, so it works against a control plane that is down, +half-migrated, or gone — exactly the situation a backup tool has to survive. + +:::danger The key comes first +Vantage encrypts SSH private keys, key passphrases, vault secrets, SSO client +secrets and console credentials with `KEY_ENCRYPTION_KEY`. **It is not in your +backup, and it is not recoverable.** A database restored without it is +permanently unreadable — not degraded, not partially readable, unreadable. + +Store it wherever you store the credentials you could not rebuild: a password +manager, a secrets vault outside this control plane, a piece of paper in a +safe. Anywhere but next to the archive. +::: + +## What a backup holds + +Every collection in the database, the index definitions each one needs to be +useful again, and a SHA-256 **fingerprint** of `KEY_ENCRYPTION_KEY` — never the +key itself. The fingerprint is what lets a later `restore` or `verify` tell you +that the key you are holding is the wrong one, before it writes a database +nobody can read. + +## What it does not hold + +- **Redis sessions.** Everyone signs in again after a restore, which is already + true whenever Redis itself restarts. +- **The vulnerability database.** It is re-pulled automatically on next boot. +- **Agent state on managed servers.** Nothing needs re-enrolling: agents + reconnect on their own, because `servers.agent_token_hash` — the thing an + agent authenticates with — is itself in the backup. + +## Taking a backup + +The loose binary: + +```bash +export MONGO_URI=mongodb://localhost:27017 +export MONGO_DB=vantage +export KEY_ENCRYPTION_KEY= +vantagectl backup --out /backups +``` + +The container: + +```bash +docker run --rm \ + -e MONGO_URI=mongodb://mongo:27017 \ + -e MONGO_DB=vantage \ + -e KEY_ENCRYPTION_KEY= \ + -v /backups:/backups \ + gitea.hostxtra.co.uk/mrhid6/vantage/vantagectl:latest backup --out /backups +``` + +Kubernetes, as a scheduled `CronJob` the Helm chart can render for you: + +```yaml +backup: + enabled: true + schedule: "0 2 * * *" + image: "gitea.hostxtra.co.uk/mrhid6/vantage/vantagectl:latest" + pvcName: "vantage-backups" +``` + +`backup.enabled` defaults to `false`, and the chart refuses to render if it is +turned on without both `backup.image` and `backup.pvcName` — a backup needs a +known image and somewhere durable to land, and guessing at either is worse than +refusing to start. `backup.exclude` names collections to leave out (recorded in +the archive's manifest, so an archive never claims to be complete when it is +not), and `backup.successfulJobsHistoryLimit` / `backup.failedJobsHistoryLimit` +/ `backup.resources` behave exactly as they do on any other `CronJob`. + +`backup` refuses to run without `KEY_ENCRYPTION_KEY` set in the environment, +unless you pass `--allow-no-key` — for a deployment that genuinely stores no +encrypted data. Everywhere else, treat the refusal as the tool doing its job. + +## Where to put the archive + +`--out -` streams the tarball to stdout instead of writing a file, and every +line of progress output goes to stderr — so piping the archive into something +else is always safe, nothing progress-related lands in the stream. + +Into `restic`: + +```bash +vantagectl backup --out - | restic backup --stdin --stdin-filename vantage.tar.gz +``` + +Into S3: + +```bash +vantagectl backup --out - | aws s3 cp - s3://my-backups/vantage-$(date +%F).tar.gz +``` + +An archive is as sensitive as a raw database dump — it carries every SSH key +assignment, every secret group, every session-adjacent setting, in a form the +right `KEY_ENCRYPTION_KEY` can decrypt. Whatever you pipe it into should +encrypt it at rest; `vantagectl` itself does not. + +## Checking a backup is real + +```bash +vantagectl verify /backups/vantage-backup-vantage-20260907T020000Z.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage +``` + +Each line of output answers a different question: + +- **`Archive`** — every member's checksum still matches; the tarball has not + been truncated or corrupted. +- **`Archive key`** / **`Your key`** — the fingerprint stored in the archive + next to the fingerprint of the `KEY_ENCRYPTION_KEY` in your environment. +- **`Key match`** — whether those two fingerprints agree. +- **`Live probe`** — given `--mongo-uri`, `verify` goes one step further and + decrypts a real ciphertext value from that database with the key you hold. + A fingerprint match proves two archives agree about a key; only the probe + proves the key in your hand actually reads the data. + +`verify` exits non-zero the moment anything above is wrong, which is what makes +it worth putting on a schedule — a backup job that "succeeded" last night is +not the same claim as a backup that will actually restore. + +## Restoring + +`restore` expects the target database to be empty. Pointed at one that already +holds data, it refuses outright: there are no merge semantics, because merging +two control planes reconciles nothing and upserting old data over new would +resurrect revoked keys and deleted users. + +```bash +vantagectl restore /backups/vantage-backup-vantage-20260907T020000Z.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage_restore +``` + +To overwrite a database that is not empty, add `--force`, which drops each +collection named in the archive before loading it. On a terminal, `--force` +alone prompts you to type the target database's name back — a deliberate pause +before something destructive. With no terminal — a Kubernetes Job, a CI step, a +cron entry — that prompt cannot happen, so `--force` instead requires +`--confirm-db NAME` naming the target exactly; a mismatch is refused. Naming +the database in the command itself means a copy-pasted invocation carries its +intended target with it and cannot destroy a different one by accident. + +`restore` also refuses when the archive's key fingerprint does not match the +`KEY_ENCRYPTION_KEY` in your environment — see "When the key is wrong" below. + +## The restore drill + +An untested backup is a hypothesis, not a backup. Rehearse the whole path, +monthly: + +1. Restore last night's archive into a scratch database: + ```bash + vantagectl restore /backups/vantage-backup-vantage-.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage_drill + ``` +2. Run `verify` against the result to confirm the data that landed is actually + readable with your current key: + ```bash + vantagectl verify /backups/vantage-backup-vantage-.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage_drill + ``` +3. Drop the scratch database. It served its purpose. + +The failure this catches is not "the archive is corrupt" — `verify` alone +catches that. It is "the archive is fine but nobody can actually stand a +control plane back up from it," which only a real restore proves. + +## When the key is wrong + +If `restore` finds the archive's key fingerprint does not match the +`KEY_ENCRYPTION_KEY` you are running with, it stops. Passing +`--ignore-key-mismatch` proceeds anyway, but says plainly which collections +will come back with ciphertext nobody can read: + +- `keys` — SSH private keys and passphrases +- `secrets` — the vault +- `auth_providers` — OIDC/SSO client secrets +- `console_sessions` — RDP/VNC credentials +- `settings` — anything encrypted at the instance level + +There is no way to recover that ciphertext afterwards. If you have reached +this point, the right key was lost along with the chance to read those rows — +the fix is to re-enter each of them by hand (re-upload SSH keys, re-save vault +secrets, reconfigure SSO), not to keep searching for a way to decrypt what is +already in the database. diff --git a/docsite/sidebars.ts b/docsite/sidebars.ts index 925e5a9..0cab3d0 100644 --- a/docsite/sidebars.ts +++ b/docsite/sidebars.ts @@ -49,7 +49,7 @@ const sidebars: SidebarsConfig = { { type: "category", label: "Operations", - items: ["operations/upgrading", "operations/backups", "operations/agent-updates"], + items: ["operations/upgrading", "operations/backups", "operations/backup-and-restore", "operations/agent-updates"], }, ], };