From 54671a36524694c2fbccfa2f08efdbff47b75414 Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Mon, 7 Sep 2026 14:29:03 +0000 Subject: [PATCH] docs: Document backup and restore The page leads with KEY_ENCRYPTION_KEY rather than mentioning it in a note, because holding a good database dump and no key is the way this goes wrong. --- docs/operations/backup-and-restore.md | 193 ++++++++++++++++++++++++++ sidebars.ts | 2 +- 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 docs/operations/backup-and-restore.md diff --git a/docs/operations/backup-and-restore.md b/docs/operations/backup-and-restore.md new file mode 100644 index 0000000..b319155 --- /dev/null +++ b/docs/operations/backup-and-restore.md @@ -0,0 +1,193 @@ +--- +id: backup-and-restore +title: Backup and restore +sidebar_label: Backup and restore +--- + +`vantagectl` is a separate command-line tool that backs up and restores the +MongoDB database behind a Vantage control plane. It talks to MongoDB directly, +never to the Vantage API, so it works against a control plane that is down, +half-migrated, or gone — exactly the situation a backup tool has to survive. + +:::danger The key comes first +Vantage encrypts SSH private keys, key passphrases, vault secrets, SSO client +secrets and console credentials with `KEY_ENCRYPTION_KEY`. **It is not in your +backup, and it is not recoverable.** A database restored without it is +permanently unreadable — not degraded, not partially readable, unreadable. + +Store it wherever you store the credentials you could not rebuild: a password +manager, a secrets vault outside this control plane, a piece of paper in a +safe. Anywhere but next to the archive. +::: + +## What a backup holds + +Every collection in the database, the index definitions each one needs to be +useful again, and a SHA-256 **fingerprint** of `KEY_ENCRYPTION_KEY` — never the +key itself. The fingerprint is what lets a later `restore` or `verify` tell you +that the key you are holding is the wrong one, before it writes a database +nobody can read. + +## What it does not hold + +- **Redis sessions.** Everyone signs in again after a restore, which is already + true whenever Redis itself restarts. +- **The vulnerability database.** It is re-pulled automatically on next boot. +- **Agent state on managed servers.** Nothing needs re-enrolling: agents + reconnect on their own, because `servers.agent_token_hash` — the thing an + agent authenticates with — is itself in the backup. + +## Taking a backup + +The loose binary: + +```bash +export MONGO_URI=mongodb://localhost:27017 +export MONGO_DB=vantage +export KEY_ENCRYPTION_KEY= +vantagectl backup --out /backups +``` + +The container: + +```bash +docker run --rm \ + -e MONGO_URI=mongodb://mongo:27017 \ + -e MONGO_DB=vantage \ + -e KEY_ENCRYPTION_KEY= \ + -v /backups:/backups \ + gitea.hostxtra.co.uk/mrhid6/vantage/vantagectl:latest backup --out /backups +``` + +Kubernetes, as a scheduled `CronJob` the Helm chart can render for you: + +```yaml +backup: + enabled: true + schedule: "0 2 * * *" + image: "gitea.hostxtra.co.uk/mrhid6/vantage/vantagectl:latest" + pvcName: "vantage-backups" +``` + +`backup.enabled` defaults to `false`, and the chart refuses to render if it is +turned on without both `backup.image` and `backup.pvcName` — a backup needs a +known image and somewhere durable to land, and guessing at either is worse than +refusing to start. `backup.exclude` names collections to leave out (recorded in +the archive's manifest, so an archive never claims to be complete when it is +not), and `backup.successfulJobsHistoryLimit` / `backup.failedJobsHistoryLimit` +/ `backup.resources` behave exactly as they do on any other `CronJob`. + +`backup` refuses to run without `KEY_ENCRYPTION_KEY` set in the environment, +unless you pass `--allow-no-key` — for a deployment that genuinely stores no +encrypted data. Everywhere else, treat the refusal as the tool doing its job. + +## Where to put the archive + +`--out -` streams the tarball to stdout instead of writing a file, and every +line of progress output goes to stderr — so piping the archive into something +else is always safe, nothing progress-related lands in the stream. + +Into `restic`: + +```bash +vantagectl backup --out - | restic backup --stdin --stdin-filename vantage.tar.gz +``` + +Into S3: + +```bash +vantagectl backup --out - | aws s3 cp - s3://my-backups/vantage-$(date +%F).tar.gz +``` + +An archive is as sensitive as a raw database dump — it carries every SSH key +assignment, every secret group, every session-adjacent setting, in a form the +right `KEY_ENCRYPTION_KEY` can decrypt. Whatever you pipe it into should +encrypt it at rest; `vantagectl` itself does not. + +## Checking a backup is real + +```bash +vantagectl verify /backups/vantage-backup-vantage-20260907T020000Z.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage +``` + +Each line of output answers a different question: + +- **`Archive`** — every member's checksum still matches; the tarball has not + been truncated or corrupted. +- **`Archive key`** / **`Your key`** — the fingerprint stored in the archive + next to the fingerprint of the `KEY_ENCRYPTION_KEY` in your environment. +- **`Key match`** — whether those two fingerprints agree. +- **`Live probe`** — given `--mongo-uri`, `verify` goes one step further and + decrypts a real ciphertext value from that database with the key you hold. + A fingerprint match proves two archives agree about a key; only the probe + proves the key in your hand actually reads the data. + +`verify` exits non-zero the moment anything above is wrong, which is what makes +it worth putting on a schedule — a backup job that "succeeded" last night is +not the same claim as a backup that will actually restore. + +## Restoring + +`restore` expects the target database to be empty. Pointed at one that already +holds data, it refuses outright: there are no merge semantics, because merging +two control planes reconciles nothing and upserting old data over new would +resurrect revoked keys and deleted users. + +```bash +vantagectl restore /backups/vantage-backup-vantage-20260907T020000Z.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage_restore +``` + +To overwrite a database that is not empty, add `--force`, which drops each +collection named in the archive before loading it. On a terminal, `--force` +alone prompts you to type the target database's name back — a deliberate pause +before something destructive. With no terminal — a Kubernetes Job, a CI step, a +cron entry — that prompt cannot happen, so `--force` instead requires +`--confirm-db NAME` naming the target exactly; a mismatch is refused. Naming +the database in the command itself means a copy-pasted invocation carries its +intended target with it and cannot destroy a different one by accident. + +`restore` also refuses when the archive's key fingerprint does not match the +`KEY_ENCRYPTION_KEY` in your environment — see "When the key is wrong" below. + +## The restore drill + +An untested backup is a hypothesis, not a backup. Rehearse the whole path, +monthly: + +1. Restore last night's archive into a scratch database: + ```bash + vantagectl restore /backups/vantage-backup-vantage-.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage_drill + ``` +2. Run `verify` against the result to confirm the data that landed is actually + readable with your current key: + ```bash + vantagectl verify /backups/vantage-backup-vantage-.tar.gz \ + --mongo-uri mongodb://localhost:27017 --db vantage_drill + ``` +3. Drop the scratch database. It served its purpose. + +The failure this catches is not "the archive is corrupt" — `verify` alone +catches that. It is "the archive is fine but nobody can actually stand a +control plane back up from it," which only a real restore proves. + +## When the key is wrong + +If `restore` finds the archive's key fingerprint does not match the +`KEY_ENCRYPTION_KEY` you are running with, it stops. Passing +`--ignore-key-mismatch` proceeds anyway, but says plainly which collections +will come back with ciphertext nobody can read: + +- `keys` — SSH private keys and passphrases +- `secrets` — the vault +- `auth_providers` — OIDC/SSO client secrets +- `console_sessions` — RDP/VNC credentials +- `settings` — anything encrypted at the instance level + +There is no way to recover that ciphertext afterwards. If you have reached +this point, the right key was lost along with the chance to read those rows — +the fix is to re-enter each of them by hand (re-upload SSH keys, re-save vault +secrets, reconfigure SSO), not to keep searching for a way to decrypt what is +already in the database. diff --git a/sidebars.ts b/sidebars.ts index 925e5a9..0cab3d0 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -49,7 +49,7 @@ const sidebars: SidebarsConfig = { { type: "category", label: "Operations", - items: ["operations/upgrading", "operations/backups", "operations/agent-updates"], + items: ["operations/upgrading", "operations/backups", "operations/backup-and-restore", "operations/agent-updates"], }, ], };