From 8908b435a047e9d95fa629fccf6d799cca5d4900 Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Thu, 17 Sep 2026 09:16:53 +0000 Subject: [PATCH] docs(monitors): document heartbeat and metric monitors; add heartbeat e2e --- CLAUDE.md | 16 +++++++++--- web/e2e/heartbeat.spec.ts | 51 +++++++++++++++++++++++++++++++++++++++ web/e2e/helpers.ts | 36 +++++++++++++++++++++++++++ web/e2e/mfa.spec.ts | 49 ++++++------------------------------- 4 files changed, 108 insertions(+), 44 deletions(-) create mode 100644 web/e2e/heartbeat.spec.ts create mode 100644 web/e2e/helpers.ts diff --git a/CLAUDE.md b/CLAUDE.md index 8d8d6d3..2cdde80 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -267,7 +267,16 @@ a dark ground. ### Monitors -HTTP, TCP, ICMP and TLS checks. Each monitor has a `runner`: `"server"` (executed by the server-side scheduler) or a `server_id` (pushed to that agent, which runs it locally and reports results). Consecutive failures beyond `retries` flip state to `down`, open an `Incident`, and notify. Hourly `Rollup` documents back the uptime graphs. +HTTP, TCP, ICMP and TLS checks are pull-based. Each has a `runner`: `"server"` (executed by `monitorsched`) or a `server_id` (pushed to that agent, which runs it locally and reports results). Consecutive failures beyond `retries` flip state to `down`, open an `Incident`, and notify. Hourly `Rollup` documents back the uptime graphs. + +Two passive types are never run by `monitorsched` or agents; `metricsched` sweeps them every 30s: + +- `heartbeat`: jobs call `/public/hb/` (plus `/start`, `/fail`), or send the token in `X-Vantage-Token` to `/public/hb[/start|/fail]`; a URL token wins. Request logs (gin formatter via `api.MaskLogPath`, bundled nginx `log_format vantage`) mask URL tokens. Down when a ping is overdue past period + grace, a start never finishes, or `/fail` is called. The token is stored hashed; plaintext is shown on create and `POST /api/monitors/:id/rotate-token` only. +- `metric`: a tag `selector` plus a rule (`disk_pct`, `disk_free_gb`, `mem_pct`, `load_per_core`, `unit_failed`, `container_unhealthy`, `reboot_pending_days`, `agent_offline_min`) evaluated against stored inventory and workloads. State is per server in `monitor_server_states`, with one incident per breaching server (`Incident.ServerID`). Inventory older than 5 minutes is skipped. Restricted tokens may only use selectors inside their tag scope. + +All three paths share `applyTransition` (services/monitortransition.go) for incidents and notifications. + +**Two known gaps, deferred.** Nginx's error log (not the access log) and gin's panic recovery dump can still record a URL token even with `MaskLogPath` masking normal request logs; the `X-Vantage-Token` header avoids putting the token in the URL at all, which is the mitigation until those paths are masked too. And the bundled nginx `vantage.conf` is reproduced in `vantage-docs`' self-hosted install docs, so its `log_format` change must be mirrored there as well - the routing-change rule above already says the file must be, this extends it to the log format. ### Notification channels @@ -1084,7 +1093,8 @@ workflows GET,POST /steps · PUT,DELETE /steps/:id · GET /steps/:id/export GET /runs/:runId · POST /runs/:runId/cancel GET /runs/:runId/servers/:serverId/logs[/stream] monitors GET,POST /monitors · GET,PUT,DELETE /monitors/:id - GET /monitors/:id/{incidents,uptime} + GET /monitors/:id/{incidents,uptime,samples,servers} + POST /monitors/:id/rotate-token channels GET,POST /channels · PUT,DELETE /channels/:id · POST /channels/:id/test secrets GET,POST /secrets · GET,PUT,DELETE /secrets/:group POST /secrets/:group/reveal · DELETE /secrets/:group/:key @@ -1174,7 +1184,7 @@ plane, each of which this codebase enforces: ## MongoDB Collections -`servers` · `keys` · `assignments` · `orgs` · `users` · `auth_providers` · `settings` · `secrets` · `workflows` · `workflow_steps` · `workflow_runs` · `workflow_log_lines` · `workflow_log_seq` · `monitors` · `incidents` · `monitor_rollups` · `notification_channels` · `console_sessions` · `audit_logs` · `server_packages` · `vuln_findings` · `vuln_alert_rules` · `vulndb_meta` · `server_workloads` · `api_tokens` · `status_pages` · `status_incidents` · `maintenance_windows` · `patch_policies` · `patch_runs` · `patch_run_outputs` · `user_mfa` · `webauthn_credentials` · `migrations` +`servers` · `keys` · `assignments` · `orgs` · `users` · `auth_providers` · `settings` · `secrets` · `workflows` · `workflow_steps` · `workflow_runs` · `workflow_log_lines` · `workflow_log_seq` · `monitors` · `incidents` · `monitor_rollups` · `monitor_server_states` · `notification_channels` · `console_sessions` · `audit_logs` · `server_packages` · `vuln_findings` · `vuln_alert_rules` · `vulndb_meta` · `server_workloads` · `api_tokens` · `status_pages` · `status_incidents` · `maintenance_windows` · `patch_policies` · `patch_runs` · `patch_run_outputs` · `user_mfa` · `webauthn_credentials` · `migrations` Every document except `migrations` carries `org_id`. Struct definitions are the source of truth - see `server/internal/models/`. diff --git a/web/e2e/heartbeat.spec.ts b/web/e2e/heartbeat.spec.ts new file mode 100644 index 0000000..735038b --- /dev/null +++ b/web/e2e/heartbeat.spec.ts @@ -0,0 +1,51 @@ +import { test, expect } from "@playwright/test"; +import { ensureOwner, signIn } from "./helpers"; + +const BASE_URL = process.env.E2E_BASE_URL ?? "http://localhost:3000"; + +test("heartbeat monitor: URL ping, /fail, header ping", async ({ page, request }) => { + const owner = await ensureOwner(request); + await signIn(page, owner.email, owner.password); + + const create = await page.request.post(`${BASE_URL}/api/monitors`, { + data: { + name: `e2e heartbeat ${Date.now()}`, + type: "heartbeat", + target: { period_sec: 3600, grace_sec: 300 }, + enabled: true, + }, + }); + expect(create.status()).toBe(201); + const monitor = await create.json(); + expect(monitor.heartbeat_token).toBeTruthy(); + + const ping = await request.get(`${BASE_URL}/public/hb/${monitor.heartbeat_token}`); + expect(ping.status()).toBe(200); + expect(await ping.text()).toBe("OK"); + + let got = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}`)).json(); + expect(got.state.status).toBe("up"); + + await new Promise((r) => setTimeout(r, 1100)); // per-token limit is 1/s + const fail = await request.post(`${BASE_URL}/public/hb/${monitor.heartbeat_token}/fail`, { data: "disk full" }); + expect(fail.status()).toBe(200); + + got = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}`)).json(); + expect(got.state.status).toBe("down"); + expect(got.state.message).toContain("disk full"); + + const incidents = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}/incidents`)).json(); + expect(incidents.length).toBeGreaterThan(0); + + await new Promise((r) => setTimeout(r, 1100)); + const headerPing = await request.post(`${BASE_URL}/public/hb`, { headers: { "X-Vantage-Token": monitor.heartbeat_token } }); + expect(headerPing.status()).toBe(200); + got = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}`)).json(); + expect(got.state.status).toBe("up"); + + expect((await request.get(`${BASE_URL}/public/hb/not-a-real-token`)).status()).toBe(404); + expect((await request.post(`${BASE_URL}/public/hb`)).status()).toBe(404); + + await page.goto(`${BASE_URL}/monitors/${monitor.monitor_id}`); + await expect(page.getByText(/last ping/i)).toBeVisible(); +}); diff --git a/web/e2e/helpers.ts b/web/e2e/helpers.ts new file mode 100644 index 0000000..5299c73 --- /dev/null +++ b/web/e2e/helpers.ts @@ -0,0 +1,36 @@ +import { type APIRequestContext, type Page } from "@playwright/test"; + +export const BASE_URL = process.env.E2E_BASE_URL ?? "http://localhost:3000"; + +let ownerEmail = process.env.E2E_OWNER_EMAIL ?? ""; +let ownerPassword = process.env.E2E_OWNER_PASSWORD ?? ""; + +/** Ensures an owner account exists and returns its credentials, bootstrapping the instance if needed. */ +export async function ensureOwner(request: APIRequestContext): Promise<{ email: string; password: string }> { + if (ownerEmail && ownerPassword) return { email: ownerEmail, password: ownerPassword }; + + const status = await request.get(`${BASE_URL}/auth/bootstrap-status`); + const body = await status.json(); + if (!body.needs_setup) { + throw new Error( + "Instance is already bootstrapped and E2E_OWNER_EMAIL/E2E_OWNER_PASSWORD were not set - " + + "cannot create the owner account this suite needs to provision fresh members per test.", + ); + } + + ownerEmail = `e2e-owner-${Date.now()}@vantage.test`; + ownerPassword = "correct horse battery staple 1"; + const res = await request.post(`${BASE_URL}/auth/bootstrap`, { + data: { instance_name: "E2E", email: ownerEmail, password: ownerPassword }, + }); + if (!res.ok()) throw new Error(`bootstrap failed: ${res.status()} ${await res.text()}`); + return { email: ownerEmail, password: ownerPassword }; +} + +/** Fills the login form and submits it. */ +export async function signIn(page: Page, email: string, password: string) { + await page.goto("/login"); + await page.getByLabel("Email").fill(email); + await page.getByLabel("Password").fill(password); + await page.getByRole("button", { name: "Sign In" }).click(); +} diff --git a/web/e2e/mfa.spec.ts b/web/e2e/mfa.spec.ts index 2ba112a..889095d 100644 --- a/web/e2e/mfa.spec.ts +++ b/web/e2e/mfa.spec.ts @@ -1,5 +1,6 @@ import { test, expect, type APIRequestContext, type BrowserContext, type Page } from "@playwright/test"; import { authenticator } from "otplib"; +import { BASE_URL, ensureOwner, signIn } from "./helpers"; /** * End-to-end coverage for Task 15 of the MFA design @@ -17,33 +18,6 @@ import { authenticator } from "otplib"; * at an already-bootstrapped instance's owner instead. */ -const BASE_URL = process.env.E2E_BASE_URL ?? "http://localhost:3000"; - -let ownerEmail = process.env.E2E_OWNER_EMAIL ?? ""; -let ownerPassword = process.env.E2E_OWNER_PASSWORD ?? ""; - -/** Ensures an owner account exists and returns its credentials, bootstrapping the instance if needed. */ -async function ensureOwner(request: APIRequestContext): Promise<{ email: string; password: string }> { - if (ownerEmail && ownerPassword) return { email: ownerEmail, password: ownerPassword }; - - const status = await request.get(`${BASE_URL}/auth/bootstrap-status`); - const body = await status.json(); - if (!body.needs_setup) { - throw new Error( - "Instance is already bootstrapped and E2E_OWNER_EMAIL/E2E_OWNER_PASSWORD were not set - " + - "cannot create the owner account this suite needs to provision fresh members per test.", - ); - } - - ownerEmail = `e2e-owner-${Date.now()}@vantage.test`; - ownerPassword = "correct horse battery staple 1"; - const res = await request.post(`${BASE_URL}/auth/bootstrap`, { - data: { instance_name: "MFA E2E", email: ownerEmail, password: ownerPassword }, - }); - if (!res.ok()) throw new Error(`bootstrap failed: ${res.status()} ${await res.text()}`); - return { email: ownerEmail, password: ownerPassword }; -} - /** Creates a fresh, unique local member with no MFA enrolled, using an owner session. */ async function createMember(request: APIRequestContext): Promise<{ email: string; password: string }> { const owner = await ensureOwner(request); @@ -80,19 +54,12 @@ async function addVirtualAuthenticator(context: BrowserContext, page: Page): Pro return authenticatorId; } -async function fillCredentials(page: Page, email: string, password: string) { - await page.goto("/login"); - await page.getByLabel("Email").fill(email); - await page.getByLabel("Password").fill(password); - await page.getByRole("button", { name: "Sign In" }).click(); -} - test.describe("MFA end-to-end", () => { test("TOTP sign-in", async ({ page, request }) => { const { email, password } = await createMember(request); // Sign in, land on the account security page, enrol TOTP. - await fillCredentials(page, email, password); + await signIn(page, email, password); await expect(page).toHaveURL("/"); await page.goto("/account/security"); @@ -111,7 +78,7 @@ test.describe("MFA end-to-end", () => { // Sign out and sign back in - TOTP is now required. await page.request.post(`${BASE_URL}/auth/logout`); - await fillCredentials(page, email, password); + await signIn(page, email, password); await expect(page.getByText("Enter the 6-digit code")).toBeVisible(); const nextCode = authenticator.generate(secret.trim()); @@ -125,7 +92,7 @@ test.describe("MFA end-to-end", () => { const { email, password } = await createMember(request); await addVirtualAuthenticator(context, page); - await fillCredentials(page, email, password); + await signIn(page, email, password); await expect(page).toHaveURL("/"); await page.goto("/account/security"); @@ -138,7 +105,7 @@ test.describe("MFA end-to-end", () => { await expect(page.getByText(/Added .* · Last used/)).toBeVisible(); await page.request.post(`${BASE_URL}/auth/logout`); - await fillCredentials(page, email, password); + await signIn(page, email, password); await expect(page.getByText("Enter the 6-digit code")).toBeVisible(); await page.getByRole("button", { name: "Use passkey" }).click(); @@ -151,7 +118,7 @@ test.describe("MFA end-to-end", () => { await addVirtualAuthenticator(context, page); // Enrol a passkey first (typing the password once, during setup only). - await fillCredentials(page, email, password); + await signIn(page, email, password); await expect(page).toHaveURL("/"); await page.goto("/account/security"); await page.getByRole("button", { name: "Add a passkey" }).click(); @@ -180,7 +147,7 @@ test.describe("MFA end-to-end", () => { await request.post(`${BASE_URL}/auth/logout`); // The member, who has no MFA yet, must enrol before reaching the app. - await fillCredentials(page, email, password); + await signIn(page, email, password); await expect(page.getByText("This instance requires a second sign-in factor. Set one up to continue.")).toBeVisible(); await page.getByRole("button", { name: "Use an authenticator app" }).click(); @@ -204,7 +171,7 @@ test.describe("MFA end-to-end", () => { // Enrol TOTP so the account has a factor - not exercised in this test, // since sign-in itself sets StepUpAt and the window has not gone stale. - await fillCredentials(page, email, password); + await signIn(page, email, password); await page.goto("/account/security"); await page.getByRole("button", { name: "Set up" }).click(); await page.getByRole("button", { name: "Use an authenticator app" }).click();