fix(monitors): document 403 and 404 on monitor update and delete; regenerate openapi

Mask base64 and URL-encoded forms of secret values, each line of a
multi-line secret, and secrets loaded by earlier steps of the run.
Replace longer values first so overlapping secrets mask cleanly.

CLAUDE.md

@@ -267,7 +267,16 @@ a dark ground.
 
 ### Monitors
 
-HTTP, TCP, ICMP and TLS checks. Each monitor has a `runner`: `"server"` (executed by the server-side scheduler) or a `server_id` (pushed to that agent, which runs it locally and reports results). Consecutive failures beyond `retries` flip state to `down`, open an `Incident`, and notify. Hourly `Rollup` documents back the uptime graphs.
+HTTP, TCP, ICMP and TLS checks are pull-based. Each has a `runner`: `"server"` (executed by `monitorsched`) or a `server_id` (pushed to that agent, which runs it locally and reports results). Consecutive failures beyond `retries` flip state to `down`, open an `Incident`, and notify. Hourly `Rollup` documents back the uptime graphs.
+
+Two passive types are never run by `monitorsched` or agents; `metricsched` sweeps them every 30s:
+
+- `heartbeat`: jobs call `/public/hb/<token>` (plus `/start`, `/fail`), or send the token in `X-Vantage-Token` to `/public/hb[/start|/fail]`; a URL token wins. Request logs (gin formatter via `api.MaskLogPath`, bundled nginx `log_format vantage`) mask URL tokens. Down when a ping is overdue past period + grace, a start never finishes, or `/fail` is called. The token is stored hashed; plaintext is shown on create and `POST /api/monitors/:id/rotate-token` only.
+- `metric`: a tag `selector` plus a rule (`disk_pct`, `disk_free_gb`, `mem_pct`, `load_per_core`, `unit_failed`, `container_unhealthy`, `reboot_pending_days`, `agent_offline_min`) evaluated against stored inventory and workloads. State is per server in `monitor_server_states`, with one incident per breaching server (`Incident.ServerID`). Inventory older than 5 minutes is skipped. Restricted tokens may only use selectors inside their tag scope.
+
+All three paths share `applyTransition` (services/monitortransition.go) for incidents and notifications.
+
+**Two known gaps, deferred.** Nginx's error log (not the access log) and gin's panic recovery dump can still record a URL token even with `MaskLogPath` masking normal request logs; the `X-Vantage-Token` header avoids putting the token in the URL at all, which is the mitigation until those paths are masked too. And the bundled nginx `vantage.conf` is reproduced in `vantage-docs`' self-hosted install docs, so its `log_format` change must be mirrored there as well - the routing-change rule above already says the file must be, this extends it to the log format.
 
 ### Notification channels
 
@@ -746,6 +755,15 @@ purpose - the collection is `api_tokens`, the prefix is `vt_`, the routes are
 `/api/tokens`, and renaming a published endpoint to match a nav label would
 break every script already written against it.
 
+**Every `/api` route must be declared twice or the server refuses to boot**:
+once in `routeScopes` (or `sessionOnlyRoutes`) and once in `serverScopedRoutes`.
+`sessionOnlyRoutes` holds the MFA routes, which no API token may reach - a
+token that could enrol a passkey or satisfy step-up would be a way around MFA.
+They have no scope on purpose, and `RequireScopes` refuses any unmapped route
+to a token. `TestRegisteredRoutesPassBootAssertions` registers the real routes
+and runs both boot assertions, so a missing declaration fails CI rather than
+production startup.
+
 `server/internal/api/docs/openapi.json` is a **generated, committed** OpenAPI
 3.1 document - `swag v2` reading `@…` annotations off the handlers - served at
 `GET /api/openapi.json` and rendered as a reference page by a vendored Scalar
@@ -1075,7 +1093,8 @@ workflows    GET,POST /steps · PUT,DELETE /steps/:id · GET /steps/:id/export
              GET /runs/:runId · POST /runs/:runId/cancel
              GET /runs/:runId/servers/:serverId/logs[/stream]
 monitors     GET,POST /monitors · GET,PUT,DELETE /monitors/:id
-             GET /monitors/:id/{incidents,uptime}
+             GET /monitors/:id/{incidents,uptime,samples,servers}
+             POST /monitors/:id/rotate-token
 channels     GET,POST /channels · PUT,DELETE /channels/:id · POST /channels/:id/test
 secrets      GET,POST /secrets · GET,PUT,DELETE /secrets/:group
              POST /secrets/:group/reveal · DELETE /secrets/:group/:key
@@ -1165,7 +1184,7 @@ plane, each of which this codebase enforces:
 
 ## MongoDB Collections
 
-`servers` · `keys` · `assignments` · `orgs` · `users` · `auth_providers` · `settings` · `secrets` · `workflows` · `workflow_steps` · `workflow_runs` · `workflow_log_lines` · `workflow_log_seq` · `monitors` · `incidents` · `monitor_rollups` · `notification_channels` · `console_sessions` · `audit_logs` · `server_packages` · `vuln_findings` · `vuln_alert_rules` · `vulndb_meta` · `server_workloads` · `api_tokens` · `status_pages` · `status_incidents` · `maintenance_windows` · `patch_policies` · `patch_runs` · `patch_run_outputs` · `user_mfa` · `webauthn_credentials` · `migrations`
+`servers` · `keys` · `assignments` · `orgs` · `users` · `auth_providers` · `settings` · `secrets` · `workflows` · `workflow_steps` · `workflow_runs` · `workflow_log_lines` · `workflow_log_seq` · `monitors` · `incidents` · `monitor_rollups` · `monitor_server_states` · `notification_channels` · `console_sessions` · `audit_logs` · `server_packages` · `vuln_findings` · `vuln_alert_rules` · `vulndb_meta` · `server_workloads` · `api_tokens` · `status_pages` · `status_incidents` · `maintenance_windows` · `patch_policies` · `patch_runs` · `patch_run_outputs` · `user_mfa` · `webauthn_credentials` · `migrations`
 
 Every document except `migrations` carries `org_id`. Struct definitions are the source of truth - see `server/internal/models/`.
 

deploy/docker/nginx/vantage.conf

@@ -12,6 +12,17 @@ map $http_upgrade $connection_upgrade {
     ''      close;
 }
 
+# Heartbeat ping URLs carry a credential. Log them with the token replaced;
+# the header form (X-Vantage-Token) is never logged by this format.
+map $request_uri $vantage_log_uri {
+    "~^/public/hb/(?!start(?:[/?]|$)|fail(?:[/?]|$))[^/?]+(?<hb_rest>.*)$" "/public/hb/***$hb_rest";
+    default $request_uri;
+}
+
+log_format vantage '$remote_addr - $remote_user [$time_local] '
+                   '"$request_method $vantage_log_uri $server_protocol" '
+                   '$status $body_bytes_sent "$http_referer" "$http_user_agent"';
+
 upstream vantage_server {
     server server:8080;
     keepalive 16;
@@ -27,6 +38,8 @@ server {
     listen [::]:80;
     server_name _;
 
+    access_log /var/log/nginx/access.log vantage;
+
     # Step imports and licence pastes are the largest request bodies.
     client_max_body_size 10m;
 

docs/superpowers/specs/2026-09-15-mfa-local-signin-design.md

@@ -17,7 +17,7 @@ In:
 
 - TOTP as a second factor, with 10 single-use recovery codes.
 - WebAuthn passkeys, both as a second factor after a password and as
-  passwordless sign-in (discoverable credentials, user verification required).
+  passwordless sign-in (discoverable credentials, user verification preferred).
 - An owner setting, `require_mfa`, that forces enrolment for password users.
 - Owner/admin reset of another member's MFA.
 - Step-up re-authentication before three sensitive actions: secret reveal,
@@ -174,8 +174,11 @@ at their 24h TTL and the next sign-in enforces enrolment.
    `{instance_id, credential_id}`, verifies the assertion including the UV flag,
    and mints the session with `amr: ["webauthn"]`.
 
-A user-verified passkey is possession plus biometric or PIN, so it satisfies
-`require_mfa` on its own.
+User verification is preferred, not required. Requiring it made password
+managers such as NordPass prompt for their master password on every use, so it
+was relaxed on 2026-09-16 at the product owner's decision. A passkey used
+without verification is possession-only; passwordless sign-in and step-up then
+rest on the device or vault being unlocked, and still satisfy `require_mfa`.
 
 The discoverable lookup is scoped by `instance_id` from the host, never by the
 credential alone - the same rule that makes `users` lookups instance-scoped.

docs/superpowers/specs/2026-09-17-metric-alerts-heartbeats-design.md

@@ -0,0 +1,186 @@
+# Metric alerts and heartbeat monitors
+
+Date: 2026-09-17
+Source: competitive gap review, "Alerts on agent metrics" (effort M) and "Heartbeat (push) monitors" (effort S).
+
+## Goal
+
+1. Alert on data agents already report (disk, memory, load, failed units, unhealthy containers, reboot pending, agent offline), targeted by tag so one rule covers the fleet.
+2. Alert when a push-based job (backup, cron) stops pinging, reports failure, or starts and never finishes.
+
+Both reuse the existing monitor model, states, incidents, notification channels, groups and status pages.
+
+## Out of scope
+
+- Scheduled workflows pinging a heartbeat automatically.
+- Metric history graphs (separate gap review item).
+- Agent-side evaluation. No agent release is required.
+
+## Data model
+
+New constants in `models/monitor.go`: `MonitorMetric = "metric"`, `MonitorHeartbeat = "heartbeat"`.
+
+`MonitorTarget` gains:
+
+| Field | Type | Used by | Meaning |
+|---|---|---|---|
+| `Selector` | `map[string]string` | metric | Server tag selector. Empty means the whole org fleet. |
+| `Metric` | `string` | metric | One of the metric kinds below. |
+| `Threshold` | `float64` | metric | Breach threshold, unit depends on kind. |
+| `Mount` | `string` | metric (disk kinds) | Mountpoint. Empty means any mount breaches. |
+| `PeriodSec` | `int` | heartbeat | Expected time between pings. |
+| `GraceSec` | `int` | heartbeat | Extra time allowed before overdue, and the max run time after `/start`. |
+
+`Monitor` gains `ForSec int` (metric): the condition must hold continuously this long before a server goes down. 0 means on first evaluation.
+
+`Monitor` gains `HeartbeatTokenHash string` (`json:"-"`), SHA-256 of the ping token. The plaintext token is returned only on create and on rotate.
+
+`MonitorState` gains, for heartbeats: `LastPingAt *time.Time`, `StartedAt *time.Time`.
+
+`Incident` gains `ServerID string` (`omitempty`). Empty for every non-metric monitor.
+
+`Inventory` gains `RebootRequiredSince *time.Time`, set by `StoreInventory` when `reboot_required` turns true and unset when it turns false.
+
+New collection `monitor_server_states`, one document per (metric monitor, matching server):
+
+```go
+type MonitorServerState struct {
+    InstanceID  string     // org
+    MonitorID   string
+    ServerID    string
+    Status      string     // up | down | pending
+    BreachSince *time.Time // first evaluation where the condition held; nil when clear
+    Value       float64    // last evaluated value, for display
+    Message     string     // e.g. "/var 94.2% used"
+    UpdatedAt   time.Time
+}
+```
+
+Unique index on `(monitor_id, server_id)`.
+
+### Metric kinds
+
+| Kind | Source | Breach when | Threshold unit |
+|---|---|---|---|
+| `disk_pct` | `inventory.partitions` | used/total*100 >= threshold on `Mount` (or any mount) | percent |
+| `disk_free_gb` | `inventory.partitions` | (total-used)/1e9 <= threshold | GB |
+| `mem_pct` | `inventory.memory` | used/total*100 >= threshold | percent |
+| `load_per_core` | `inventory.cpu.load1 / cores` | >= threshold | ratio |
+| `unit_failed` | workloads, kind `unit` | any unit state `failed` (threshold unused) | - |
+| `container_unhealthy` | workloads, kind `container` | any health `unhealthy` (threshold unused) | - |
+| `reboot_pending_days` | `inventory.reboot_required_since` | now - since >= threshold days | days |
+| `agent_offline_min` | `server.last_seen` | now - last_seen >= threshold minutes | minutes |
+
+Validation on create/update: `Metric` is a known kind; `Threshold` > 0 for kinds that use it; `disk_pct` and `mem_pct` threshold <= 100; `Mount`, if set, is an absolute path; `Selector` keys and values follow existing tag rules. `IntervalSec`, `Runner` and `Retries` are ignored for metric and heartbeat monitors and stored as `runner = "server"`, so `ListServerScheduledMonitors` must exclude these two types from the pull scheduler.
+
+Heartbeat validation: `PeriodSec` >= 60, `GraceSec` >= 0 (default 300 when unset).
+
+## Shared transition logic
+
+Extract from `ingestResult` in `services/monitors.go`:
+
+```go
+func applyTransition(ctx context.Context, m *models.Monitor, serverID, prev, next, message string, now time.Time)
+```
+
+It opens an incident (with `ServerID`) on a change to down, resolves the matching open incident (same `monitor_id` and `server_id`) on down to up, and calls `notifyTransition`. `ingestResult` calls it with `serverID = ""`. Behaviour of existing monitor types does not change.
+
+`notify.Event` gains `ServerName string`. Channel formatters include it when set ("disk on web-01: /var 94.2% used").
+
+## Metric evaluation
+
+New package `server/internal/metricsched`, started from `main.go` beside `monitorsched`. Like monitors, it runs regardless of licence state.
+
+Every 30 seconds, for each enabled metric monitor:
+
+1. Resolve servers with `ListServersFiltered(instanceID, selector)`.
+2. Load existing `monitor_server_states` for the monitor.
+3. For each server:
+   - For kinds other than `agent_offline_min`: skip (keep previous state) when `inventory.metrics_at` is older than 5 minutes or missing. A dead agent must not flap other alerts; `agent_offline_min` covers it.
+   - Evaluate the kind to get `(breach bool, value float64, message string)`.
+   - Breach: set `BreachSince` if nil. If `now - BreachSince >= ForSec`, next = down, otherwise next = pending when previously up or new.
+   - No breach: `BreachSince = nil`, next = up.
+   - Upsert the state doc. On a status change call `applyTransition` with the server ID. A new server's first state is never treated as a transition from down.
+4. State docs for servers no longer matched (tag removed, server deleted): resolve any open incident without notifying, and delete the doc.
+5. Roll up the parent `Monitor.State`: status = down if any child down, else pending if any pending, else up (no matching servers means up). Message is "N of M servers breaching". `last_check_at = now`. Write one `MonitorSample` (up = no child down, latency 0) and increment the hourly rollup, so uptime graphs and status pages work unchanged.
+
+Deleting a metric monitor deletes its `monitor_server_states`.
+
+Evaluators are pure functions `func(kind string, t models.MonitorTarget, srv models.Server, wls []models.Workload, now time.Time) (bool, float64, string)` so they can be table-tested without Mongo. Workloads are loaded only for monitors whose kind needs them.
+
+## Heartbeats
+
+### Public endpoints
+
+Registered outside `/api`, unauthenticated, no scope declarations needed:
+
+- `GET|POST /public/hb/:token`: success ping
+- `GET|POST /public/hb/:token/start`: run started
+- `GET|POST /public/hb/:token/fail`: run failed
+
+Mounted under /public because every deployment already routes that prefix to the server. The token may instead be sent in the `X-Vantage-Token` header to `/public/hb`, `/public/hb/start` or `/public/hb/fail`; a URL token wins when both are present. The server's request log and the bundled nginx access log mask the URL token.
+
+Lookup is by SHA-256 of the token. Unknown token or disabled monitor gives 404. Rate limit is one accepted request per second per token (in-process, Redis-backed if a limiter helper already exists); excess gives 429. The response body is `OK`. Request bodies over 1 KB are truncated; only `/fail` uses the body.
+
+Behaviour:
+
+- **ping**: `LastPingAt = now`. If `StartedAt` is set, duration = now - StartedAt, write a sample with `LatencyMs = duration`, then clear `StartedAt`. Otherwise write a sample with latency 0. Next status is up; call `applyTransition` if changed.
+- **start**: `StartedAt = now`. No status change.
+- **fail**: clear `StartedAt`, write a down sample, next status is down immediately with message `reported failure: <body>` (body defaults to empty). The failure is also recorded as `LastPingAt` so the overdue timer restarts from it.
+
+All state writes go through one conditional update per request, so concurrent pings don't lose updates.
+
+### Overdue sweep
+
+The same 30-second `metricsched` loop checks enabled heartbeat monitors:
+
+- New monitor with no ping yet: pending, never down. The overdue clock starts at the first ping.
+- `now > LastPingAt + PeriodSec + GraceSec`: down, message `no ping since <time>`.
+- `StartedAt != nil && now > StartedAt + GraceSec`: down, message `started <time>, never finished`.
+
+Recovery happens only through a ping.
+
+### Token management
+
+- Create returns `heartbeat_token` and `heartbeat_url` once.
+- `POST /api/monitors/:id/rotate-token` issues a new token and invalidates the old one. It needs `routeScopes` (`monitors:write`) and `serverScopedRoutes` entries.
+- The UI shows the URL only right after create or rotate, with a copy button and curl examples for ping, start and fail.
+
+## API
+
+- Existing create/update monitor endpoints accept the new types and fields. OpenAPI annotations are updated.
+- `GET /api/monitors/:id/servers` returns `monitor_server_states` for a metric monitor, with hostnames. It needs `routeScopes` (`monitors:read`) and `serverScopedRoutes` entries, and filters rows to servers visible to the acting token.
+- A tag-scoped token may create or update a metric monitor only when its selector includes every key/value of the token's scope. Otherwise the response is 403.
+- The MCP `create_monitor` tool accepts the new types.
+
+## Web UI
+
+- Monitor form: a type picker adds Metric and Heartbeat. Metric shows a selector editor (reuse the tag selector component), a kind dropdown, a threshold with a unit label, mount (disk kinds only) and "for N minutes". Heartbeat shows period and grace.
+- Monitors list: metric rows show "N of M servers breaching". Heartbeat rows show last ping as relative time.
+- Metric detail page: a per-server table with server, status, value, since and message, linking to the server page. The incident list shows the server name.
+- Heartbeat detail page: URL panel (after create or rotate), rotate button with confirm, last ping, and duration in the latency chart.
+
+## Error handling
+
+- Evaluator panics or bad inventory data: recover per monitor, log, and leave states unchanged.
+- Mongo errors in the sweep: log and continue to the next monitor. The next tick retries.
+- Notification failures are logged only, as today.
+
+## Testing
+
+- Table tests for every evaluator: breach, clear, mount filter, missing inventory, zero totals.
+- Sweep tests (Mongo test helper): the for-duration gate, pending to down, stale inventory skipped, a server leaving the selector resolves its incident silently, parent rollup counts.
+- Heartbeat handler tests: ping/start/fail transitions, duration sample, unknown token 404, rate limit 429, fail body truncation.
+- Overdue tests: never pinged stays pending, overdue goes down, start without finish goes down, ping recovers.
+- `applyTransition` regression: existing http monitor incident open/resolve unchanged.
+- `go test ./internal/api/` including `TestRegisteredRoutesPassBootAssertions`.
+- Playwright: create a heartbeat, curl a ping, see up.
+
+## Phases
+
+1. Shared `applyTransition` refactor, heartbeat type, public endpoints, overdue sweep, rotate route, UI. Ships alone.
+2. Metric type, `RebootRequiredSince`, `monitor_server_states`, evaluators, sweep, servers route, UI.
+
+## Documentation
+
+Update the Monitors section of `CLAUDE.md` and add a user guide page in vantage-docs for both types.

server/cmd/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"log"
 	"net/http"
 	"os"
@@ -23,6 +24,7 @@ import (
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
 	grpcserver "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/grpc"
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/mcp"
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/metricsched"
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/monitorsched"
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/patchsched"
@@ -173,6 +175,10 @@ func runSchemaSetup() {
 		log.Printf("warning: failed to ensure monitor sample indexes: %v", err)
 	}
 
+	if err := services.EnsureMonitorServerStateIndexes(); err != nil {
+		log.Printf("warning: %v", err)
+	}
+
 	if err := services.EnsureVulnIndexes(); err != nil {
 		log.Printf("warning: failed to ensure vuln indexes: %v", err)
 	}
@@ -259,6 +265,7 @@ func serve() {
 		services.StartAuditSweeper(jobCtx)
 		services.StartReaper(jobCtx)
 		monitorsched.Start(jobCtx)
+		metricsched.Start(jobCtx)
 		workflowsched.Start(jobCtx, workflowsched.Deps{
 			TriggerWorkflow: services.TriggerWorkflow,
 			LogEvent:        services.LogEvent,
@@ -305,7 +312,16 @@ func serve() {
 		log.Fatalf("trusted proxies: %v", err)
 	}
 	r.Use(gin.Recovery())
-	r.Use(gin.LoggerWithConfig(gin.LoggerConfig{SkipPaths: []string{"/api/console/tunnel"}}))
+	r.Use(gin.LoggerWithConfig(gin.LoggerConfig{
+		SkipPaths: []string{"/api/console/tunnel"},
+		// Heartbeat URLs carry a credential; the request log must not.
+		Formatter: func(p gin.LogFormatterParams) string {
+			return fmt.Sprintf("[GIN] %v | %3d | %13v | %15s | %-7s %#v\n%s",
+				p.TimeStamp.Format("2006/01/02 - 15:04:05"),
+				p.StatusCode, p.Latency, p.ClientIP, p.Method,
+				api.MaskLogPath(p.Path), p.ErrorMessage)
+		},
+	}))
 	r.Use(corsMiddleware())
 	services.SetStatusRedis(auth.Redis())
 

server/internal/api/bootassert_test.go

@@ -0,0 +1,42 @@
+package api
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+)
+
+// The two boot assertions only run against a real engine in main.go, so a
+// route added without its declarations compiled, passed every unit test and
+// then refused to start in production. This registers the real routes and
+// runs both assertions, so that failure lands in CI instead.
+func TestRegisteredRoutesPassBootAssertions(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	r := gin.New()
+	RegisterRoutes(r)
+
+	if err := AssertScopeMapComplete(r); err != nil {
+		t.Fatalf("scope map: %v", err)
+	}
+
+	var routes []string
+	for _, route := range r.Routes() {
+		if strings.HasPrefix(route.Path, "/api/") {
+			routes = append(routes, route.Method+" "+route.Path)
+		}
+	}
+	if err := AssertServerScopeMapComplete(routes); err != nil {
+		t.Fatalf("server scope map: %v", err)
+	}
+}
+
+// A route cannot be both session-only and token-reachable: an entry in both
+// maps would silently hand the MFA routes to any token holding that scope.
+func TestSessionOnlyRoutesHaveNoTokenScope(t *testing.T) {
+	for route := range sessionOnlyRoutes {
+		if scope, ok := routeScopes[route]; ok {
+			t.Errorf("%s is session-only but routeScopes grants it to tokens with %q", route, scope)
+		}
+	}
+}

server/internal/api/docs/openapi.json

@@ -1045,6 +1045,10 @@
                     "resolved_at": {
                         "type": "string"
                     },
+                    "server_id": {
+                        "description": "ServerID is set only for metric monitors, which keep one incident per\nbreaching server.",
+                        "type": "string"
+                    },
                     "started_at": {
                         "type": "string"
                     }
@@ -1114,6 +1118,10 @@
                     "reboot_required": {
                         "type": "boolean"
                     },
+                    "reboot_required_since": {
+                        "description": "RebootRequiredSince is when the host first reported a pending reboot.",
+                        "type": "string"
+                    },
                     "static_at": {
                         "type": "string"
                     },
@@ -1224,10 +1232,18 @@
                     "enabled": {
                         "type": "boolean"
                     },
+                    "for_sec": {
+                        "description": "ForSec is how long a metric condition must hold before a server is down.",
+                        "type": "integer"
+                    },
                     "group": {
                         "description": "Group is a display-only label. It buckets rows on the monitors page and\nhas no effect on scheduling, alerting or scope; an empty group means the\nmonitor is listed on its own under \"Ungrouped\".",
                         "type": "string"
                     },
+                    "heartbeat_token": {
+                        "description": "HeartbeatToken is the plaintext token, set only on the create response.",
+                        "type": "string"
+                    },
                     "instance_id": {
                         "type": "string"
                     },
@@ -1278,6 +1294,38 @@
                 },
                 "type": "object"
             },
+            "models.MonitorServerState": {
+                "properties": {
+                    "breach_since": {
+                        "type": "string"
+                    },
+                    "hostname": {
+                        "type": "string"
+                    },
+                    "instance_id": {
+                        "type": "string"
+                    },
+                    "message": {
+                        "type": "string"
+                    },
+                    "monitor_id": {
+                        "type": "string"
+                    },
+                    "server_id": {
+                        "type": "string"
+                    },
+                    "status": {
+                        "type": "string"
+                    },
+                    "updated_at": {
+                        "type": "string"
+                    },
+                    "value": {
+                        "type": "number"
+                    }
+                },
+                "type": "object"
+            },
             "models.MonitorState": {
                 "properties": {
                     "cert_expiry_at": {
@@ -1292,12 +1340,18 @@
                     "last_notified_at": {
                         "type": "string"
                     },
+                    "last_ping_at": {
+                        "type": "string"
+                    },
                     "latency_ms": {
                         "type": "integer"
                     },
                     "message": {
                         "type": "string"
                     },
+                    "started_at": {
+                        "type": "string"
+                    },
                     "status": {
                         "type": "string"
                     }
@@ -1309,6 +1363,9 @@
                     "expected_status": {
                         "type": "integer"
                     },
+                    "grace_sec": {
+                        "type": "integer"
+                    },
                     "host": {
                         "type": "string"
                     },
@@ -1321,9 +1378,29 @@
                     "method": {
                         "type": "string"
                     },
+                    "metric": {
+                        "type": "string"
+                    },
+                    "mount": {
+                        "type": "string"
+                    },
+                    "period_sec": {
+                        "description": "Heartbeat monitors.",
+                        "type": "integer"
+                    },
                     "port": {
                         "type": "integer"
                     },
+                    "selector": {
+                        "additionalProperties": {
+                            "type": "string"
+                        },
+                        "description": "Metric monitors.",
+                        "type": "object"
+                    },
+                    "threshold": {
+                        "type": "number"
+                    },
                     "tls_warn_days": {
                         "type": "integer"
                     },
@@ -6290,6 +6367,16 @@
                     "204": {
                         "description": "No Content"
                     },
+                    "403": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/api.ErrorResponse"
+                                }
+                            }
+                        },
+                        "description": "Forbidden"
+                    },
                     "500": {
                         "content": {
                             "application/json": {
@@ -6402,6 +6489,9 @@
                                             "enabled": {
                                                 "type": "boolean"
                                             },
+                                            "for_sec": {
+                                                "type": "integer"
+                                            },
                                             "group": {
                                                 "type": "string"
                                             },
@@ -6448,6 +6538,26 @@
                         },
                         "description": "Bad Request"
                     },
+                    "403": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/api.ErrorResponse"
+                                }
+                            }
+                        },
+                        "description": "Forbidden"
+                    },
+                    "404": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/api.ErrorResponse"
+                                }
+                            }
+                        },
+                        "description": "Not Found"
+                    },
                     "500": {
                         "content": {
                             "application/json": {
@@ -6535,6 +6645,71 @@
                 ]
             }
         },
+        "/monitors/{id}/rotate-token": {
+            "post": {
+                "description": "Issues a new token and invalidates the old ping URL immediately. The token is returned only in this response.",
+                "parameters": [
+                    {
+                        "description": "Monitor ID",
+                        "in": "path",
+                        "name": "id",
+                        "required": true,
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "properties": {
+                                        "heartbeat_token": {
+                                            "type": "string"
+                                        }
+                                    },
+                                    "type": "object"
+                                }
+                            }
+                        },
+                        "description": "OK"
+                    },
+                    "404": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/api.ErrorResponse"
+                                }
+                            }
+                        },
+                        "description": "Not Found"
+                    },
+                    "500": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/api.ErrorResponse"
+                                }
+                            }
+                        },
+                        "description": "Internal Server Error"
+                    }
+                },
+                "security": [
+                    {
+                        "cookieAuth": []
+                    },
+                    {
+                        "bearerAuth": []
+                    }
+                ],
+                "summary": "Rotate a heartbeat monitor's ping token",
+                "tags": [
+                    "monitors"
+                ]
+            }
+        },
         "/monitors/{id}/samples": {
             "get": {
                 "description": "Raw check results for the last `minutes` minutes, oldest first. Samples expire after 48 hours; use the uptime rollups for longer ranges.",
@@ -6606,6 +6781,68 @@
                 ]
             }
         },
+        "/monitors/{id}/servers": {
+            "get": {
+                "parameters": [
+                    {
+                        "description": "Monitor ID",
+                        "in": "path",
+                        "name": "id",
+                        "required": true,
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "items": {
+                                        "$ref": "#/components/schemas/models.MonitorServerState"
+                                    },
+                                    "type": "array"
+                                }
+                            }
+                        },
+                        "description": "OK"
+                    },
+                    "404": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/api.ErrorResponse"
+                                }
+                            }
+                        },
+                        "description": "Not Found"
+                    },
+                    "500": {
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/api.ErrorResponse"
+                                }
+                            }
+                        },
+                        "description": "Internal Server Error"
+                    }
+                },
+                "security": [
+                    {
+                        "cookieAuth": []
+                    },
+                    {
+                        "bearerAuth": []
+                    }
+                ],
+                "summary": "List a metric monitor's per-server states",
+                "tags": [
+                    "monitors"
+                ]
+            }
+        },
         "/monitors/{id}/uptime": {
             "get": {
                 "description": "Hourly rollups for the last 30 days.",

server/internal/api/handlers.go

@@ -71,6 +71,14 @@ func RegisterRoutes(r *gin.Engine) {
 	// rather than under /api precisely so that none of those apply.
 	r.GET("/public/status/:pageId", RateLimitPublicStatus(), getPublicStatusPage)
 
+	// Ping endpoints for heartbeat monitors. The token is in the URL or the
+	// X-Vantage-Token header; see resolveHeartbeat for the shapes. Rate
+	// limiting is per token inside the handler.
+	for _, p := range []string{"/public/hb", "/public/hb/:a", "/public/hb/:a/:b"} {
+		r.GET(p, handleHeartbeat)
+		r.POST(p, handleHeartbeat)
+	}
+
 	apiGroup := r.Group("/api")
 	apiGroup.Use(auth.Middleware())
 	// Scope enforcement sits between authentication and the licence gate, and

server/internal/api/heartbeats.go

@@ -0,0 +1,176 @@
+package api
+
+import (
+	"errors"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/auth"
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/services"
+	"github.com/gin-gonic/gin"
+)
+
+// HeartbeatTokenHeader carries the ping token for callers who want it out of
+// URLs, and so out of their own proxies' access logs.
+const HeartbeatTokenHeader = "X-Vantage-Token"
+
+const heartbeatPathPrefix = "/public/hb/"
+
+func heartbeatKind(seg string) string {
+	switch seg {
+	case "":
+		return services.HeartbeatPing
+	case "start":
+		return services.HeartbeatStart
+	case "fail":
+		return services.HeartbeatFail
+	}
+	return ""
+}
+
+// resolveHeartbeat maps the three route shapes onto a token and a kind. a and b
+// are the first and second path segments after /public/hb. A token in the URL
+// wins over the header, so a URL copied from the UI behaves the same no matter
+// what headers a client adds.
+func resolveHeartbeat(header, a, b string) (string, string, bool) {
+	var token, kindSeg string
+	switch {
+	// a is a URL token unless a header is present, no second segment follows
+	// and a is itself a kind word (the header routes /public/hb/start|fail).
+	case a != "" && (header == "" || b != "" || heartbeatKind(a) == ""):
+		token, kindSeg = a, b
+	case header != "":
+		token, kindSeg = header, a
+	default:
+		return "", "", false
+	}
+	kind := heartbeatKind(kindSeg)
+	if token == "" || kind == "" {
+		return "", "", false
+	}
+	return token, kind, true
+}
+
+// MaskLogPath hides a heartbeat token in a request path before it is logged.
+// "start" and "fail" directly under the prefix are the header-token routes and
+// carry no secret, so they are left readable.
+func MaskLogPath(path string) string {
+	rest, found := strings.CutPrefix(path, heartbeatPathPrefix)
+	if !found {
+		return path
+	}
+	end := strings.IndexAny(rest, "/?")
+	if end < 0 {
+		end = len(rest)
+	}
+	seg := rest[:end]
+	if seg == "" || seg == "start" || seg == "fail" {
+		return path
+	}
+	return heartbeatPathPrefix + "***" + rest[end:]
+}
+
+// heartbeatLimitKey buckets requests by token, kind and wall-clock second. The
+// kind is part of the key so a fast job's success ping is not rejected for
+// landing in the same second as its /start, which would leave the run marked
+// started and page as never finished. The token is hashed so Redis never holds
+// the secret.
+func heartbeatLimitKey(token, kind string, now time.Time) string {
+	return "vantage:hbrl:" + services.HashHeartbeatToken(token) + ":" + kind + ":" + strconv.FormatInt(now.Unix(), 10)
+}
+
+// readHeartbeatBody keeps up to MaxHeartbeatBody bytes of a /fail body as the
+// incident message. Other kinds carry no message, so their bodies are ignored.
+func readHeartbeatBody(kind string, r io.Reader) string {
+	if kind != services.HeartbeatFail || r == nil {
+		return ""
+	}
+	b, _ := io.ReadAll(io.LimitReader(r, services.MaxHeartbeatBody))
+	return string(b)
+}
+
+func writeHeartbeatLimited(c *gin.Context) {
+	c.Header("Retry-After", "1")
+	c.String(http.StatusTooManyRequests, "too many requests")
+}
+
+// heartbeatAllowed admits one request per token and kind per second. A cron job pinging
+// in a loop should not become a write per request, and a leaked URL should not
+// be a way to hammer Mongo. Like the status page limiter it allows when Redis
+// is down: a missed ping pages someone. It runs inside the handler rather than
+// as middleware because the token may come from a header.
+func heartbeatAllowed(c *gin.Context, token, kind string) bool {
+	rdb := auth.Redis()
+	if rdb == nil {
+		return true
+	}
+	key := heartbeatLimitKey(token, kind, time.Now())
+	count, err := rdb.Incr(c.Request.Context(), key).Result()
+	if err != nil {
+		return true
+	}
+	if count == 1 {
+		rdb.Expire(c.Request.Context(), key, 2*time.Second)
+	}
+	return count <= 1
+}
+
+// handleHeartbeat records a push from a job. It is mounted on the gin root
+// under /public for the same reasons as the status page (see
+// getPublicStatusPage): no session, no token, no licence gate, and /public is
+// already routed to this server by every deployment.
+//
+// Unknown token, disabled monitor and an unresolvable path all answer the same
+// 404.
+func handleHeartbeat(c *gin.Context) {
+	token, kind, ok := resolveHeartbeat(c.GetHeader(HeartbeatTokenHeader), c.Param("a"), c.Param("b"))
+	if !ok {
+		c.String(http.StatusNotFound, "not found")
+		return
+	}
+	if !heartbeatAllowed(c, token, kind) {
+		writeHeartbeatLimited(c)
+		return
+	}
+	err := services.RecordHeartbeat(token, kind, readHeartbeatBody(kind, c.Request.Body), time.Now())
+	if errors.Is(err, services.ErrHeartbeatNotFound) {
+		c.String(http.StatusNotFound, "not found")
+		return
+	}
+	if err != nil {
+		log.Printf("heartbeat: %v", err)
+		c.String(http.StatusInternalServerError, "error")
+		return
+	}
+	c.String(http.StatusOK, "OK")
+}
+
+// rotateHeartbeatToken godoc
+//
+//	@Summary		Rotate a heartbeat monitor's ping token
+//	@Description	Issues a new token and invalidates the old ping URL immediately. The token is returned only in this response.
+//	@Tags			monitors
+//	@Produce		json
+//	@Param			id	path		string	true	"Monitor ID"
+//	@Success		200	{object}	object{heartbeat_token=string}
+//	@Failure		404	{object}	ErrorResponse
+//	@Failure		500	{object}	ErrorResponse
+//	@Security		cookieAuth
+//	@Security		bearerAuth
+//	@Router			/monitors/{id}/rotate-token [post]
+func rotateHeartbeatToken(c *gin.Context) {
+	tok, err := services.RotateHeartbeatToken(auth.InstanceID(c), c.Param("id"))
+	if errors.Is(err, services.ErrHeartbeatNotFound) {
+		c.JSON(http.StatusNotFound, gin.H{"error": "heartbeat monitor not found"})
+		return
+	}
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"heartbeat_token": tok})
+}

server/internal/api/heartbeats_test.go

@@ -0,0 +1,112 @@
+package api
+
+import (
+	"strings"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/services"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+)
+
+func TestResolveHeartbeat(t *testing.T) {
+	cases := []struct {
+		name, header, a, b string
+		token, kind        string
+		ok                 bool
+	}{
+		{"url ping", "", "tok", "", "tok", "ping", true},
+		{"url start", "", "tok", "start", "tok", "start", true},
+		{"url fail", "", "tok", "fail", "tok", "fail", true},
+		{"url bad kind", "", "tok", "explode", "", "", false},
+		{"header ping", "htok", "", "", "htok", "ping", true},
+		{"header start", "htok", "start", "", "htok", "start", true},
+		{"header fail", "htok", "fail", "", "htok", "fail", true},
+		{"url token wins over header", "htok", "tok", "", "tok", "ping", true},
+		{"url token and kind win over header", "htok", "tok", "fail", "tok", "fail", true},
+		{"nothing", "", "", "", "", "", false},
+		// Without a header, /public/hb/start is a token called "start": it
+		// resolves, and the lookup answers 404 like any unknown token.
+		{"bare start without header", "", "start", "", "start", "ping", true},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			tok, kind, ok := resolveHeartbeat(c.header, c.a, c.b)
+			if tok != c.token || kind != c.kind || ok != c.ok {
+				t.Fatalf("got (%q,%q,%v), want (%q,%q,%v)", tok, kind, ok, c.token, c.kind, c.ok)
+			}
+		})
+	}
+}
+
+// A ping URL is a credential. Anything that logs request paths must see a
+// masked one.
+func TestMaskLogPath(t *testing.T) {
+	cases := map[string]string{
+		"/public/hb/abc123":        "/public/hb/***",
+		"/public/hb/abc123/fail":   "/public/hb/***/fail",
+		"/public/hb/abc123?x=1":    "/public/hb/***?x=1",
+		"/public/hb/start":         "/public/hb/start",
+		"/public/hb/fail":          "/public/hb/fail",
+		"/public/hb":               "/public/hb",
+		"/public/status/page":      "/public/status/page",
+		"/api/monitors/abc/uptime": "/api/monitors/abc/uptime",
+	}
+	for in, want := range cases {
+		if got := MaskLogPath(in); got != want {
+			t.Errorf("MaskLogPath(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestHeartbeatUnresolvableIs404(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	r := gin.New()
+	r.POST("/public/hb/:a/:b", handleHeartbeat)
+	w := httptest.NewRecorder()
+	r.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/public/hb/abc/explode", nil))
+	if w.Code != http.StatusNotFound {
+		t.Fatalf("code = %d, want 404", w.Code)
+	}
+}
+
+func TestHeartbeatLimitKey(t *testing.T) {
+	now := time.Unix(1700000000, 0)
+	start := heartbeatLimitKey("tok", services.HeartbeatStart, now)
+	ping := heartbeatLimitKey("tok", services.HeartbeatPing, now)
+	if start == ping {
+		t.Fatalf("start and ping share a limiter key: %s", start)
+	}
+	if ping != heartbeatLimitKey("tok", services.HeartbeatPing, now.Add(500*time.Millisecond)) {
+		t.Fatal("same kind in the same second should share a key")
+	}
+	if strings.Contains(ping, "tok") {
+		t.Fatalf("key leaks the plaintext token: %s", ping)
+	}
+}
+
+func TestReadHeartbeatBody(t *testing.T) {
+	long := strings.Repeat("x", services.MaxHeartbeatBody+500)
+	if got := readHeartbeatBody(services.HeartbeatFail, strings.NewReader(long)); len(got) != services.MaxHeartbeatBody {
+		t.Fatalf("fail body not capped: %d", len(got))
+	}
+	if got := readHeartbeatBody(services.HeartbeatPing, strings.NewReader("hi")); got != "" {
+		t.Fatalf("ping body should be ignored, got %q", got)
+	}
+	if got := readHeartbeatBody(services.HeartbeatFail, nil); got != "" {
+		t.Fatalf("nil body: %q", got)
+	}
+}
+
+func TestHeartbeatRateLimitedWritesRetryAfter(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	w := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(w)
+	writeHeartbeatLimited(c)
+	if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" {
+		t.Fatalf("got %d retry-after %q", w.Code, w.Header().Get("Retry-After"))
+	}
+}

server/internal/api/monitors.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"errors"
 	"net/http"
 	"strconv"
 	"time"
@@ -21,6 +22,8 @@ func registerMonitorRoutes(g *gin.RouterGroup) {
 	g.GET("/monitors/:id/incidents", getMonitorIncidents)
 	g.GET("/monitors/:id/uptime", getMonitorUptime)
 	g.GET("/monitors/:id/samples", getMonitorSamples)
+	g.GET("/monitors/:id/servers", getMonitorServers)
+	g.POST("/monitors/:id/rotate-token", rotateHeartbeatToken)
 }
 
 // listMonitors godoc
@@ -85,6 +88,14 @@ func createMonitor(c *gin.Context) {
 	}
 	created, err := services.CreateMonitor(auth.InstanceID(c), &m, auth.ServerScope(c))
 	if err != nil {
+		if errors.Is(err, services.ErrInvalidMonitor) {
+			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+		if errors.Is(err, services.ErrMonitorOutOfScope) {
+			c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
+			return
+		}
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
@@ -131,9 +142,11 @@ func getMonitor(c *gin.Context) {
 //	@Accept			json
 //	@Produce		json
 //	@Param			id		path	string												true	"Monitor ID"
-//	@Param			body	body	object{name=string,group=string,type=string,target=models.MonitorTarget,interval_sec=int,runner=string,retries=int,enabled=bool,channel_ids=[]string}	true	"Fields to update"
+//	@Param			body	body	object{name=string,group=string,type=string,target=models.MonitorTarget,interval_sec=int,runner=string,retries=int,enabled=bool,channel_ids=[]string,for_sec=int}	true	"Fields to update"
 //	@Success		204
 //	@Failure		400	{object}	ErrorResponse
+//	@Failure		403	{object}	ErrorResponse
+//	@Failure		404	{object}	ErrorResponse
 //	@Failure		500	{object}	ErrorResponse
 //	@Security		cookieAuth
 //	@Security		bearerAuth
@@ -149,6 +162,7 @@ func updateMonitor(c *gin.Context) {
 		Retries     *int                  `json:"retries"`
 		Enabled     *bool                 `json:"enabled"`
 		ChannelIDs  *[]string             `json:"channel_ids"`
+		ForSec      *int                  `json:"for_sec"`
 	}
 	if err := c.ShouldBindJSON(&body); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
@@ -182,11 +196,26 @@ func updateMonitor(c *gin.Context) {
 	if body.ChannelIDs != nil {
 		upd["channel_ids"] = *body.ChannelIDs
 	}
+	if body.ForSec != nil {
+		upd["for_sec"] = *body.ForSec
+	}
 	if len(upd) == 0 {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "no fields to update"})
 		return
 	}
 	if err := services.UpdateMonitor(auth.InstanceID(c), c.Param("id"), upd, auth.ServerScope(c)); err != nil {
+		if errors.Is(err, services.ErrInvalidMonitor) {
+			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+		if errors.Is(err, services.ErrMonitorOutOfScope) {
+			c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
+			return
+		}
+		if errors.Is(err, services.ErrMonitorNotFound) {
+			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
+			return
+		}
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
@@ -199,12 +228,17 @@ func updateMonitor(c *gin.Context) {
 //	@Tags			monitors
 //	@Param			id	path	string	true	"Monitor ID"
 //	@Success		204
+//	@Failure		403	{object}	ErrorResponse
 //	@Failure		500	{object}	ErrorResponse
 //	@Security		cookieAuth
 //	@Security		bearerAuth
 //	@Router			/monitors/{id} [delete]
 func deleteMonitor(c *gin.Context) {
-	if err := services.DeleteMonitor(auth.InstanceID(c), c.Param("id")); err != nil {
+	if err := services.DeleteMonitor(auth.InstanceID(c), c.Param("id"), auth.ServerScope(c)); err != nil {
+		if errors.Is(err, services.ErrMonitorOutOfScope) {
+			c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
+			return
+		}
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
@@ -238,9 +272,63 @@ func getMonitorIncidents(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
+	visible, restricted, err := services.VisibleServerIDs(auth.InstanceID(c), auth.ServerScope(c))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	incidents = services.FilterByVisibleServer(incidents, func(i models.Incident) string { return i.ServerID }, visible, restricted)
 	c.JSON(http.StatusOK, incidents)
 }
 
+// getMonitorServers godoc
+//
+//	@Summary		List a metric monitor's per-server states
+//	@Tags			monitors
+//	@Produce		json
+//	@Param			id	path		string	true	"Monitor ID"
+//	@Success		200	{array}		models.MonitorServerState
+//	@Failure		404	{object}	ErrorResponse
+//	@Failure		500	{object}	ErrorResponse
+//	@Security		cookieAuth
+//	@Security		bearerAuth
+//	@Router			/monitors/{id}/servers [get]
+func getMonitorServers(c *gin.Context) {
+	instanceID := auth.InstanceID(c)
+	m, err := services.GetMonitor(instanceID, c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	if m == nil || m.Type != models.MonitorMetric {
+		c.JSON(http.StatusNotFound, gin.H{"error": "metric monitor not found"})
+		return
+	}
+	states, err := services.ListMonitorServerStates(instanceID, m.MonitorID)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	visible, restricted, err := services.VisibleServerIDs(instanceID, auth.ServerScope(c))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	states = services.FilterByVisibleServer(states, func(s models.MonitorServerState) string { return s.ServerID }, visible, restricted)
+
+	servers, err := services.ListServersFiltered(instanceID, m.Target.Selector)
+	if err == nil {
+		names := map[string]string{}
+		for _, s := range servers {
+			names[s.ServerID] = s.Hostname
+		}
+		for i := range states {
+			states[i].Hostname = names[states[i].ServerID]
+		}
+	}
+	c.JSON(http.StatusOK, states)
+}
+
 // getMonitorSamples godoc
 //
 //	@Summary		Get a monitor's individual check results

server/internal/api/scopes.go

@@ -99,14 +99,16 @@ var routeScopes = map[string]string{
 	"GET /api/runs/:runId/servers/:serverId/logs/stream": "workflows:read",
 
 	// Monitor and incident routes, registered by registerMonitorRoutes.
-	"GET /api/monitors":               "monitors:read",
-	"POST /api/monitors":              "monitors:write",
-	"GET /api/monitors/:id":           "monitors:read",
-	"PUT /api/monitors/:id":           "monitors:write",
-	"DELETE /api/monitors/:id":        "monitors:write",
-	"GET /api/monitors/:id/incidents": "monitors:read",
-	"GET /api/monitors/:id/uptime":    "monitors:read",
-	"GET /api/monitors/:id/samples":   "monitors:read",
+	"GET /api/monitors":                   "monitors:read",
+	"POST /api/monitors":                  "monitors:write",
+	"GET /api/monitors/:id":               "monitors:read",
+	"PUT /api/monitors/:id":               "monitors:write",
+	"DELETE /api/monitors/:id":            "monitors:write",
+	"GET /api/monitors/:id/incidents":     "monitors:read",
+	"GET /api/monitors/:id/uptime":        "monitors:read",
+	"GET /api/monitors/:id/samples":       "monitors:read",
+	"GET /api/monitors/:id/servers":       "monitors:read",
+	"POST /api/monitors/:id/rotate-token": "monitors:write",
 
 	// Channel routes, registered by registerChannelRoutes. Channels exist to
 	// serve alerts, so they share the monitors scope rather than getting their
@@ -238,6 +240,30 @@ var routesOutsideAPIGroup = map[string]bool{
 	"GET /api/secrets/:group/values": true,
 }
 
+// sessionOnlyRoutes are /api routes that an API token must never reach, so they
+// deliberately have no entry in routeScopes. RequireScopes already refuses any
+// unmapped route to a token; this set is what tells AssertScopeMapComplete the
+// absence is a decision rather than an omission.
+//
+// Every entry manages the caller's own second factors or proves the caller is
+// present. A token that could enrol a passkey, regenerate recovery codes or
+// satisfy step-up would turn one leaked token into a way around MFA itself.
+var sessionOnlyRoutes = map[string]bool{
+	"GET /api/me/mfa":                      true,
+	"POST /api/me/mfa/totp/setup":          true,
+	"POST /api/me/mfa/totp/confirm":        true,
+	"DELETE /api/me/mfa/totp":              true,
+	"POST /api/me/mfa/recovery/regenerate": true,
+	"POST /api/me/passkeys/begin":          true,
+	"POST /api/me/passkeys/finish":         true,
+	"PATCH /api/me/passkeys/:id":           true,
+	"DELETE /api/me/passkeys/:id":          true,
+	"POST /api/me/step-up":                 true,
+	"POST /api/me/step-up/webauthn/begin":  true,
+	"POST /api/me/step-up/webauthn/finish": true,
+	"DELETE /api/org/users/:id/mfa":        true,
+}
+
 // AssertScopeMapComplete fails boot when a registered /api route has no scope.
 //
 // Without it, adding a route silently makes it unreachable by every token, and
@@ -253,6 +279,10 @@ func AssertScopeMapComplete(r *gin.Engine) error {
 		if routesOutsideAPIGroup[route.Method+" "+route.Path] {
 			continue
 		}
+		// Session-only routes are refused to every token by having no scope.
+		if sessionOnlyRoutes[route.Method+" "+route.Path] {
+			continue
+		}
 		if _, ok := routeScopes[route.Method+" "+route.Path]; !ok {
 			missing = append(missing, route.Method+" "+route.Path)
 		}

server/internal/api/serverscope.go

@@ -292,13 +292,25 @@ var serverScopedRoutes = map[string]scopeDecl{
 	// then act on.
 	"DELETE /api/monitors/:id": fleetWide,
 
-	// A monitor's incidents, uptime rollups and recent samples are all about
-	// the monitored endpoint - status, latency, timestamps - and carry no
-	// server identifier at all; the runner is a field of the monitor
-	// document, which these do not return.
-	"GET /api/monitors/:id/incidents": exempt,
-	"GET /api/monitors/:id/uptime":    exempt,
-	"GET /api/monitors/:id/samples":   exempt,
+	// Rotating a ping token touches no server and returns only the token.
+	"POST /api/monitors/:id/rotate-token": fleetWide,
+
+	// Uptime rollups and recent samples are about the monitored endpoint -
+	// status, latency, timestamps - and carry no server identifier at all;
+	// the runner is a field of the monitor document, which these do not
+	// return.
+	"GET /api/monitors/:id/uptime":  exempt,
+	"GET /api/monitors/:id/samples": exempt,
+
+	// A metric monitor's incidents carry server_id, so a restricted token
+	// must not see an incident for a server outside its scope. Handled by
+	// services.FilterByVisibleServer rather than by the route classification
+	// itself.
+	"GET /api/monitors/:id/incidents": scoped,
+
+	// Per-server metric state names servers, so it is filtered to the ones the
+	// credential can see.
+	"GET /api/monitors/:id/servers": scoped,
 
 	// ---- notification channels ----
 
@@ -424,6 +436,23 @@ var serverScopedRoutes = map[string]scopeDecl{
 	"GET /api/patch-runs":                scoped,
 	"GET /api/patch-runs/:runId":         scoped,
 	"POST /api/patch-runs/:runId/cancel": scoped,
+
+	// Multi-factor authentication. These act on the caller's own factors (or,
+	// for the reset, on another member's), never on a server, and no API token
+	// can reach them at all - see sessionOnlyRoutes.
+	"GET /api/me/mfa":                      exempt,
+	"POST /api/me/mfa/totp/setup":          exempt,
+	"POST /api/me/mfa/totp/confirm":        exempt,
+	"DELETE /api/me/mfa/totp":              exempt,
+	"POST /api/me/mfa/recovery/regenerate": exempt,
+	"POST /api/me/passkeys/begin":          exempt,
+	"POST /api/me/passkeys/finish":         exempt,
+	"PATCH /api/me/passkeys/:id":           exempt,
+	"DELETE /api/me/passkeys/:id":          exempt,
+	"POST /api/me/step-up":                 exempt,
+	"POST /api/me/step-up/webauthn/begin":  exempt,
+	"POST /api/me/step-up/webauthn/finish": exempt,
+	"DELETE /api/org/users/:id/mfa":        exempt,
 }
 
 // AssertServerScopeMapComplete refuses to boot when any registered /api route

server/internal/api/serverscope_test.go

@@ -16,7 +16,7 @@ func TestServerScopeMapCoversEveryScopedRoute(t *testing.T) {
 		if _, ok := routeScopes[r]; ok {
 			continue
 		}
-		if routesOutsideAPIGroup[r] {
+		if routesOutsideAPIGroup[r] || sessionOnlyRoutes[r] {
 			continue
 		}
 		t.Errorf("route %q is declared in serverScopedRoutes but is not a registered route", r)

server/internal/auth/mfa_enrol.go

@@ -212,20 +212,17 @@ func HandleEnrolPasskeyFinish(c *gin.Context) {
 	}
 	cred, err := w.CreateCredential(waUser{handle: handle, name: t.Email}, *sessionData, parsed)
 	if err != nil {
+		logWebAuthnFailure(c, "enrolment registration", err)
 		c.JSON(http.StatusBadRequest, gin.H{"error": "that passkey could not be verified"})
 		return
 	}
-	if !cred.Flags.UserVerified {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "this passkey does not verify the user"})
-		return
-	}
 	transports := make([]string, 0, len(parsed.Response.Transports))
 	for _, tr := range parsed.Response.Transports {
 		transports = append(transports, string(tr))
 	}
 	if err := services.SavePasskey(t.InstanceID, t.UserID, body.Name,
 		cred.ID, cred.PublicKey, cred.Authenticator.AAGUID, cred.Authenticator.SignCount,
-		transports); err != nil {
+		transports, cred.Flags.BackupEligible, cred.Flags.BackupState); err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not save the passkey"})
 		return
 	}

server/internal/auth/passkey_login.go

@@ -44,7 +44,7 @@ func HandlePasskeyLoginBegin(c *gin.Context) {
 	// Discoverable login: no allowCredentials, so the authenticator offers
 	// whichever resident credential it holds for this RP ID.
 	options, sessionData, err := w.BeginDiscoverableLogin(
-		webauthn.WithUserVerification(protocol.VerificationRequired))
+		webauthn.WithUserVerification(protocol.VerificationPreferred))
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not start sign-in"})
 		return
@@ -85,6 +85,7 @@ func HandlePasskeyLoginFinish(c *gin.Context) {
 	// that owner's.
 	cred, err := finishAssertion(c, instanceID, "", "")
 	if err != nil {
+		logWebAuthnFailure(c, "passwordless assertion", err)
 		c.JSON(http.StatusUnauthorized, gin.H{
 			"error": "that passkey could not be verified", "code": "invalid_assertion",
 		})
@@ -104,6 +105,6 @@ func HandlePasskeyLoginFinish(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "session save failed"})
 		return
 	}
-	_ = services.TouchPasskey(instanceID, cred.ID, cred.Authenticator.SignCount)
+	_ = services.TouchPasskey(instanceID, cred.ID, cred.Authenticator.SignCount, cred.Flags.BackupEligible, cred.Flags.BackupState)
 	c.JSON(http.StatusOK, gin.H{"ok": true})
 }

server/internal/auth/webauthn.go

@@ -6,6 +6,7 @@ import (
 	"crypto/subtle"
 	"encoding/json"
 	"errors"
+	"log"
 	"net"
 	"net/http"
 	"time"
@@ -42,6 +43,23 @@ func rpConfig(c *gin.Context) (string, string) {
 	return rpID, scheme + "://" + host
 }
 
+// logWebAuthnFailure records why a ceremony was refused. The response stays
+// deliberately vague, so without this the only evidence of a misconfigured
+// relying party - most often a proxy that terminates TLS without passing
+// X-Forwarded-Proto: https - is a user reporting that their passkey "could not
+// be verified". Nothing logged here is secret: RP ID, origins, and the
+// library's error, whose DevInfo names expected and received values.
+func logWebAuthnFailure(c *gin.Context, stage string, err error) {
+	rpID, origin := rpConfig(c)
+	detail := err.Error()
+	var perr *protocol.Error
+	if errors.As(err, &perr) && perr.DevInfo != "" {
+		detail += " (" + perr.DevInfo + ")"
+	}
+	log.Printf("webauthn: %s refused: rp_id=%q expected_origin=%q request_origin=%q x_forwarded_proto=%q: %s",
+		stage, rpID, origin, c.GetHeader("Origin"), c.GetHeader("X-Forwarded-Proto"), detail)
+}
+
 func webAuthnFor(c *gin.Context) (*webauthn.WebAuthn, error) {
 	rpID, origin := rpConfig(c)
 	return webauthn.New(&webauthn.Config{
@@ -49,8 +67,14 @@ func webAuthnFor(c *gin.Context) (*webauthn.WebAuthn, error) {
 		RPID:          rpID,
 		RPOrigins:     []string{origin},
 		AuthenticatorSelection: protocol.AuthenticatorSelection{
-			ResidentKey:      protocol.ResidentKeyRequirementRequired,
-			UserVerification: protocol.VerificationRequired,
+			ResidentKey: protocol.ResidentKeyRequirementRequired,
+			// Preferred, not required, and not checked on the result. Requiring
+			// it made password managers such as NordPass ask for their master
+			// password on every use, which other sites avoid by preferring it.
+			// The cost is deliberate: an authenticator that skips verification
+			// makes a passkey possession-only, so passwordless sign-in and
+			// step-up then rest on the device or vault being unlocked.
+			UserVerification: protocol.VerificationPreferred,
 		},
 	})
 }
@@ -74,6 +98,10 @@ func toLibCredential(c models.WebAuthnCredential) webauthn.Credential {
 		ID:              c.CredentialID,
 		PublicKey:       c.PublicKey,
 		AttestationType: "none",
+		Flags: webauthn.CredentialFlags{
+			BackupEligible: c.BackupEligible != nil && *c.BackupEligible,
+			BackupState:    c.BackupState,
+		},
 		Authenticator: webauthn.Authenticator{
 			AAGUID:    c.AAGUID,
 			SignCount: c.SignCount,
@@ -81,6 +109,20 @@ func toLibCredential(c models.WebAuthnCredential) webauthn.Credential {
 	}
 }
 
+// backupEligibleFor is the eligibility to validate an assertion against.
+//
+// A recorded value always stands, so the library still refuses a credential
+// whose eligibility changed. A row registered before the flag was stored has no
+// baseline at all, and comparing against a zero value refused every synced
+// passkey; for those the flag in the signed authenticator data is adopted, and
+// TouchPasskey records it after the assertion verifies.
+func backupEligibleFor(stored *bool, observed bool) bool {
+	if stored != nil {
+		return *stored
+	}
+	return observed
+}
+
 func saveCeremony(ctx context.Context, data *webauthn.SessionData) (string, error) {
 	id, err := randomHex(32)
 	if err != nil {
@@ -146,7 +188,7 @@ func HandleMFAWebAuthnBegin(c *gin.Context) {
 		return
 	}
 	options, sessionData, err := w.BeginLogin(waUser{handle: handle, name: t.Email, credentials: lib},
-		webauthn.WithUserVerification(protocol.VerificationRequired))
+		webauthn.WithUserVerification(protocol.VerificationPreferred))
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not start verification"})
 		return
@@ -176,6 +218,7 @@ func HandleMFAWebAuthnFinish(c *gin.Context) {
 	}
 	cred, err := finishAssertion(c, t.InstanceID, t.UserID, t.Email)
 	if err != nil {
+		logWebAuthnFailure(c, "second-factor assertion", err)
 		left, ferr := FailTicket(c.Request.Context(), ticketID)
 		services.LogEvent(t.InstanceID, "mfa.failed", t.Email, "", "", "factor=webauthn")
 		if ferr != nil || left == 0 {
@@ -187,7 +230,7 @@ func HandleMFAWebAuthnFinish(c *gin.Context) {
 		})
 		return
 	}
-	_ = services.TouchPasskey(t.InstanceID, cred.ID, cred.Authenticator.SignCount)
+	_ = services.TouchPasskey(t.InstanceID, cred.ID, cred.Authenticator.SignCount, cred.Flags.BackupEligible, cred.Flags.BackupState)
 	u, err := services.GetUserInInstance(t.InstanceID, t.UserID)
 	if err != nil {
 		c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid credentials"})
@@ -287,20 +330,17 @@ func HandleRegisterPasskeyFinish(c *gin.Context) {
 	}
 	cred, err := w.CreateCredential(waUser{handle: handle, name: sess.Email}, *sessionData, parsed)
 	if err != nil {
+		logWebAuthnFailure(c, "registration", err)
 		c.JSON(http.StatusBadRequest, gin.H{"error": "that passkey could not be verified"})
 		return
 	}
-	if !cred.Flags.UserVerified {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "this passkey does not verify the user"})
-		return
-	}
 	transports := make([]string, 0, len(parsed.Response.Transports))
 	for _, t := range parsed.Response.Transports {
 		transports = append(transports, string(t))
 	}
 	if err := services.SavePasskey(sess.InstanceID, sess.UserID, body.Name,
 		cred.ID, cred.PublicKey, cred.Authenticator.AAGUID, cred.Authenticator.SignCount,
-		transports); err != nil {
+		transports, cred.Flags.BackupEligible, cred.Flags.BackupState); err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not save the passkey"})
 		return
 	}
@@ -350,7 +390,7 @@ func HandleStepUpWebAuthnBegin(c *gin.Context) {
 		return
 	}
 	options, sessionData, err := w.BeginLogin(waUser{handle: handle, name: sess.Email, credentials: lib},
-		webauthn.WithUserVerification(protocol.VerificationRequired))
+		webauthn.WithUserVerification(protocol.VerificationPreferred))
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not start verification"})
 		return
@@ -382,11 +422,12 @@ func HandleStepUpWebAuthnFinish(c *gin.Context) {
 	sess := GetSessionFromContext(c)
 	cred, err := finishAssertion(c, sess.InstanceID, sess.UserID, sess.Email)
 	if err != nil {
+		logWebAuthnFailure(c, "step-up assertion", err)
 		services.LogEvent(sess.InstanceID, "step_up.failed", sess.Email, "", "", "factor=webauthn")
 		c.JSON(http.StatusUnauthorized, gin.H{"error": "that passkey could not be verified", "code": "invalid_assertion"})
 		return
 	}
-	_ = services.TouchPasskey(sess.InstanceID, cred.ID, cred.Authenticator.SignCount)
+	_ = services.TouchPasskey(sess.InstanceID, cred.ID, cred.Authenticator.SignCount, cred.Flags.BackupEligible, cred.Flags.BackupState)
 	if err := TouchStepUpFromRequest(c); err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not record re-authentication"})
 		return
@@ -396,7 +437,7 @@ func HandleStepUpWebAuthnFinish(c *gin.Context) {
 }
 
 // finishAssertion is shared by second-factor sign-in, passwordless sign-in and
-// step-up, so the verification rules (user verification, clone detection,
+// step-up, so the verification rules (backup eligibility, clone detection,
 // instance scope) exist once.
 func finishAssertion(c *gin.Context, instanceID, userID, email string) (*webauthn.Credential, error) {
 	var body struct {
@@ -429,7 +470,10 @@ func finishAssertion(c *gin.Context, instanceID, userID, email string) (*webauth
 	if err != nil {
 		return nil, err
 	}
-	user := waUser{handle: handle, name: email, credentials: []webauthn.Credential{toLibCredential(*stored)}}
+	libCred := toLibCredential(*stored)
+	libCred.Flags.BackupEligible = backupEligibleFor(stored.BackupEligible,
+		parsed.Response.AuthenticatorData.Flags.HasBackupEligible())
+	user := waUser{handle: handle, name: email, credentials: []webauthn.Credential{libCred}}
 	var cred *webauthn.Credential
 	if userID == "" {
 		// Discoverable (passwordless) ceremony: the session carries no user,
@@ -448,9 +492,6 @@ func finishAssertion(c *gin.Context, instanceID, userID, email string) (*webauth
 	if err != nil {
 		return nil, err
 	}
-	if !cred.Flags.UserVerified {
-		return nil, errors.New("user verification was not performed")
-	}
 	// A counter that fails to advance is the library's clone signal. Zero on
 	// both sides means the authenticator does not keep one, which is normal.
 	if cred.Authenticator.CloneWarning {

server/internal/auth/webauthn_test.go

@@ -2,6 +2,7 @@ package auth
 
 import (
 	"bytes"
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
 	"net/http/httptest"
 	"testing"
 
@@ -50,3 +51,40 @@ func TestDiscoverableHandleMatches(t *testing.T) {
 		t.Fatal("an empty handle must never match")
 	}
 }
+
+// Synced passkeys (iCloud Keychain, Google Password Manager, 1Password) report
+// backup eligibility, and go-webauthn refuses any assertion whose flag differs
+// from the stored credential's. Not storing it made every synced passkey fail
+// with "Backup Eligible flag inconsistency".
+func TestLibCredentialCarriesStoredBackupFlags(t *testing.T) {
+	yes := true
+	lc := toLibCredential(models.WebAuthnCredential{BackupEligible: &yes, BackupState: true})
+	if !lc.Flags.BackupEligible || !lc.Flags.BackupState {
+		t.Fatalf("flags not carried: %+v", lc.Flags)
+	}
+}
+
+func TestBackupEligibleFor(t *testing.T) {
+	yes, no := true, false
+	cases := []struct {
+		name     string
+		stored   *bool
+		observed bool
+		want     bool
+	}{
+		// Registered before the flag was recorded: no baseline, adopt it.
+		{"legacy row, synced passkey", nil, true, true},
+		{"legacy row, hardware key", nil, false, false},
+		// Recorded: the stored value stands, so the library still refuses a
+		// credential whose eligibility genuinely changed.
+		{"recorded eligible", &yes, true, true},
+		{"recorded not eligible, now claims eligible", &no, true, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := backupEligibleFor(tc.stored, tc.observed); got != tc.want {
+				t.Fatalf("backupEligibleFor = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

server/internal/mcp/tools_create.go

@@ -118,6 +118,25 @@ func buildMonitor(args map[string]any) (models.Monitor, error) {
 	if b, ok := rawTarget["insecure"].(bool); ok {
 		target.Insecure = b
 	}
+	target.Metric = stringArg(rawTarget, "metric")
+	target.Mount = stringArg(rawTarget, "mount")
+	if n, ok := rawTarget["threshold"].(float64); ok {
+		target.Threshold = n
+	}
+	if n, ok := rawTarget["period_sec"].(float64); ok {
+		target.PeriodSec = int(n)
+	}
+	if n, ok := rawTarget["grace_sec"].(float64); ok {
+		target.GraceSec = int(n)
+	}
+	if sel, ok := rawTarget["selector"].(map[string]any); ok {
+		target.Selector = map[string]string{}
+		for k, v := range sel {
+			if s, ok := v.(string); ok {
+				target.Selector[k] = s
+			}
+		}
+	}
 
 	switch monitorType {
 	case models.MonitorHTTP, models.MonitorTLS:
@@ -131,6 +150,9 @@ func buildMonitor(args map[string]any) (models.Monitor, error) {
 		if monitorType == models.MonitorTCP && target.Port == 0 {
 			return models.Monitor{}, fmt.Errorf("target.port is required for a tcp monitor")
 		}
+	case models.MonitorHeartbeat, models.MonitorMetric:
+		// Full validation (validateHeartbeat / validateMetric) runs inside
+		// services.CreateMonitor; nothing further is required here.
 	default:
 		return models.Monitor{}, fmt.Errorf("unknown monitor type %q", monitorType)
 	}
@@ -150,6 +172,10 @@ func buildMonitor(args map[string]any) (models.Monitor, error) {
 	if n, ok := args["interval_sec"].(float64); ok && int(n) > 0 {
 		interval = int(n)
 	}
+	forSec := 0
+	if n, ok := args["for_sec"].(float64); ok && int(n) > 0 {
+		forSec = int(n)
+	}
 
 	return models.Monitor{
 		Name:        name,
@@ -157,6 +183,7 @@ func buildMonitor(args map[string]any) (models.Monitor, error) {
 		Type:        monitorType,
 		Target:      target,
 		IntervalSec: interval,
+		ForSec:      forSec,
 		// Never armed on creation. A monitor that started enabled would begin
 		// alerting real people the moment a model invented it, and creating
 		// must stay a separate decision from acting.
@@ -257,10 +284,11 @@ func init() {
 		Name: "create_monitor",
 		Args: []ToolArg{
 			{Name: "name", Type: ArgString, Description: "Name for the monitor.", Required: true},
-			{Name: "type", Type: ArgString, Description: "Check type: http, tcp, icmp or tls.", Required: true},
-			{Name: "target", Type: ArgObject, Description: "What to check. http/tls take url; tcp/icmp take host, and tcp also port. Optional: method, keyword, expected_status, tls_warn_days, insecure.", Required: true},
+			{Name: "type", Type: ArgString, Description: "Check type: http, tcp, icmp, tls, heartbeat or metric.", Required: true},
+			{Name: "target", Type: ArgObject, Description: "What to check. http/tls take url; tcp/icmp take host, and tcp also port. Optional: method, keyword, expected_status, tls_warn_days, insecure. heartbeat takes period_sec and optional grace_sec; metric takes metric (disk_pct, disk_free_gb, mem_pct, load_per_core, unit_failed, container_unhealthy, reboot_pending_days, agent_offline_min), threshold, optional mount and selector (tag map).", Required: true},
 			{Name: "group", Type: ArgString, Description: "Optional group name to file the monitor under."},
 			{Name: "interval_sec", Type: ArgInteger, Description: "Seconds between checks; defaults to 60."},
+			{Name: "for_sec", Type: ArgInteger, Description: "For a metric monitor, how long the condition must hold before a server counts as down."},
 		},
 		Write: true,
 		Scope: "monitors:write",
@@ -277,12 +305,17 @@ func init() {
 				return nil, fmt.Errorf("could not create the monitor: %w", err)
 			}
 			LogCreated(c, "monitor", created.MonitorID, created.Name)
-			return map[string]any{
+			out := map[string]any{
 				"monitor_id": created.MonitorID,
 				"name":       created.Name,
 				"enabled":    false,
 				"note":       "Created disabled. Enable it in Vantage to start checking.",
-			}, nil
+			}
+			if created.HeartbeatToken != "" {
+				out["heartbeat_token"] = created.HeartbeatToken
+				out["heartbeat_token_note"] = "Shown once."
+			}
+			return out, nil
 		},
 	})
 }

server/internal/mcp/tools_health.go

@@ -187,6 +187,12 @@ func init() {
 				}
 			}
 
+			// Metric incidents name a server, so a server-scoped token sees only
+			// the ones on servers it can see, matching the REST endpoint.
+			visible, restricted, err := services.VisibleServerIDs(c.InstanceID, c.TokenScope)
+			if err != nil {
+				return nil, fmt.Errorf("could not resolve visible servers: %w", err)
+			}
 			out := []incidentSummary{}
 			for _, mid := range monitorIDs {
 				if len(out) >= int(limit) {
@@ -196,6 +202,7 @@ func init() {
 				if err != nil {
 					return nil, fmt.Errorf("could not list incidents: %w", err)
 				}
+				incidents = services.FilterByVisibleServer(incidents, func(i models.Incident) string { return i.ServerID }, visible, restricted)
 				name := mid
 				if monitorNames != nil {
 					if n, ok := monitorNames[mid]; ok {

server/internal/metricsched/scheduler.go

@@ -0,0 +1,41 @@
+// Package metricsched sweeps passive monitors - heartbeats and, from phase 2,
+// metric monitors - on a fixed tick. Nothing here runs a check: it reads what
+// pings and agents already delivered and decides what is overdue or breaching.
+package metricsched
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/services"
+)
+
+const tick = 30 * time.Second
+
+func Start(ctx context.Context) {
+	go func() {
+		t := time.NewTicker(tick)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case now := <-t.C:
+				sweep(now)
+			}
+		}
+	}()
+}
+
+// sweep recovers per sweep so one bad document cannot stop alerting for the
+// whole instance until the next deploy.
+func sweep(now time.Time) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("metricsched: sweep panic: %v", r)
+		}
+	}()
+	services.SweepHeartbeats(now)
+	services.SweepMetricMonitors(now)
+}

server/internal/models/monitor.go

@@ -7,12 +7,21 @@ import (
 )
 
 const (
-	MonitorHTTP = "http"
-	MonitorTCP  = "tcp"
-	MonitorICMP = "icmp"
-	MonitorTLS  = "tls"
+	MonitorHTTP      = "http"
+	MonitorTCP       = "tcp"
+	MonitorICMP      = "icmp"
+	MonitorTLS       = "tls"
+	MonitorMetric    = "metric"
+	MonitorHeartbeat = "heartbeat"
 )
 
+// IsPassiveMonitor reports whether a monitor type is evaluated from data that
+// arrives (a ping, an agent report) rather than by running a check. Passive
+// monitors are never handed to monitorsched or to an agent.
+func IsPassiveMonitor(t string) bool {
+	return t == MonitorMetric || t == MonitorHeartbeat
+}
+
 const (
 	StatusUp      = "up"
 	StatusDown    = "down"
@@ -38,6 +47,14 @@ type MonitorTarget struct {
 	Keyword        string `bson:"keyword,omitempty" json:"keyword,omitempty"`
 	TLSWarnDays    int    `bson:"tls_warn_days,omitempty" json:"tls_warn_days,omitempty"`
 	Insecure       bool   `bson:"insecure,omitempty" json:"insecure,omitempty"`
+	// Metric monitors.
+	Selector  map[string]string `bson:"selector,omitempty" json:"selector,omitempty"`
+	Metric    string            `bson:"metric,omitempty" json:"metric,omitempty"`
+	Threshold float64           `bson:"threshold,omitempty" json:"threshold,omitempty"`
+	Mount     string            `bson:"mount,omitempty" json:"mount,omitempty"`
+	// Heartbeat monitors.
+	PeriodSec int `bson:"period_sec,omitempty" json:"period_sec,omitempty"`
+	GraceSec  int `bson:"grace_sec,omitempty" json:"grace_sec,omitempty"`
 }
 
 type MonitorState struct {
@@ -48,6 +65,8 @@ type MonitorState struct {
 	CertExpiryAt   *time.Time `bson:"cert_expiry_at,omitempty" json:"cert_expiry_at,omitempty"`
 	Fails          int        `bson:"fails" json:"fails"`
 	LastNotifiedAt *time.Time `bson:"last_notified_at,omitempty" json:"last_notified_at,omitempty"`
+	LastPingAt     *time.Time `bson:"last_ping_at,omitempty" json:"last_ping_at,omitempty"`
+	StartedAt      *time.Time `bson:"started_at,omitempty" json:"started_at,omitempty"`
 }
 
 type Monitor struct {
@@ -68,6 +87,13 @@ type Monitor struct {
 	ChannelIDs  []string      `bson:"channel_ids,omitempty" json:"channel_ids,omitempty"`
 	State       MonitorState  `bson:"state" json:"state"`
 	CreatedAt   time.Time     `bson:"created_at" json:"created_at"`
+	// ForSec is how long a metric condition must hold before a server is down.
+	ForSec int `bson:"for_sec,omitempty" json:"for_sec,omitempty"`
+	// HeartbeatTokenHash is the SHA-256 of the ping token. The token itself is
+	// shown once, on create or rotate, and never stored.
+	HeartbeatTokenHash string `bson:"heartbeat_token_hash,omitempty" json:"-"`
+	// HeartbeatToken is the plaintext token, set only on the create response.
+	HeartbeatToken string `bson:"-" json:"heartbeat_token,omitempty"`
 }
 
 type Incident struct {
@@ -77,6 +103,22 @@ type Incident struct {
 	StartedAt  time.Time  `bson:"started_at" json:"started_at"`
 	ResolvedAt *time.Time `bson:"resolved_at,omitempty" json:"resolved_at,omitempty"`
 	Cause      string     `bson:"cause,omitempty" json:"cause,omitempty"`
+	// ServerID is set only for metric monitors, which keep one incident per
+	// breaching server.
+	ServerID string `bson:"server_id,omitempty" json:"server_id,omitempty"`
+}
+
+// MonitorServerState is one metric monitor's view of one matching server.
+type MonitorServerState struct {
+	InstanceID  string     `bson:"instance_id" json:"instance_id"`
+	MonitorID   string     `bson:"monitor_id" json:"monitor_id"`
+	ServerID    string     `bson:"server_id" json:"server_id"`
+	Hostname    string     `bson:"-" json:"hostname,omitempty"`
+	Status      string     `bson:"status" json:"status"`
+	BreachSince *time.Time `bson:"breach_since,omitempty" json:"breach_since,omitempty"`
+	Value       float64    `bson:"value" json:"value"`
+	Message     string     `bson:"message,omitempty" json:"message,omitempty"`
+	UpdatedAt   time.Time  `bson:"updated_at" json:"updated_at"`
 }
 
 // MonitorSample is one check result, kept only long enough to draw the

server/internal/models/server.go

@@ -48,6 +48,8 @@ type Inventory struct {
 	// BootTime is the host's last reported boot time, stored on every report
 	// that carries one so a patch reboot can be proven by a changed boot.
 	BootTime *time.Time `bson:"boot_time,omitempty" json:"boot_time,omitempty"`
+	// RebootRequiredSince is when the host first reported a pending reboot.
+	RebootRequiredSince *time.Time `bson:"reboot_required_since,omitempty" json:"reboot_required_since,omitempty"`
 }
 
 type Server struct {

server/internal/models/user_mfa.go

@@ -55,6 +55,14 @@ type WebAuthnCredential struct {
 	AAGUID       []byte   `bson:"aaguid" json:"-"`
 	Transports   []string `bson:"transports,omitempty" json:"transports,omitempty"`
 
+	// BackupEligible is whether the credential can sync between devices. It
+	// never changes for a real credential, and go-webauthn refuses an assertion
+	// whose flag differs from this one, so it must be stored. Nil only on rows
+	// registered before it was recorded; see auth.backupEligibleFor.
+	BackupEligible *bool `bson:"backup_eligible,omitempty" json:"-"`
+	// BackupState is whether it is currently synced. It may change.
+	BackupState bool `bson:"backup_state" json:"-"`
+
 	// CredentialIDHex is the browser-facing identifier for rename and delete.
 	// The raw bytes never reach a URL.
 	CredentialIDHex string `bson:"credential_id_hex" json:"id"`

server/internal/notify/dispatch.go

@@ -17,6 +17,7 @@ const TypePatch = "patch"
 
 type Event struct {
 	MonitorName string
+	ServerName  string
 	Type        string
 	OldStatus   string
 	NewStatus   string
@@ -42,7 +43,13 @@ func (e Event) title() string {
 		}
 		s = fmt.Sprintf("[Vantage] Server %s %s", e.MonitorName, verb)
 	} else {
-		s = fmt.Sprintf("[Vantage] %s (%s) %s", e.MonitorName, e.Type, verb)
+		name := e.MonitorName
+		if e.ServerName != "" {
+			name = fmt.Sprintf("%s (%s) on %s", e.MonitorName, e.Type, e.ServerName)
+		} else {
+			name = fmt.Sprintf("%s (%s)", e.MonitorName, e.Type)
+		}
+		s = fmt.Sprintf("[Vantage] %s %s", name, verb)
 	}
 	if e.Message != "" {
 		s += ": " + e.Message

server/internal/notify/dispatch_title_test.go

@@ -0,0 +1,24 @@
+package notify
+
+import (
+	"strings"
+	"testing"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+)
+
+func TestTitleIncludesServerName(t *testing.T) {
+	ev := Event{MonitorName: "Disk full", Type: models.MonitorMetric, NewStatus: models.StatusDown, ServerName: "web-01", Message: "/var 94.2% used"}
+	got := ev.title()
+	want := "[Vantage] Disk full (metric) on web-01 is DOWN: /var 94.2% used"
+	if got != want {
+		t.Fatalf("title = %q, want %q", got, want)
+	}
+}
+
+func TestTitleWithoutServerNameUnchanged(t *testing.T) {
+	ev := Event{MonitorName: "Site", Type: models.MonitorHTTP, NewStatus: models.StatusUp}
+	if got := ev.title(); strings.Contains(got, " on ") || got != "[Vantage] Site (http) recovered" {
+		t.Fatalf("title = %q", got)
+	}
+}

server/internal/notify/http.go

@@ -33,14 +33,18 @@ func dispatchWebhook(ch models.NotificationChannel, ev Event) error {
 	if target == "" {
 		return fmt.Errorf("webhook: missing url")
 	}
-	return postJSON(target, map[string]any{
+	payload := map[string]any{
 		"monitor":    ev.MonitorName,
 		"type":       ev.Type,
 		"old_status": ev.OldStatus,
 		"new_status": ev.NewStatus,
 		"message":    ev.Message,
 		"time":       ev.Time.Format(time.RFC3339),
-	})
+	}
+	if ev.ServerName != "" {
+		payload["server_name"] = ev.ServerName
+	}
+	return postJSON(target, payload)
 }
 
 func dispatchDiscord(ch models.NotificationChannel, ev Event) error {

server/internal/notify/smtp.go

@@ -27,7 +27,7 @@ func dispatchSMTP(ch models.NotificationChannel, ev Event) error {
 	}
 
 	return sender.SendMonitorAlert(to, mail.MonitorEvent{
-		MonitorName: ev.MonitorName,
+		MonitorName: smtpAlertName(ev),
 		Type:        ev.Type,
 		OldStatus:   ev.OldStatus,
 		NewStatus:   ev.NewStatus,
@@ -36,3 +36,13 @@ func dispatchSMTP(ch models.NotificationChannel, ev Event) error {
 		Down:        ev.NewStatus == models.StatusDown,
 	})
 }
+
+// smtpAlertName is the name shown in the alert email. mail.MonitorEvent (in
+// vantage-shared) has no ServerName field, so the server is folded into the
+// name the same way title() folds it into the notification title.
+func smtpAlertName(ev Event) string {
+	if ev.ServerName == "" {
+		return ev.MonitorName
+	}
+	return fmt.Sprintf("%s on %s", ev.MonitorName, ev.ServerName)
+}

server/internal/notify/smtp_test.go

@@ -0,0 +1,17 @@
+package notify
+
+import "testing"
+
+func TestSMTPAlertNameIncludesServerName(t *testing.T) {
+	ev := Event{MonitorName: "Disk full", ServerName: "web-01"}
+	if got, want := smtpAlertName(ev), "Disk full on web-01"; got != want {
+		t.Fatalf("smtpAlertName = %q, want %q", got, want)
+	}
+}
+
+func TestSMTPAlertNameWithoutServerNameUnchanged(t *testing.T) {
+	ev := Event{MonitorName: "Site"}
+	if got, want := smtpAlertName(ev), "Site"; got != want {
+		t.Fatalf("smtpAlertName = %q, want %q", got, want)
+	}
+}

server/internal/services/heartbeats.go

@@ -0,0 +1,237 @@
+package services
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
+)
+
+const (
+	HeartbeatPing  = "ping"
+	HeartbeatStart = "start"
+	HeartbeatFail  = "fail"
+)
+
+// MaxHeartbeatBody bounds what a /fail request can put into an incident cause
+// and a notification. A job's stderr can be megabytes; the first kilobyte is
+// what a human reads.
+const MaxHeartbeatBody = 1024
+
+const defaultHeartbeatGraceSec = 300
+
+var ErrHeartbeatNotFound = errors.New("heartbeat not found")
+
+// ErrInvalidMonitor marks a validation failure, which handlers answer with 400.
+var ErrInvalidMonitor = errors.New("invalid monitor")
+
+func NewHeartbeatToken() (string, string, error) {
+	raw := make([]byte, 24)
+	if _, err := rand.Read(raw); err != nil {
+		return "", "", err
+	}
+	tok := base64.RawURLEncoding.EncodeToString(raw)
+	return tok, HashHeartbeatToken(tok), nil
+}
+
+func HashHeartbeatToken(token string) string {
+	sum := sha256.Sum256([]byte(token))
+	return hex.EncodeToString(sum[:])
+}
+
+func validateHeartbeat(t *models.MonitorTarget) error {
+	if t.PeriodSec < 60 {
+		return fmt.Errorf("%w: period_sec must be at least 60", ErrInvalidMonitor)
+	}
+	if t.GraceSec < 0 {
+		return fmt.Errorf("%w: grace_sec must not be negative", ErrInvalidMonitor)
+	}
+	if t.GraceSec == 0 {
+		t.GraceSec = defaultHeartbeatGraceSec
+	}
+	return nil
+}
+
+func failMessage(body string) string {
+	body = strings.TrimSpace(body)
+	if len(body) > MaxHeartbeatBody {
+		body = body[:MaxHeartbeatBody]
+	}
+	if body == "" {
+		return "reported failure"
+	}
+	return "reported failure: " + body
+}
+
+// heartbeatVerdict decides whether a heartbeat is overdue. A heartbeat that has
+// never pinged is never down: the clock starts at the first ping, so creating
+// one before the job is deployed does not page anyone.
+func heartbeatVerdict(m models.Monitor, now time.Time) (bool, string) {
+	grace := time.Duration(m.Target.GraceSec) * time.Second
+	if m.State.StartedAt != nil && now.After(m.State.StartedAt.Add(grace)) {
+		return true, fmt.Sprintf("started %s, never finished", m.State.StartedAt.UTC().Format(time.RFC3339))
+	}
+	if m.State.LastPingAt == nil {
+		return false, ""
+	}
+	deadline := m.State.LastPingAt.Add(time.Duration(m.Target.PeriodSec)*time.Second + grace)
+	if now.After(deadline) {
+		return true, fmt.Sprintf("no ping since %s", m.State.LastPingAt.UTC().Format(time.RFC3339))
+	}
+	return false, ""
+}
+
+// RecordHeartbeat applies one ping. The state change is a single
+// FindOneAndUpdate on the token hash, so two pings racing each other cannot
+// both read the old state and lose one of the writes.
+func RecordHeartbeat(token, kind, body string, now time.Time) error {
+	ctx, cancel := monCtx()
+	defer cancel()
+
+	filter := bson.M{"heartbeat_token_hash": HashHeartbeatToken(token), "type": models.MonitorHeartbeat, "enabled": true}
+	set := bson.M{"state.last_check_at": now}
+	unset := bson.M{}
+	var next, message string
+	switch kind {
+	case HeartbeatStart:
+		set["state.started_at"] = now
+	case HeartbeatPing:
+		set["state.last_ping_at"] = now
+		set["state.status"] = models.StatusUp
+		set["state.message"] = ""
+		unset["state.started_at"] = ""
+		next = models.StatusUp
+	case HeartbeatFail:
+		message = failMessage(body)
+		set["state.last_ping_at"] = now
+		set["state.status"] = models.StatusDown
+		set["state.message"] = message
+		unset["state.started_at"] = ""
+		next = models.StatusDown
+	default:
+		return fmt.Errorf("unknown heartbeat kind %q", kind)
+	}
+	upd := bson.M{"$set": set}
+	if len(unset) > 0 {
+		upd["$unset"] = unset
+	}
+
+	// ReturnDocument Before: the previous status and started_at are what the
+	// transition and the duration need.
+	var before models.Monitor
+	err := db.Col("monitors").FindOneAndUpdate(ctx, filter, upd,
+		options.FindOneAndUpdate().SetReturnDocument(options.Before)).Decode(&before)
+	if errors.Is(err, mongo.ErrNoDocuments) {
+		return ErrHeartbeatNotFound
+	}
+	if err != nil {
+		return err
+	}
+	if kind == HeartbeatStart {
+		return nil
+	}
+
+	latency := 0
+	if kind == HeartbeatPing && before.State.StartedAt != nil {
+		latency = int(now.Sub(*before.State.StartedAt).Milliseconds())
+	}
+	if latency > 0 {
+		db.Col("monitors").UpdateOne(ctx, bson.M{"monitor_id": before.MonitorID}, bson.M{"$set": bson.M{"state.latency_ms": latency}})
+	}
+	recordSample(ctx, &before, kind == HeartbeatPing, latency, now)
+	applyTransition(ctx, &before, "", "", before.State.Status, next, message, now)
+	return nil
+}
+
+// SweepHeartbeats marks overdue heartbeats down. Recovery only ever comes from
+// a ping, so the sweep never moves anything up.
+func SweepHeartbeats(now time.Time) {
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	cur, err := db.Col("monitors").Find(ctx, bson.M{
+		"type": models.MonitorHeartbeat, "enabled": true,
+		"state.status": bson.M{"$ne": models.StatusDown},
+	})
+	if err != nil {
+		log.Printf("heartbeats: list: %v", err)
+		return
+	}
+	var monitors []models.Monitor
+	if err := cur.All(ctx, &monitors); err != nil {
+		log.Printf("heartbeats: decode: %v", err)
+		return
+	}
+	for i := range monitors {
+		m := &monitors[i]
+		down, msg := heartbeatVerdict(*m, now)
+		if !down {
+			continue
+		}
+		// Guarded on the exact fields the verdict was computed from, so a
+		// ping landing between the read and this write (which changes
+		// last_ping_at or started_at without necessarily changing status)
+		// makes the filter match nothing rather than overwriting a monitor
+		// that is no longer overdue.
+		res, err := db.Col("monitors").UpdateOne(ctx,
+			sweepGuardFilter(*m),
+			bson.M{"$set": bson.M{"state.status": models.StatusDown, "state.message": msg, "state.last_check_at": now},
+				"$unset": bson.M{"state.started_at": ""}})
+		if err != nil || res.ModifiedCount == 0 {
+			continue
+		}
+		recordSample(ctx, m, false, 0, now)
+		applyTransition(ctx, m, "", "", m.State.Status, models.StatusDown, msg, now)
+	}
+}
+
+// sweepGuardFilter is the optimistic-concurrency filter for SweepHeartbeats'
+// update: it pins monitor_id, status, last_ping_at and started_at to the
+// values the verdict was computed from, so the update only applies when
+// nothing about the ping state changed underneath the sweep. last_ping_at is
+// absent before the first ping and started_at is $unset on every ping, so an
+// unset field is pinned with $exists:false rather than equality to nil.
+func sweepGuardFilter(m models.Monitor) bson.M {
+	filter := bson.M{"monitor_id": m.MonitorID, "state.status": m.State.Status}
+	if m.State.LastPingAt != nil {
+		filter["state.last_ping_at"] = *m.State.LastPingAt
+	} else {
+		filter["state.last_ping_at"] = bson.M{"$exists": false}
+	}
+	if m.State.StartedAt != nil {
+		filter["state.started_at"] = *m.State.StartedAt
+	} else {
+		filter["state.started_at"] = bson.M{"$exists": false}
+	}
+	return filter
+}
+
+func RotateHeartbeatToken(instanceID, monitorID string) (string, error) {
+	ctx, cancel := monCtx()
+	defer cancel()
+	tok, hash, err := NewHeartbeatToken()
+	if err != nil {
+		return "", err
+	}
+	res, err := db.Col("monitors").UpdateOne(ctx,
+		bson.M{"monitor_id": monitorID, "instance_id": instanceID, "type": models.MonitorHeartbeat},
+		bson.M{"$set": bson.M{"heartbeat_token_hash": hash}})
+	if err != nil {
+		return "", err
+	}
+	if res.MatchedCount == 0 {
+		return "", ErrHeartbeatNotFound
+	}
+	return tok, nil
+}

server/internal/services/heartbeats_test.go

@@ -0,0 +1,117 @@
+package services
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+func hbMonitor(status string, lastPing, started *time.Time) models.Monitor {
+	return models.Monitor{
+		Type:   models.MonitorHeartbeat,
+		Target: models.MonitorTarget{PeriodSec: 3600, GraceSec: 300},
+		State:  models.MonitorState{Status: status, LastPingAt: lastPing, StartedAt: started},
+	}
+}
+
+func TestHeartbeatVerdict(t *testing.T) {
+	now := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC)
+	at := func(d time.Duration) *time.Time { v := now.Add(-d); return &v }
+
+	cases := []struct {
+		name     string
+		m        models.Monitor
+		wantDown bool
+		wantMsg  string
+	}{
+		{"never pinged stays pending", hbMonitor(models.StatusPending, nil, nil), false, ""},
+		{"within period", hbMonitor(models.StatusUp, at(30*time.Minute), nil), false, ""},
+		{"inside grace", hbMonitor(models.StatusUp, at(62*time.Minute), nil), false, ""},
+		{"overdue", hbMonitor(models.StatusUp, at(66*time.Minute), nil), true, "no ping since"},
+		{"started inside grace", hbMonitor(models.StatusUp, at(10*time.Minute), at(4*time.Minute)), false, ""},
+		{"started never finished", hbMonitor(models.StatusUp, at(10*time.Minute), at(6*time.Minute)), true, "never finished"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			down, msg := heartbeatVerdict(c.m, now)
+			if down != c.wantDown || !strings.Contains(msg, c.wantMsg) {
+				t.Fatalf("got (%v,%q), want (%v, contains %q)", down, msg, c.wantDown, c.wantMsg)
+			}
+		})
+	}
+}
+
+func TestNewHeartbeatTokenHashes(t *testing.T) {
+	tok, hash, err := NewHeartbeatToken()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(tok) < 32 {
+		t.Fatalf("token too short: %d", len(tok))
+	}
+	if hash != HashHeartbeatToken(tok) || hash == tok {
+		t.Fatal("hash must be the SHA-256 of the token and differ from it")
+	}
+}
+
+func TestValidateHeartbeat(t *testing.T) {
+	tg := models.MonitorTarget{PeriodSec: 59}
+	if err := validateHeartbeat(&tg); err == nil {
+		t.Fatal("period under 60 must be rejected")
+	}
+	tg = models.MonitorTarget{PeriodSec: 60}
+	if err := validateHeartbeat(&tg); err != nil || tg.GraceSec != 300 {
+		t.Fatalf("grace should default to 300, got %d err %v", tg.GraceSec, err)
+	}
+	tg = models.MonitorTarget{PeriodSec: 60, GraceSec: -1}
+	if err := validateHeartbeat(&tg); err == nil {
+		t.Fatal("negative grace must be rejected")
+	}
+}
+
+func TestSweepGuardFilterPinsFields(t *testing.T) {
+	now := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC)
+
+	// Both set: filter pins status, last_ping_at and started_at to their
+	// exact values.
+	m := hbMonitor(models.StatusUp, &now, &now)
+	m.MonitorID = "mon-1"
+	got := sweepGuardFilter(m)
+	want := bson.M{
+		"monitor_id":         "mon-1",
+		"state.status":       models.StatusUp,
+		"state.last_ping_at": now,
+		"state.started_at":   now,
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %+v, want %+v", got, want)
+	}
+
+	// Both unset: filter requires the fields to still be absent.
+	m2 := hbMonitor(models.StatusPending, nil, nil)
+	m2.MonitorID = "mon-2"
+	got2 := sweepGuardFilter(m2)
+	want2 := bson.M{
+		"monitor_id":         "mon-2",
+		"state.status":       models.StatusPending,
+		"state.last_ping_at": bson.M{"$exists": false},
+		"state.started_at":   bson.M{"$exists": false},
+	}
+	if !reflect.DeepEqual(got2, want2) {
+		t.Fatalf("got %+v, want %+v", got2, want2)
+	}
+}
+
+func TestTruncateHeartbeatBody(t *testing.T) {
+	long := strings.Repeat("x", MaxHeartbeatBody+50)
+	if got := failMessage(long); len(got) != len("reported failure: ")+MaxHeartbeatBody {
+		t.Fatalf("len = %d", len(got))
+	}
+	if got := failMessage("  "); got != "reported failure" {
+		t.Fatalf("empty body message = %q", got)
+	}
+}

server/internal/services/inventory.go

@@ -2,11 +2,15 @@ package services
 
 import (
 	"context"
+	"errors"
+	"log"
 	"time"
 
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
 	"gitea.hostxtra.co.uk/vantage/vantage-shared/grpc/pb"
 	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
 )
 
 func StoreInventory(serverID string, r *pb.InventoryReport) error {
@@ -29,6 +33,7 @@ func StoreInventory(serverID string, r *pb.InventoryReport) error {
 		set["inventory.boot_time"] = time.Unix(r.BootTimeUnix, 0).UTC()
 	}
 
+	var unset bson.M
 	if r.IncludeStatic {
 		set["inventory.static_at"] = now
 		set["inventory.swap_total_bytes"] = r.SwapTotal
@@ -49,8 +54,44 @@ func StoreInventory(serverID string, r *pb.InventoryReport) error {
 			})
 		}
 		set["inventory.partitions"] = parts
+
+		var prev struct {
+			Inventory struct {
+				RebootRequired      bool       `bson:"reboot_required"`
+				RebootRequiredSince *time.Time `bson:"reboot_required_since"`
+			} `bson:"inventory"`
+		}
+		err := db.Col("servers").FindOne(ctx, bson.M{"server_id": serverID},
+			options.FindOne().SetProjection(bson.M{"inventory.reboot_required": 1, "inventory.reboot_required_since": 1})).Decode(&prev)
+		// On a transient read error the previous stamp is unknown. Leaving it
+		// alone beats resetting it to now, which would restart "pending for N
+		// days" and hide a long-overdue reboot.
+		if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
+			log.Printf("inventory: read reboot state for %s: %v", serverID, err)
+		} else if since, clear := rebootSinceUpdate(prev.Inventory.RebootRequired, prev.Inventory.RebootRequiredSince, r.RebootRequired, now); since != nil {
+			set["inventory.reboot_required_since"] = *since
+		} else if clear {
+			unset = bson.M{"inventory.reboot_required_since": ""}
+		}
 	}
 
-	_, err := db.Col("servers").UpdateOne(ctx, bson.M{"server_id": serverID}, bson.M{"$set": set})
+	upd := bson.M{"$set": set}
+	if unset != nil {
+		upd["$unset"] = unset
+	}
+	_, err := db.Col("servers").UpdateOne(ctx, bson.M{"server_id": serverID}, upd)
 	return err
 }
+
+// rebootSinceUpdate decides how reboot_required_since changes. The stamp is
+// kept from the first report that needed a reboot, so "pending for 7 days"
+// means seven days, not seven days since the last static report.
+func rebootSinceUpdate(prevRequired bool, prevSince *time.Time, nowRequired bool, now time.Time) (*time.Time, bool) {
+	if !nowRequired {
+		return nil, prevSince != nil || prevRequired
+	}
+	if prevSince == nil {
+		return &now, false
+	}
+	return nil, false
+}

server/internal/services/inventory_reboot_test.go

@@ -0,0 +1,24 @@
+package services
+
+import (
+	"testing"
+	"time"
+)
+
+func TestRebootSinceUpdate(t *testing.T) {
+	now := time.Date(2026, 9, 17, 0, 0, 0, 0, time.UTC)
+	earlier := now.Add(-72 * time.Hour)
+
+	if set, unset := rebootSinceUpdate(false, nil, true, now); set == nil || !set.Equal(now) || unset {
+		t.Fatal("turning on must stamp now")
+	}
+	if set, unset := rebootSinceUpdate(true, &earlier, true, now); set != nil || unset {
+		t.Fatal("staying on must keep the original stamp")
+	}
+	if set, _ := rebootSinceUpdate(true, nil, true, now); set == nil {
+		t.Fatal("on with no stamp (pre-upgrade data) must stamp now")
+	}
+	if set, unset := rebootSinceUpdate(true, &earlier, false, now); set != nil || !unset {
+		t.Fatal("turning off must clear the stamp")
+	}
+}

server/internal/services/metricfilter_test.go

@@ -0,0 +1,20 @@
+package services
+
+import (
+	"testing"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+)
+
+func TestFilterByVisibleServer(t *testing.T) {
+	incs := []models.Incident{{IncidentID: "a"}, {IncidentID: "b", ServerID: "s1"}, {IncidentID: "c", ServerID: "s2"}}
+	id := func(i models.Incident) string { return i.ServerID }
+
+	if got := FilterByVisibleServer(incs, id, nil, false); len(got) != 3 {
+		t.Fatalf("unrestricted keeps all, got %d", len(got))
+	}
+	got := FilterByVisibleServer(incs, id, map[string]bool{"s1": true}, true)
+	if len(got) != 2 || got[0].IncidentID != "a" || got[1].IncidentID != "b" {
+		t.Fatalf("restricted keeps serverless and visible only, got %+v", got)
+	}
+}

server/internal/services/metricrules.go

@@ -0,0 +1,215 @@
+package services
+
+import (
+	"errors"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+)
+
+// ErrMonitorOutOfScope marks a metric monitor whose selector reaches beyond a
+// restricted token's tag scope, on create, update or delete.
+var ErrMonitorOutOfScope = errors.New("selector is outside this credential's tag scope")
+
+// selectorWithinScope holds when every server the selector can match is also
+// inside the token's scope - true exactly when the selector pins every pair
+// the scope does.
+func selectorWithinScope(sel, tokenScope map[string]string) bool {
+	for k, v := range tokenScope {
+		if sel[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+const (
+	MetricDiskPct            = "disk_pct"
+	MetricDiskFreeGB         = "disk_free_gb"
+	MetricMemPct             = "mem_pct"
+	MetricLoadPerCore        = "load_per_core"
+	MetricUnitFailed         = "unit_failed"
+	MetricContainerUnhealthy = "container_unhealthy"
+	MetricRebootPendingDays  = "reboot_pending_days"
+	MetricAgentOfflineMin    = "agent_offline_min"
+)
+
+// metricStaleAfter is how old an agent's metrics can be before a rule stops
+// judging them. A dead agent's last report must not hold an alert open or
+// clear one; agent_offline_min is the rule for a dead agent.
+const metricStaleAfter = 5 * time.Minute
+
+func metricNeedsWorkloads(kind string) bool {
+	return kind == MetricUnitFailed || kind == MetricContainerUnhealthy
+}
+
+func metricUsesThreshold(kind string) bool {
+	return !metricNeedsWorkloads(kind)
+}
+
+func validateMetric(m *models.Monitor) error {
+	t := &m.Target
+	switch t.Metric {
+	case MetricDiskPct, MetricDiskFreeGB, MetricMemPct, MetricLoadPerCore,
+		MetricUnitFailed, MetricContainerUnhealthy, MetricRebootPendingDays, MetricAgentOfflineMin:
+	default:
+		return fmt.Errorf("%w: unknown metric %q", ErrInvalidMonitor, t.Metric)
+	}
+	if metricUsesThreshold(t.Metric) && t.Threshold <= 0 {
+		return fmt.Errorf("%w: threshold must be greater than 0", ErrInvalidMonitor)
+	}
+	if (t.Metric == MetricDiskPct || t.Metric == MetricMemPct) && t.Threshold > 100 {
+		return fmt.Errorf("%w: threshold must be 100 or less", ErrInvalidMonitor)
+	}
+	if t.Mount != "" && !path.IsAbs(t.Mount) {
+		return fmt.Errorf("%w: mount must be an absolute path", ErrInvalidMonitor)
+	}
+	if m.ForSec < 0 {
+		m.ForSec = 0
+	}
+	return nil
+}
+
+// EvaluateMetric judges one server against one rule. ok=false means there is
+// nothing trustworthy to judge - stale metrics, a mount that does not exist on
+// this host - and the caller keeps the previous state rather than guessing.
+func EvaluateMetric(t models.MonitorTarget, srv models.Server, wls []models.Workload, now time.Time) (bool, float64, string, bool) {
+	if t.Metric == MetricAgentOfflineMin {
+		if srv.LastSeen == nil {
+			return false, 0, "", false
+		}
+		mins := now.Sub(*srv.LastSeen).Minutes()
+		return mins >= t.Threshold, mins, fmt.Sprintf("agent not seen for %.0f minutes", mins), true
+	}
+	inv := srv.Inventory
+	if inv == nil || inv.MetricsAt == nil || now.Sub(*inv.MetricsAt) > metricStaleAfter {
+		return false, 0, "", false
+	}
+
+	switch t.Metric {
+	case MetricDiskPct, MetricDiskFreeGB:
+		return evalDisk(t, inv.Partitions)
+	case MetricMemPct:
+		if inv.Memory.TotalBytes == 0 {
+			return false, 0, "", false
+		}
+		pct := float64(inv.Memory.UsedBytes) / float64(inv.Memory.TotalBytes) * 100
+		return pct >= t.Threshold, pct, fmt.Sprintf("memory %.1f%% used", pct), true
+	case MetricLoadPerCore:
+		if inv.CPU.Cores == 0 {
+			return false, 0, "", false
+		}
+		v := inv.CPU.Load1 / float64(inv.CPU.Cores)
+		return v >= t.Threshold, v, fmt.Sprintf("load %.2f per core", v), true
+	case MetricRebootPendingDays:
+		if !inv.RebootRequired || inv.RebootRequiredSince == nil {
+			return false, 0, "", true
+		}
+		days := now.Sub(*inv.RebootRequiredSince).Hours() / 24
+		return days >= t.Threshold, days, fmt.Sprintf("reboot pending for %.0f days", days), true
+	case MetricUnitFailed:
+		return evalWorkloads(wls, "unit", func(w models.Workload) bool { return w.State == "failed" }, "failed")
+	case MetricContainerUnhealthy:
+		return evalWorkloads(wls, "container", func(w models.Workload) bool { return w.Health == "unhealthy" }, "unhealthy")
+	}
+	return false, 0, "", false
+}
+
+// evalDisk reports the worst matching partition, so "any mount" names the one
+// that is actually full.
+func evalDisk(t models.MonitorTarget, parts []models.Partition) (bool, float64, string, bool) {
+	found := false
+	var worstBreach bool
+	var worstVal float64
+	var worstMsg string
+	for _, p := range parts {
+		if p.TotalBytes == 0 || p.UsedBytes > p.TotalBytes || (t.Mount != "" && p.Mountpoint != t.Mount) {
+			continue
+		}
+		var breach bool
+		var val float64
+		var msg string
+		if t.Metric == MetricDiskPct {
+			val = float64(p.UsedBytes) / float64(p.TotalBytes) * 100
+			breach = val >= t.Threshold
+			msg = fmt.Sprintf("%s %.1f%% used", p.Mountpoint, val)
+		} else {
+			val = float64(p.TotalBytes-p.UsedBytes) / 1e9
+			breach = val <= t.Threshold
+			msg = fmt.Sprintf("%s %.1f GB free", p.Mountpoint, val)
+		}
+		worse := !found ||
+			(t.Metric == MetricDiskPct && val > worstVal) ||
+			(t.Metric == MetricDiskFreeGB && val < worstVal)
+		if worse {
+			worstBreach, worstVal, worstMsg = breach, val, msg
+		}
+		found = true
+	}
+	if !found {
+		return false, 0, "", false
+	}
+	return worstBreach, worstVal, worstMsg, true
+}
+
+func evalWorkloads(wls []models.Workload, kind string, bad func(models.Workload) bool, word string) (bool, float64, string, bool) {
+	var names []string
+	for _, w := range wls {
+		if w.Kind == kind && bad(w) {
+			names = append(names, w.Name)
+		}
+	}
+	if len(names) == 0 {
+		return false, 0, "", true
+	}
+	return true, float64(len(names)), fmt.Sprintf("%s %s: %s", kind, word, strings.Join(names, ", ")), true
+}
+
+// nextServerState applies the "for N seconds" gate. BreachSince is the first
+// sweep that saw the condition, so the gate measures continuous breach and a
+// single clear sweep restarts it.
+func nextServerState(prev *models.MonitorServerState, breach bool, forSec int, now time.Time) (string, *time.Time) {
+	if !breach {
+		return models.StatusUp, nil
+	}
+	since := now
+	if prev != nil && prev.BreachSince != nil {
+		since = *prev.BreachSince
+	}
+	// A server already down stays down while the breach lasts, even if for_sec
+	// was raised meanwhile. Dropping it to pending would never resolve the open
+	// incident, because only an up transition closes it.
+	if prev != nil && prev.Status == models.StatusDown {
+		return models.StatusDown, &since
+	}
+	if now.Sub(since) >= time.Duration(forSec)*time.Second {
+		return models.StatusDown, &since
+	}
+	return models.StatusPending, &since
+}
+
+func rollupParent(states []models.MonitorServerState) (string, string) {
+	if len(states) == 0 {
+		return models.StatusUp, "no matching servers"
+	}
+	down, pending := 0, 0
+	for _, s := range states {
+		switch s.Status {
+		case models.StatusDown:
+			down++
+		case models.StatusPending:
+			pending++
+		}
+	}
+	msg := fmt.Sprintf("%d of %d servers breaching", down, len(states))
+	switch {
+	case down > 0:
+		return models.StatusDown, msg
+	case pending > 0:
+		return models.StatusPending, msg
+	}
+	return models.StatusUp, msg
+}

server/internal/services/metricrules_test.go

@@ -0,0 +1,130 @@
+package services
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+)
+
+var evalNow = time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC)
+
+func fresh() *time.Time { t := evalNow.Add(-time.Minute); return &t }
+
+func srvWith(inv models.Inventory) models.Server {
+	if inv.MetricsAt == nil {
+		inv.MetricsAt = fresh()
+	}
+	ls := evalNow.Add(-30 * time.Second)
+	return models.Server{ServerID: "s1", Hostname: "web-01", Inventory: &inv, LastSeen: &ls}
+}
+
+func TestEvaluateMetric(t *testing.T) {
+	disk := models.Inventory{Partitions: []models.Partition{
+		{Mountpoint: "/", TotalBytes: 100e9, UsedBytes: 50e9},
+		{Mountpoint: "/var", TotalBytes: 100e9, UsedBytes: 95e9},
+	}}
+	stale := evalNow.Add(-10 * time.Minute)
+	since := evalNow.Add(-8 * 24 * time.Hour)
+	oldSeen := evalNow.Add(-20 * time.Minute)
+
+	cases := []struct {
+		name       string
+		t          models.MonitorTarget
+		srv        models.Server
+		wls        []models.Workload
+		wantBreach bool
+		wantOK     bool
+		msgHas     string
+	}{
+		{"disk pct any mount breaches", models.MonitorTarget{Metric: MetricDiskPct, Threshold: 90}, srvWith(disk), nil, true, true, "/var"},
+		{"disk pct named mount clear", models.MonitorTarget{Metric: MetricDiskPct, Threshold: 90, Mount: "/"}, srvWith(disk), nil, false, true, "/"},
+		{"disk pct missing mount skips", models.MonitorTarget{Metric: MetricDiskPct, Threshold: 90, Mount: "/data"}, srvWith(disk), nil, false, false, ""},
+		{"disk free gb breaches", models.MonitorTarget{Metric: MetricDiskFreeGB, Threshold: 10}, srvWith(disk), nil, true, true, "GB free"},
+		{"zero total partition ignored", models.MonitorTarget{Metric: MetricDiskPct, Threshold: 1}, srvWith(models.Inventory{Partitions: []models.Partition{{Mountpoint: "/proc"}}}), nil, false, false, ""},
+		{"used over total partition ignored", models.MonitorTarget{Metric: MetricDiskFreeGB, Threshold: 10}, srvWith(models.Inventory{Partitions: []models.Partition{{Mountpoint: "/", TotalBytes: 100e9, UsedBytes: 150e9}}}), nil, false, false, ""},
+		{"mem pct", models.MonitorTarget{Metric: MetricMemPct, Threshold: 80}, srvWith(models.Inventory{Memory: models.MemInfo{TotalBytes: 100, UsedBytes: 85}}), nil, true, true, "memory"},
+		{"load per core", models.MonitorTarget{Metric: MetricLoadPerCore, Threshold: 1.5}, srvWith(models.Inventory{CPU: models.CPUInfo{Cores: 4, Load1: 8}}), nil, true, true, "load"},
+		{"load no cores skips", models.MonitorTarget{Metric: MetricLoadPerCore, Threshold: 1.5}, srvWith(models.Inventory{CPU: models.CPUInfo{Load1: 8}}), nil, false, false, ""},
+		{"stale inventory skips", models.MonitorTarget{Metric: MetricMemPct, Threshold: 1}, srvWith(models.Inventory{MetricsAt: &stale, Memory: models.MemInfo{TotalBytes: 100, UsedBytes: 99}}), nil, false, false, ""},
+		{"unit failed", models.MonitorTarget{Metric: MetricUnitFailed}, srvWith(models.Inventory{}), []models.Workload{{Kind: "unit", Name: "backup.service", State: "failed"}}, true, true, "backup.service"},
+		{"container unhealthy clear", models.MonitorTarget{Metric: MetricContainerUnhealthy}, srvWith(models.Inventory{}), []models.Workload{{Kind: "container", Name: "db", Health: "healthy"}}, false, true, ""},
+		{"reboot pending days", models.MonitorTarget{Metric: MetricRebootPendingDays, Threshold: 7}, srvWith(models.Inventory{RebootRequired: true, RebootRequiredSince: &since}), nil, true, true, "reboot"},
+		{"reboot not required", models.MonitorTarget{Metric: MetricRebootPendingDays, Threshold: 7}, srvWith(models.Inventory{}), nil, false, true, ""},
+		{"agent offline ignores stale inventory", models.MonitorTarget{Metric: MetricAgentOfflineMin, Threshold: 10},
+			func() models.Server { s := srvWith(models.Inventory{MetricsAt: &stale}); s.LastSeen = &oldSeen; return s }(), nil, true, true, "not seen"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			breach, _, msg, ok := EvaluateMetric(c.t, c.srv, c.wls, evalNow)
+			if breach != c.wantBreach || ok != c.wantOK || !strings.Contains(msg, c.msgHas) {
+				t.Fatalf("got breach=%v ok=%v msg=%q", breach, ok, msg)
+			}
+		})
+	}
+}
+
+func TestNextServerState(t *testing.T) {
+	past := func(d time.Duration) *time.Time { v := evalNow.Add(-d); return &v }
+
+	st, since := nextServerState(nil, true, 300, evalNow)
+	if st != models.StatusPending || since == nil || !since.Equal(evalNow) {
+		t.Fatalf("new breach: %s %v", st, since)
+	}
+	st, _ = nextServerState(&models.MonitorServerState{Status: models.StatusPending, BreachSince: past(2 * time.Minute)}, true, 300, evalNow)
+	if st != models.StatusPending {
+		t.Fatalf("under for_sec should stay pending, got %s", st)
+	}
+	st, since = nextServerState(&models.MonitorServerState{Status: models.StatusPending, BreachSince: past(6 * time.Minute)}, true, 300, evalNow)
+	if st != models.StatusDown || !since.Equal(*past(6 * time.Minute)) {
+		t.Fatalf("over for_sec should be down keeping since, got %s %v", st, since)
+	}
+	st, _ = nextServerState(nil, true, 0, evalNow)
+	if st != models.StatusDown {
+		t.Fatalf("for_sec 0 is down immediately, got %s", st)
+	}
+	// Raising for_sec mid-breach must not demote a down server to pending:
+	// pending never resolves an incident, so it would stay open forever.
+	st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(2 * time.Minute)}, true, 3600, evalNow)
+	if st != models.StatusDown || !since.Equal(*past(2 * time.Minute)) {
+		t.Fatalf("down with larger for_sec should stay down keeping since, got %s %v", st, since)
+	}
+	st, since = nextServerState(&models.MonitorServerState{Status: models.StatusDown, BreachSince: past(time.Hour)}, false, 300, evalNow)
+	if st != models.StatusUp || since != nil {
+		t.Fatalf("clear should be up with no since, got %s %v", st, since)
+	}
+}
+
+func TestRollupParent(t *testing.T) {
+	if s, m := rollupParent(nil); s != models.StatusUp || m != "no matching servers" {
+		t.Fatalf("empty: %s %q", s, m)
+	}
+	states := []models.MonitorServerState{{Status: models.StatusUp}, {Status: models.StatusPending}, {Status: models.StatusDown}}
+	if s, m := rollupParent(states); s != models.StatusDown || m != "1 of 3 servers breaching" {
+		t.Fatalf("got %s %q", s, m)
+	}
+	if s, _ := rollupParent(states[:2]); s != models.StatusPending {
+		t.Fatalf("pending wins over up, got %s", s)
+	}
+}
+
+func TestValidateMetric(t *testing.T) {
+	bad := []models.MonitorTarget{
+		{Metric: "cpu_magic", Threshold: 1},
+		{Metric: MetricDiskPct, Threshold: 0},
+		{Metric: MetricDiskPct, Threshold: 101},
+		{Metric: MetricDiskPct, Threshold: 90, Mount: "var"},
+	}
+	for _, tg := range bad {
+		m := models.Monitor{Type: models.MonitorMetric, Target: tg}
+		if err := validateMetric(&m); !errors.Is(err, ErrInvalidMonitor) {
+			t.Errorf("%+v: want ErrInvalidMonitor, got %v", tg, err)
+		}
+	}
+	ok := models.Monitor{Type: models.MonitorMetric, ForSec: -5, Target: models.MonitorTarget{Metric: MetricUnitFailed}}
+	if err := validateMetric(&ok); err != nil || ok.ForSec != 0 {
+		t.Fatalf("unit_failed needs no threshold and negative for_sec clamps to 0: %v %d", err, ok.ForSec)
+	}
+}

server/internal/services/metricscope_test.go

@@ -0,0 +1,27 @@
+package services
+
+import "testing"
+
+// A restricted token must not be able to create a rule that watches servers it
+// cannot see: the per-server table and incident messages would disclose them.
+func TestSelectorWithinScope(t *testing.T) {
+	scope := map[string]string{"env": "prod"}
+	cases := []struct {
+		sel  map[string]string
+		want bool
+	}{
+		{nil, false},
+		{map[string]string{"env": "dev"}, false},
+		{map[string]string{"role": "web"}, false},
+		{map[string]string{"env": "prod"}, true},
+		{map[string]string{"env": "prod", "role": "web"}, true},
+	}
+	for _, c := range cases {
+		if got := selectorWithinScope(c.sel, scope); got != c.want {
+			t.Errorf("sel %v: got %v want %v", c.sel, got, c.want)
+		}
+	}
+	if !selectorWithinScope(nil, nil) {
+		t.Error("an unrestricted credential may use any selector, including the whole fleet")
+	}
+}

server/internal/services/metricsweep.go

@@ -0,0 +1,153 @@
+package services
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
+)
+
+// EnsureMonitorServerStateIndexes declares the one key every read and upsert
+// uses. Unique, so a double sweep across a leader handover cannot fork a
+// server's state into two documents.
+func EnsureMonitorServerStateIndexes() error {
+	_, err := db.Col("monitor_server_states").Indexes().CreateOne(context.Background(), mongo.IndexModel{
+		Keys:    bson.D{{Key: "monitor_id", Value: 1}, {Key: "server_id", Value: 1}},
+		Options: options.Index().SetUnique(true),
+	})
+	if err != nil {
+		log.Printf("warning: monitor_server_states indexes: %v", err)
+	}
+	// Every heartbeat ping looks its monitor up by token hash. Sparse because
+	// only heartbeat monitors carry one.
+	_, err = db.Col("monitors").Indexes().CreateOne(context.Background(), mongo.IndexModel{
+		Keys:    bson.D{{Key: "heartbeat_token_hash", Value: 1}},
+		Options: options.Index().SetSparse(true),
+	})
+	if err != nil {
+		log.Printf("warning: monitors heartbeat_token_hash index: %v", err)
+	}
+	return nil
+}
+
+func ListMonitorServerStates(instanceID, monitorID string) ([]models.MonitorServerState, error) {
+	ctx, cancel := monCtx()
+	defer cancel()
+	cur, err := db.Col("monitor_server_states").Find(ctx, bson.M{"instance_id": instanceID, "monitor_id": monitorID})
+	if err != nil {
+		return nil, err
+	}
+	out := []models.MonitorServerState{}
+	if err := cur.All(ctx, &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ponytail: every metric monitor re-lists its servers each sweep; cache the
+// fleet per instance per sweep if large fleets show up in the sweep log.
+func SweepMetricMonitors(now time.Time) {
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
+	defer cancel()
+	cur, err := db.Col("monitors").Find(ctx, bson.M{"type": models.MonitorMetric, "enabled": true})
+	if err != nil {
+		log.Printf("metrics: list: %v", err)
+		return
+	}
+	var monitors []models.Monitor
+	if err := cur.All(ctx, &monitors); err != nil {
+		log.Printf("metrics: decode: %v", err)
+		return
+	}
+	for i := range monitors {
+		sweepOneMetric(ctx, &monitors[i], now)
+	}
+}
+
+func sweepOneMetric(ctx context.Context, m *models.Monitor, now time.Time) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("metrics: monitor %s panic: %v", m.MonitorID, r)
+		}
+	}()
+
+	servers, err := ListServersFiltered(m.InstanceID, m.Target.Selector)
+	if err != nil {
+		log.Printf("metrics: servers for %s: %v", m.MonitorID, err)
+		return
+	}
+	prevStates, err := ListMonitorServerStates(m.InstanceID, m.MonitorID)
+	if err != nil {
+		log.Printf("metrics: states for %s: %v", m.MonitorID, err)
+		return
+	}
+	prevByServer := map[string]*models.MonitorServerState{}
+	for i := range prevStates {
+		prevByServer[prevStates[i].ServerID] = &prevStates[i]
+	}
+
+	col := db.Col("monitor_server_states")
+	current := make([]models.MonitorServerState, 0, len(servers))
+	matched := map[string]bool{}
+	for _, srv := range servers {
+		matched[srv.ServerID] = true
+		prev := prevByServer[srv.ServerID]
+
+		var wls []models.Workload
+		if metricNeedsWorkloads(m.Target.Metric) {
+			if sw, err := GetWorkloads(m.InstanceID, srv.ServerID); err == nil && sw != nil {
+				wls = sw.Workloads
+			}
+		}
+		breach, value, msg, ok := EvaluateMetric(m.Target, srv, wls, now)
+		if !ok {
+			if prev != nil {
+				current = append(current, *prev)
+			}
+			continue
+		}
+
+		status, since := nextServerState(prev, breach, m.ForSec, now)
+		st := models.MonitorServerState{
+			InstanceID: m.InstanceID, MonitorID: m.MonitorID, ServerID: srv.ServerID,
+			Status: status, BreachSince: since, Value: value, Message: msg, UpdatedAt: now,
+		}
+		if _, err := col.ReplaceOne(ctx, bson.M{"monitor_id": m.MonitorID, "server_id": srv.ServerID}, st,
+			options.Replace().SetUpsert(true)); err != nil {
+			log.Printf("metrics: save state %s/%s: %v", m.MonitorID, srv.ServerID, err)
+			continue
+		}
+		current = append(current, st)
+
+		// A server seen for the first time has no previous status to leave, so
+		// it can open an incident but never announce a recovery.
+		prevStatus := models.StatusPending
+		if prev != nil {
+			prevStatus = prev.Status
+		}
+		applyTransition(ctx, m, srv.ServerID, srv.Hostname, prevStatus, status, msg, now)
+	}
+
+	// Servers that left the selector, or the fleet, did not recover: their
+	// incidents close quietly and their state goes.
+	for _, prev := range prevStates {
+		if matched[prev.ServerID] {
+			continue
+		}
+		if prev.Status == models.StatusDown {
+			resolveIncident(ctx, m, prev.ServerID, now)
+		}
+		col.DeleteOne(ctx, bson.M{"monitor_id": m.MonitorID, "server_id": prev.ServerID})
+	}
+
+	status, msg := rollupParent(current)
+	db.Col("monitors").UpdateOne(ctx, bson.M{"monitor_id": m.MonitorID}, bson.M{"$set": bson.M{
+		"state.status": status, "state.message": msg, "state.last_check_at": now,
+	}})
+	recordSample(ctx, m, status != models.StatusDown, 0, now)
+}

server/internal/services/mfa_scoped_test.go

@@ -19,3 +19,13 @@ func TestMFACollectionsAreScoped(t *testing.T) {
 		}
 	}
 }
+
+// Per-server metric state is tenant data and must be purged with its instance.
+func TestMonitorServerStatesAreScoped(t *testing.T) {
+	for _, got := range ScopedCollections {
+		if got == "monitor_server_states" {
+			return
+		}
+	}
+	t.Error("monitor_server_states is not in ScopedCollections")
+}

server/internal/services/migrate_instance.go

@@ -37,6 +37,7 @@ var ScopedCollections = []string{
 	"incidents",
 	"monitor_rollups",
 	"monitor_samples",
+	"monitor_server_states",
 	"notification_channels",
 	"console_sessions",
 	"audit_logs",

server/internal/services/monitors.go

@@ -103,7 +103,8 @@ func ListServerScheduledMonitors() ([]models.Monitor, error) {
 func listMonitorsForRunner(instanceID, runner string) ([]models.Monitor, error) {
 	ctx, cancel := monCtx()
 	defer cancel()
-	filter := bson.M{"runner": runner, "enabled": true}
+	filter := bson.M{"runner": runner, "enabled": true,
+		"type": bson.M{"$nin": []string{models.MonitorMetric, models.MonitorHeartbeat}}}
 	if instanceID != "" {
 		filter["instance_id"] = instanceID
 	}
@@ -187,6 +188,32 @@ func CreateMonitor(instanceID string, m *models.Monitor, tokenScope map[string]s
 	if m.Runner == "" {
 		m.Runner = models.RunnerServer
 	}
+	if m.Type == models.MonitorHeartbeat {
+		if err := validateHeartbeat(&m.Target); err != nil {
+			return nil, err
+		}
+		tok, hash, err := NewHeartbeatToken()
+		if err != nil {
+			return nil, err
+		}
+		m.HeartbeatTokenHash = hash
+		m.HeartbeatToken = tok
+		// A heartbeat is never run, so neither where nor how often applies.
+		m.Runner = models.RunnerServer
+		m.IntervalSec = 0
+	}
+	if m.Type == models.MonitorMetric {
+		if err := validateMetric(m); err != nil {
+			return nil, err
+		}
+		if !selectorWithinScope(m.Target.Selector, tokenScope) {
+			return nil, ErrMonitorOutOfScope
+		}
+		// A metric monitor is evaluated by sweeping stored server inventory,
+		// never run against a single named agent.
+		m.Runner = models.RunnerServer
+		m.IntervalSec = 0
+	}
 	m.State = models.MonitorState{Status: models.StatusPending}
 	if _, err := db.Col("monitors").InsertOne(ctx, m); err != nil {
 		return nil, err
@@ -198,6 +225,50 @@ func UpdateMonitor(instanceID, monitorID string, upd bson.M, tokenScope map[stri
 	ctx, cancel := monCtx()
 	defer cancel()
 
+	existing, err := GetMonitor(instanceID, monitorID)
+	if err != nil {
+		return err
+	}
+	if existing == nil {
+		return ErrMonitorNotFound
+	}
+	unset, err := prepareMonitorUpdate(existing, upd)
+	if err != nil {
+		return err
+	}
+	// Editing any field of a metric monitor the token could not have created is
+	// refused before any write, so a restricted token cannot rename or disable
+	// a fleet-wide rule it cannot fully see.
+	if existing.Type == models.MonitorMetric && !selectorWithinScope(existing.Target.Selector, tokenScope) {
+		return ErrMonitorOutOfScope
+	}
+
+	if raw, present := upd["target"]; present && models.IsPassiveMonitor(existing.Type) {
+		tg, ok := raw.(models.MonitorTarget)
+		if !ok {
+			return fmt.Errorf("%w: target must be an object", ErrInvalidMonitor)
+		}
+		switch existing.Type {
+		case models.MonitorHeartbeat:
+			if err := validateHeartbeat(&tg); err != nil {
+				return err
+			}
+		case models.MonitorMetric:
+			forSec := existing.ForSec
+			if v, ok := upd["for_sec"].(int); ok {
+				forSec = v
+			}
+			probe := models.Monitor{Type: existing.Type, Target: tg, ForSec: forSec}
+			if err := validateMetric(&probe); err != nil {
+				return err
+			}
+			if !selectorWithinScope(tg.Selector, tokenScope) {
+				return ErrMonitorOutOfScope
+			}
+		}
+		upd["target"] = tg
+	}
+
 	if raw, present := upd["channel_ids"]; present {
 		ids, ok := raw.([]string)
 		if !ok {
@@ -231,13 +302,68 @@ func UpdateMonitor(instanceID, monitorID string, upd bson.M, tokenScope map[stri
 			upd["runner"] = models.RunnerServer
 		}
 	}
-	_, err := db.Col("monitors").UpdateOne(ctx, bson.M{"monitor_id": monitorID, "instance_id": instanceID}, bson.M{"$set": upd})
+	change := bson.M{"$set": upd}
+	if len(unset) > 0 {
+		change["$unset"] = unset
+	}
+	_, err = db.Col("monitors").UpdateOne(ctx, bson.M{"monitor_id": monitorID, "instance_id": instanceID}, change)
 	return err
 }
 
-func DeleteMonitor(instanceID, monitorID string) error {
+// ErrMonitorNotFound is returned when an update names a monitor that does not
+// exist in the instance.
+var ErrMonitorNotFound = errors.New("monitor not found")
+
+// prepareMonitorUpdate applies the checks and rewrites that need no database:
+// refusing a type change to or from a passive type, clamping for_sec, and
+// resetting a heartbeat's ping state when it is re-enabled. It returns the
+// fields to $unset.
+func prepareMonitorUpdate(existing *models.Monitor, upd bson.M) (bson.M, error) {
+	if raw, present := upd["type"]; present {
+		if t, _ := raw.(string); t != existing.Type && (models.IsPassiveMonitor(t) || models.IsPassiveMonitor(existing.Type)) {
+			bad := existing.Type
+			if models.IsPassiveMonitor(t) {
+				bad = t
+			}
+			return nil, fmt.Errorf("%w: type cannot be changed to or from %s", ErrInvalidMonitor, bad)
+		}
+	}
+	// A negative gate means nothing; treat it as "alert immediately", as
+	// create does.
+	if v, ok := upd["for_sec"].(int); ok && v < 0 {
+		upd["for_sec"] = 0
+	}
+	unset := bson.M{}
+	// A heartbeat disabled for a while still holds its last ping, so the next
+	// sweep after re-enabling would page at once. Start it over as if new.
+	if on, _ := upd["enabled"].(bool); on && !existing.Enabled && existing.Type == models.MonitorHeartbeat {
+		unset["state.last_ping_at"] = ""
+		unset["state.started_at"] = ""
+		upd["state.status"] = models.StatusPending
+	}
+	return unset, nil
+}
+
+// DeleteMonitor deletes a monitor and its associated data. For a metric
+// monitor, deletion is refused when its selector reaches beyond tokenScope -
+// the same out-of-scope check applied on create and update - so a restricted
+// token cannot remove a fleet-wide rule it could not have made. Other monitor
+// types delete as before, unaffected by tokenScope.
+func DeleteMonitor(instanceID, monitorID string, tokenScope map[string]string) error {
 	ctx, cancel := monCtx()
 	defer cancel()
+
+	existing, err := GetMonitor(instanceID, monitorID)
+	if err != nil {
+		return err
+	}
+	if existing == nil {
+		return nil
+	}
+	if existing.Type == models.MonitorMetric && !selectorWithinScope(existing.Target.Selector, tokenScope) {
+		return ErrMonitorOutOfScope
+	}
+
 	res, err := db.Col("monitors").DeleteOne(ctx, bson.M{"monitor_id": monitorID, "instance_id": instanceID})
 	if err != nil {
 		return err
@@ -249,6 +375,7 @@ func DeleteMonitor(instanceID, monitorID string) error {
 	db.Col("incidents").DeleteMany(ctx, bson.M{"monitor_id": monitorID, "instance_id": instanceID})
 	db.Col("monitor_rollups").DeleteMany(ctx, bson.M{"monitor_id": monitorID, "instance_id": instanceID})
 	db.Col("monitor_samples").DeleteMany(ctx, bson.M{"monitor_id": monitorID, "instance_id": instanceID})
+	db.Col("monitor_server_states").DeleteMany(ctx, bson.M{"monitor_id": monitorID, "instance_id": instanceID})
 	return nil
 }
 
@@ -342,24 +469,7 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro
 
 	now := time.Now()
 	prev := m.State.Status
-	retries := m.Retries
-	if retries < 1 {
-		retries = 1
-	}
-
-	newStatus := prev
-	fails := m.State.Fails
-	if res.Up {
-		fails = 0
-		newStatus = models.StatusUp
-	} else {
-		fails++
-		if fails >= retries {
-			newStatus = models.StatusDown
-		} else if prev == "" || prev == models.StatusPending {
-			newStatus = models.StatusPending
-		}
-	}
+	newStatus, fails := decideStatus(prev, m.State.Fails, m.Retries, res.Up)
 
 	state := bson.M{
 		"state.status":        newStatus,
@@ -375,56 +485,36 @@ func ingestResult(instanceID, runner, monitorID string, res checker.Result) erro
 		return err
 	}
 
-	bucket := now.Truncate(time.Hour)
-	up := 0
-	if res.Up {
-		up = 1
-	}
+	recordSample(ctx, m, res.Up, res.LatencyMs, now)
 
-	/* The sample is the same result at full resolution, expiring by TTL. It is
-	   written next to the rollup rather than instead of it: the rollup is what
-	   survives, the sample is what the sub-hour views read. */
-	db.Col("monitor_samples").InsertOne(ctx, models.MonitorSample{
-		InstanceID: m.InstanceID,
-		MonitorID:  monitorID,
-		At:         now,
-		Up:         res.Up,
-		LatencyMs:  res.LatencyMs,
-	})
-
-	db.Col("monitor_rollups").UpdateOne(ctx,
-		bson.M{"monitor_id": monitorID, "period_start": bucket},
-		bson.M{
-			"$inc":         bson.M{"checks": 1, "up_count": up, "sum_latency": int64(res.LatencyMs)},
-			"$setOnInsert": bson.M{"instance_id": m.InstanceID},
-		},
-		options.UpdateOne().SetUpsert(true))
-
-	if newStatus != prev {
-		switch newStatus {
-		case models.StatusDown:
-			inc := models.Incident{
-				InstanceID: m.InstanceID,
-				IncidentID: uuid.NewString(),
-				MonitorID:  monitorID,
-				StartedAt:  now,
-				Cause:      res.Message,
-			}
-			db.Col("incidents").InsertOne(ctx, inc)
-			notifyTransition(m, newStatus, res.Message)
-		case models.StatusUp:
-			if prev == models.StatusDown {
-				db.Col("incidents").UpdateOne(ctx,
-					bson.M{"monitor_id": monitorID, "instance_id": m.InstanceID, "resolved_at": nil},
-					bson.M{"$set": bson.M{"resolved_at": now}})
-				notifyTransition(m, newStatus, res.Message)
-			}
-		}
-	}
+	applyTransition(ctx, m, "", "", prev, newStatus, res.Message, now)
 	return nil
 }
 
-func notifyTransition(m *models.Monitor, newStatus, message string) {
+// recordSample writes one result at full resolution and folds it into the
+// hourly rollup, the two records every history view reads.
+func recordSample(ctx context.Context, m *models.Monitor, up bool, latencyMs int, now time.Time) {
+	u := 0
+	if up {
+		u = 1
+	}
+	db.Col("monitor_samples").InsertOne(ctx, models.MonitorSample{
+		InstanceID: m.InstanceID,
+		MonitorID:  m.MonitorID,
+		At:         now,
+		Up:         up,
+		LatencyMs:  latencyMs,
+	})
+	db.Col("monitor_rollups").UpdateOne(ctx,
+		bson.M{"monitor_id": m.MonitorID, "period_start": now.Truncate(time.Hour)},
+		bson.M{
+			"$inc":         bson.M{"checks": 1, "up_count": u, "sum_latency": int64(latencyMs)},
+			"$setOnInsert": bson.M{"instance_id": m.InstanceID},
+		},
+		options.UpdateOne().SetUpsert(true))
+}
+
+func notifyTransition(m *models.Monitor, serverName, oldStatus, newStatus, message string) {
 	if len(m.ChannelIDs) == 0 {
 		return
 	}
@@ -435,8 +525,9 @@ func notifyTransition(m *models.Monitor, newStatus, message string) {
 	}
 	ev := notify.Event{
 		MonitorName: m.Name,
+		ServerName:  serverName,
 		Type:        m.Type,
-		OldStatus:   m.State.Status,
+		OldStatus:   oldStatus,
 		NewStatus:   newStatus,
 		Message:     message,
 		Time:        time.Now(),

server/internal/services/monitors_test.go

@@ -0,0 +1,45 @@
+package services
+
+import (
+	"errors"
+	"testing"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+func TestPrepareMonitorUpdate(t *testing.T) {
+	hb := &models.Monitor{Type: models.MonitorHeartbeat, Enabled: false}
+
+	if _, err := prepareMonitorUpdate(hb, bson.M{"type": models.MonitorHTTP}); !errors.Is(err, ErrInvalidMonitor) {
+		t.Fatalf("type change should be ErrInvalidMonitor, got %v", err)
+	}
+
+	upd := bson.M{"enabled": true}
+	unset, err := prepareMonitorUpdate(hb, upd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := unset["state.last_ping_at"]; !ok {
+		t.Fatalf("re-enable should clear last_ping_at, got %v", unset)
+	}
+	if _, ok := unset["state.started_at"]; !ok {
+		t.Fatalf("re-enable should clear started_at, got %v", unset)
+	}
+	if upd["state.status"] != models.StatusPending {
+		t.Fatalf("re-enable should set pending, got %v", upd)
+	}
+
+	on := &models.Monitor{Type: models.MonitorHeartbeat, Enabled: true}
+	if unset, _ := prepareMonitorUpdate(on, bson.M{"enabled": true}); len(unset) != 0 {
+		t.Fatalf("already enabled should not reset state, got %v", unset)
+	}
+
+	upd = bson.M{"for_sec": -5}
+	if _, err := prepareMonitorUpdate(&models.Monitor{Type: models.MonitorMetric}, upd); err != nil {
+		t.Fatal(err)
+	}
+	if upd["for_sec"] != 0 {
+		t.Fatalf("negative for_sec should clamp to 0, got %v", upd["for_sec"])
+	}
+}

server/internal/services/monitortransition.go

@@ -0,0 +1,78 @@
+package services
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+// decideStatus is the pull-check retry state machine: a success is always up,
+// failures below the retry count leave the status alone (or pending when there
+// is none yet), and reaching the count is down.
+func decideStatus(prev string, fails, retries int, up bool) (string, int) {
+	if retries < 1 {
+		retries = 1
+	}
+	if up {
+		return models.StatusUp, 0
+	}
+	fails++
+	if fails >= retries {
+		return models.StatusDown, fails
+	}
+	if prev == "" || prev == models.StatusPending {
+		return models.StatusPending, fails
+	}
+	return prev, fails
+}
+
+// applyTransition is the one place a status change becomes an incident and a
+// notification. serverID is empty for every monitor except a metric monitor,
+// which keeps one incident per server so each breach opens and resolves on its
+// own.
+func applyTransition(ctx context.Context, m *models.Monitor, serverID, serverName, prev, next, message string, now time.Time) {
+	if next == prev {
+		return
+	}
+	switch next {
+	case models.StatusDown:
+		inc := models.Incident{
+			InstanceID: m.InstanceID,
+			IncidentID: uuid.NewString(),
+			MonitorID:  m.MonitorID,
+			ServerID:   serverID,
+			StartedAt:  now,
+			Cause:      message,
+		}
+		if _, err := db.Col("incidents").InsertOne(ctx, inc); err != nil {
+			log.Printf("monitors: open incident for %s: %v", m.MonitorID, err)
+		}
+		notifyTransition(m, serverName, prev, next, message)
+	case models.StatusUp:
+		if prev != models.StatusDown {
+			return
+		}
+		resolveIncident(ctx, m, serverID, now)
+		notifyTransition(m, serverName, prev, next, message)
+	}
+}
+
+// resolveIncident closes the open incident for a monitor (and server, for a
+// metric monitor) without notifying. It is also used when a server stops
+// matching a metric monitor's selector: nothing recovered, so nobody is told.
+func resolveIncident(ctx context.Context, m *models.Monitor, serverID string, now time.Time) {
+	filter := bson.M{"monitor_id": m.MonitorID, "instance_id": m.InstanceID, "resolved_at": nil}
+	if serverID != "" {
+		filter["server_id"] = serverID
+	} else {
+		filter["server_id"] = bson.M{"$exists": false}
+	}
+	if _, err := db.Col("incidents").UpdateMany(ctx, filter, bson.M{"$set": bson.M{"resolved_at": now}}); err != nil {
+		log.Printf("monitors: resolve incident for %s: %v", m.MonitorID, err)
+	}
+}

server/internal/services/monitortransition_test.go

@@ -0,0 +1,36 @@
+package services
+
+import (
+	"testing"
+
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models"
+)
+
+// decideStatus is the retry state machine lifted out of ingestResult. These
+// cases pin its existing behaviour so the refactor cannot change it.
+func TestDecideStatus(t *testing.T) {
+	cases := []struct {
+		name      string
+		prev      string
+		fails     int
+		retries   int
+		up        bool
+		wantState string
+		wantFails int
+	}{
+		{"up resets fails", models.StatusDown, 3, 2, true, models.StatusUp, 0},
+		{"first failure below retries is pending", models.StatusPending, 0, 3, false, models.StatusPending, 1},
+		{"failure below retries keeps up", models.StatusUp, 0, 3, false, models.StatusUp, 1},
+		{"failure reaching retries is down", models.StatusUp, 2, 3, false, models.StatusDown, 3},
+		{"retries below one treated as one", models.StatusUp, 0, 0, false, models.StatusDown, 1},
+		{"empty prev failing below retries is pending", "", 0, 2, false, models.StatusPending, 1},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, fails := decideStatus(c.prev, c.fails, c.retries, c.up)
+			if got != c.wantState || fails != c.wantFails {
+				t.Fatalf("got (%s,%d), want (%s,%d)", got, fails, c.wantState, c.wantFails)
+			}
+		})
+	}
+}

server/internal/services/passkeys.go

@@ -47,7 +47,7 @@ func GetPasskeyByCredentialID(instanceID string, credID []byte) (*models.WebAuth
 	return &c, nil
 }
 
-func SavePasskey(instanceID, userID, name string, credID, publicKey, aaguid []byte, signCount uint32, transports []string) error {
+func SavePasskey(instanceID, userID, name string, credID, publicKey, aaguid []byte, signCount uint32, transports []string, backupEligible, backupState bool) error {
 	ctx, cancel := mfaCtx()
 	defer cancel()
 	if name == "" {
@@ -62,6 +62,8 @@ func SavePasskey(instanceID, userID, name string, credID, publicKey, aaguid []by
 		AAGUID:          aaguid,
 		SignCount:       signCount,
 		Transports:      transports,
+		BackupEligible:  &backupEligible,
+		BackupState:     backupState,
 		Name:            name,
 		CreatedAt:       time.Now(),
 	})
@@ -71,13 +73,18 @@ func SavePasskey(instanceID, userID, name string, credID, publicKey, aaguid []by
 // TouchPasskey records use and the new signature counter. A counter that fails
 // to advance can mean a cloned authenticator, so the caller checks it before
 // calling this.
-func TouchPasskey(instanceID string, credID []byte, signCount uint32) error {
+func TouchPasskey(instanceID string, credID []byte, signCount uint32, backupEligible, backupState bool) error {
 	ctx, cancel := mfaCtx()
 	defer cancel()
 	now := time.Now()
 	_, err := db.Col("webauthn_credentials").UpdateOne(ctx,
 		bson.M{"instance_id": instanceID, "credential_id": credID},
-		bson.M{"$set": bson.M{"sign_count": signCount, "last_used_at": now}})
+		bson.M{"$set": bson.M{
+			"sign_count": signCount, "last_used_at": now,
+			// Records eligibility on rows that predate it; unchanged otherwise,
+			// because an assertion only succeeds when it matched.
+			"backup_eligible": backupEligible, "backup_state": backupState,
+		}})
 	return err
 }
 

server/internal/services/secret_mask_test.go

@@ -0,0 +1,27 @@
+package services
+
+import (
+	"encoding/base64"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestMaskSecretsEncodedAndMultiline(t *testing.T) {
+	pem := "-----BEGIN KEY-----\nAbCdEfGh12345678\n-----END KEY-----"
+	secrets := map[string]string{"TOKEN": "p@ss w0rd/x+y", "KEY": pem, "SHORT": "ab"}
+	for _, in := range []string{
+		"raw p@ss w0rd/x+y",
+		"b64 " + base64.StdEncoding.EncodeToString([]byte("p@ss w0rd/x+y")),
+		"url " + url.QueryEscape("p@ss w0rd/x+y"),
+		"AbCdEfGh12345678",
+	} {
+		out := maskSecrets(in, secrets)
+		if !strings.Contains(out, "***") || strings.Contains(out, "w0rd") || strings.Contains(out, "AbCd") {
+			t.Errorf("not masked: %q -> %q", in, out)
+		}
+	}
+	if out := maskSecrets("about", secrets); out != "about" {
+		t.Errorf("short secret masked ordinary text: %q", out)
+	}
+}

server/internal/services/tokenscope.go

@@ -105,3 +105,18 @@ func SelectorNarrowerOrEqual(child, parent map[string]string) bool {
 	}
 	return true
 }
+
+// FilterByVisibleServer drops items tied to a server the credential cannot
+// see. Items with no server (every non-metric incident) are always kept.
+func FilterByVisibleServer[T any](items []T, serverID func(T) string, visible map[string]bool, restricted bool) []T {
+	if !restricted {
+		return items
+	}
+	out := items[:0:0]
+	for _, it := range items {
+		if id := serverID(it); id == "" || visible[id] {
+			out = append(out, it)
+		}
+	}
+	return out
+}

server/internal/services/workflow_runner.go

@@ -1,8 +1,11 @@
 package services
 
 import (
+	"encoding/base64"
 	"fmt"
+	"net/url"
 	"os"
+	"sort"
 	"strings"
 	"time"
 
@@ -17,6 +20,10 @@ import (
 
 const stepDispatchGrace = 15 * time.Second
 
+// minMaskLen skips masking values so short that replacing them would shred
+// ordinary output, such as a one-character line of a multi-line secret.
+const minMaskLen = 4
+
 // TriggerWorkflow starts a run of workflow workflowID.
 //
 // tokenScope is the acting credential's tag restriction, nil meaning
@@ -245,7 +252,7 @@ func runServer(instanceID, runID string, srvIdx int, steps []models.ResolvedStep
 
 		marker := fmt.Sprintf("===== step %d/%d: %s (%s) =====", step.Order+1, len(steps), step.Name, step.Interpreter)
 		offset, _ := AppendMarker(runID, serverID, marker)
-		secretsSlice := secretValues(secretVals)
+		secretsSlice := secretValues(allSecrets)
 
 		commandID := uuid.New().String()
 		for attempts < maxAttempts {
@@ -378,13 +385,7 @@ func resolveSecrets(instanceID string, refs []string) map[string]string {
 }
 
 func maskSecrets(s string, secrets map[string]string) string {
-	for _, v := range secrets {
-		if v == "" {
-			continue
-		}
-		s = strings.ReplaceAll(s, v, "***")
-	}
-	return s
+	return string(maskBytes([]byte(s), secretValues(secrets)))
 }
 
 func setServerRun(runID string, srvIdx int, set bson.M) {
@@ -423,11 +424,40 @@ func finishStep(runID, serverID string, order int, status string, attempts, exit
 	})
 }
 
+// secretValues returns every form of the secrets worth masking: the raw value,
+// its base64 and URL-encoded forms, and each line of a multi-line value, since
+// logs are masked a line at a time and a PEM key would otherwise never match.
+// Longest first, so a shorter value never breaks up a longer one before it is
+// replaced.
 func secretValues(m map[string]string) []string {
-	out := make([]string, 0, len(m))
+	seen := map[string]bool{}
+	add := func(v string) {
+		if len(strings.TrimSpace(v)) >= minMaskLen {
+			seen[v] = true
+		}
+	}
 	for _, v := range m {
+		for _, f := range []string{v,
+			base64.StdEncoding.EncodeToString([]byte(v)),
+			base64.RawStdEncoding.EncodeToString([]byte(v)),
+			base64.URLEncoding.EncodeToString([]byte(v)),
+			base64.RawURLEncoding.EncodeToString([]byte(v)),
+			url.QueryEscape(v),
+			url.PathEscape(v),
+		} {
+			add(f)
+		}
+		if strings.Contains(v, "\n") {
+			for _, line := range strings.Split(v, "\n") {
+				add(strings.TrimRight(line, "\r"))
+			}
+		}
+	}
+	out := make([]string, 0, len(seen))
+	for v := range seen {
 		out = append(out, v)
 	}
+	sort.Slice(out, func(i, j int) bool { return len(out[i]) > len(out[j]) })
 	return out
 }
 

web/app/(app)/monitors/[id]/page.tsx

@@ -1,11 +1,13 @@
 "use client";
 
-import { useState } from "react";
+import { useEffect, useState } from "react";
 import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
 import { useParams, useRouter } from "next/navigation";
 import Link from "next/link";
 import { api, Incident, Rollup } from "@/lib/api";
 import { Button, ConfirmDialog, friendlyMessage, useToast } from "@/components/ui";
+import { HeartbeatUrlPanel } from "@/components/monitors/HeartbeatUrlPanel";
+import { MetricServersTable } from "@/components/monitors/MetricServersTable";
 import {
     Slot,
     StatusChip,
@@ -17,6 +19,7 @@ import {
     formatMs,
     formatPct,
     markIncidents,
+    metricCopy,
     relativeTime,
     slotChartColor,
     slotLabel,
@@ -268,14 +271,17 @@ function Figure({ label, value, unit, tone = "" }: { label: string; value: strin
     );
 }
 
-function IncidentRow({ incident }: { incident: Incident }) {
+function IncidentRow({ incident, hostname }: { incident: Incident; hostname?: string }) {
     const open = !incident.resolved_at;
     const started = new Date(incident.started_at);
     return (
         <div className="grid grid-cols-[14px_minmax(0,1fr)] items-start gap-3 border-t border-border-soft px-5 py-3 first:border-t-0">
             <span className={`mt-1.5 block h-2 w-2 rounded-full ${open ? "bg-danger ring-4 ring-danger/20" : "bg-text-tertiary"}`} />
             <div className="min-w-0">
-                <p className="text-[13.5px] text-text-primary">{incident.cause || "Check failed"}</p>
+                <p className="text-[13.5px] text-text-primary">
+                    {incident.cause || "Check failed"}
+                    {hostname && <span className="ml-2 font-mono text-[11px] text-text-tertiary">· {hostname}</span>}
+                </p>
                 <p className="mt-1 flex flex-wrap gap-2 font-mono text-[11px] text-text-tertiary">
                     <span>
                         {started.toLocaleString(undefined, { weekday: "short", hour: "2-digit", minute: "2-digit" })}
@@ -319,9 +325,26 @@ export default function MonitorDetailPage() {
     const queryClient = useQueryClient();
     const monitorId = params.id as string;
     const [confirmDelete, setConfirmDelete] = useState(false);
+    const [confirmRotate, setConfirmRotate] = useState(false);
     const [range, setRange] = useState<Range>(RANGES[0]);
     const toast = useToast();
 
+    /* The token is only ever handed back once, on the create response. The
+       create page parks it in sessionStorage for this one navigation, because a
+       query string would reach proxy access logs. Read it once and remove it
+       straight away so a reload or another tab never shows it again. */
+    const [heartbeatToken, setHeartbeatToken] = useState<string | null>(null);
+    useEffect(() => {
+        const key = `vantage:hbtoken:${monitorId}`;
+        try {
+            const t = sessionStorage.getItem(key);
+            sessionStorage.removeItem(key);
+            if (t) setHeartbeatToken(t);
+        } catch {
+            // Storage blocked: nothing was handed over.
+        }
+    }, [monitorId]);
+
     const { data: monitor, isLoading } = useQuery({
         queryKey: ["monitors", monitorId],
         queryFn: () => api.getMonitor(monitorId),
@@ -363,6 +386,14 @@ export default function MonitorDetailPage() {
         enabled: !!monitor && monitor.runner !== "server",
     });
 
+    const { data: metricServers } = useQuery({
+        queryKey: ["monitors", monitorId, "servers"],
+        queryFn: () => api.monitorServers(monitorId),
+        refetchInterval: 30_000,
+        enabled: monitor?.type === "metric",
+    });
+    const hostnameByServerId = new Map((metricServers ?? []).map((s) => [s.server_id, s.hostname ?? s.server_id]));
+
     const {
         mutate: deleteMonitor,
         isPending: isDeleting,
@@ -387,6 +418,19 @@ export default function MonitorDetailPage() {
         onError: toast.error,
     });
 
+    const {
+        mutate: rotateToken,
+        isPending: isRotating,
+        error: rotateError,
+    } = useMutation({
+        mutationFn: () => api.rotateHeartbeatToken(monitorId),
+        onSuccess: (res) => {
+            setHeartbeatToken(res.heartbeat_token);
+            setConfirmRotate(false);
+            toast.success("Token rotated. The old URL stops working immediately.");
+        },
+    });
+
     if (isLoading) {
         return (
             <div className="flex h-full items-center justify-center">
@@ -430,17 +474,32 @@ export default function MonitorDetailPage() {
         ? Math.round((new Date(monitor.state.cert_expiry_at).getTime() - Date.now()) / 86400_000)
         : null;
 
+    const isHeartbeat = monitor.type === "heartbeat";
     const openIncidents = (incidents ?? []).filter((i) => !i.resolved_at).length;
     const runnerName =
         monitor.runner === "server" ? "Control plane" : servers?.find((s) => s.server_id === monitor.runner)?.hostname || monitor.runner;
     const alertChannels = (monitor.channel_ids ?? []).map((id) => channels?.find((c) => c.channel_id === id));
 
+    const isMetric = monitor.type === "metric";
+    let ruleSummary = "";
+    if (isMetric && monitor.target.metric) {
+        const kind = monitor.target.metric;
+        const copy = metricCopy[kind];
+        const value = copy.unit ? `${monitor.target.threshold ?? ""}${copy.unit}` : "";
+        const forMin = Math.round((monitor.for_sec ?? 0) / 60);
+        const selectorEntries = Object.entries(monitor.target.selector ?? {});
+        const on = selectorEntries.length > 0 ? selectorEntries.map(([k, v]) => `${k}=${v}`).join(", ") : "all servers";
+        ruleSummary = `${copy.label} ${value} for ${forMin} min on ${on}`.replace(/\s+/g, " ").trim();
+    }
+
     return (
         <div className="p-4 sm:p-6 lg:p-8">
             <Link href="/monitors" className="mb-2.5 inline-block text-sm text-text-secondary hover:text-text-primary">
                 ← Monitors
             </Link>
 
+            {isHeartbeat && heartbeatToken && <div className="mb-5"><HeartbeatUrlPanel token={heartbeatToken} /></div>}
+
             <div className="mb-5 flex flex-col gap-4 sm:flex-row sm:items-end sm:justify-between">
                 <div className="min-w-0">
                     <div className="flex flex-wrap items-center gap-2.5">
@@ -496,6 +555,18 @@ export default function MonitorDetailPage() {
                 }
             />
 
+            <ConfirmDialog
+                open={confirmRotate}
+                title="Rotate ping token"
+                confirmLabel="Rotate"
+                destructive={false}
+                loading={isRotating}
+                error={rotateError ? friendlyMessage(rotateError) : null}
+                onClose={() => setConfirmRotate(false)}
+                onConfirm={() => rotateToken()}
+                body={<p>The current URL stops working immediately.</p>}
+            />
+
             <div className="grid grid-cols-1 items-start gap-5 lg:grid-cols-[minmax(0,1fr)_300px]">
                 <div className="flex flex-col gap-5">
                     <div className="rounded-lg border border-border bg-surface">
@@ -525,7 +596,7 @@ export default function MonitorDetailPage() {
                             />
                             <Figure label="Uptime 30d" value={formatPct(pct30d)} unit={pct30d !== null ? "%" : undefined} />
                             <Figure
-                                label="Response now"
+                                label={isHeartbeat ? "Duration" : "Response now"}
                                 value={status === "down" ? "-" : formatMs(latency)}
                                 tone={status === "down" ? "text-danger" : ""}
                             />
@@ -548,26 +619,54 @@ export default function MonitorDetailPage() {
                                 No incidents recorded. Every check has passed since this monitor was created.
                             </p>
                         ) : (
-                            incidents.map((inc) => <IncidentRow key={inc.incident_id} incident={inc} />)
+                            incidents.map((inc) => (
+                                <IncidentRow
+                                    key={inc.incident_id}
+                                    incident={inc}
+                                    hostname={inc.server_id ? hostnameByServerId.get(inc.server_id) ?? inc.server_id : undefined}
+                                />
+                            ))
                         )}
                     </Panel>
+
+                    {isMetric && (
+                        <Panel title="Servers" aside={ruleSummary} padded={false}>
+                            <MetricServersTable monitorId={monitorId} />
+                        </Panel>
+                    )}
                 </div>
 
                 <div className="flex flex-col gap-5">
-                    <Panel title="Check">
-                        <dl className="text-[13px]">
-                            <Row label="Runs from" value={runnerName} />
-                            <Row label="Every" value={`${monitor.interval_sec} s`} />
-                            <Row label="Fails after" value={`${monitor.retries} ${monitor.retries === 1 ? "try" : "tries"}`} />
-                            {monitor.type === "http" && monitor.target.method && <Row label="Method" value={monitor.target.method} />}
-                            {monitor.type === "http" && monitor.target.expected_status && (
-                                <Row label="Expects" value={monitor.target.expected_status} />
-                            )}
-                            {monitor.target.keyword && <Row label="Body contains" value={monitor.target.keyword} />}
-                            {monitor.type === "tls" && <Row label="Warns at" value={`${monitor.target.tls_warn_days ?? 14} days`} />}
-                            <Row label="Last checked" value={relativeTime(monitor.state.last_check_at)} />
-                        </dl>
-                    </Panel>
+                    {isHeartbeat ? (
+                        <Panel title="Heartbeat">
+                            <dl className="text-[13px]">
+                                <Row label="Last ping" value={relativeTime(monitor.state.last_ping_at)} />
+                                <Row label="Expected every" value={`${Math.round((monitor.target.period_sec ?? 0) / 60)} min`} />
+                                <Row label="Grace" value={`${Math.round((monitor.target.grace_sec ?? 0) / 60)} min`} />
+                                {monitor.state.started_at && (
+                                    <Row label="Running since" value={relativeTime(monitor.state.started_at)} />
+                                )}
+                            </dl>
+                            <Button variant="secondary" size="sm" className="mt-4" onClick={() => setConfirmRotate(true)}>
+                                Rotate token
+                            </Button>
+                        </Panel>
+                    ) : isMetric ? null : (
+                        <Panel title="Check">
+                            <dl className="text-[13px]">
+                                <Row label="Runs from" value={runnerName} />
+                                <Row label="Every" value={`${monitor.interval_sec} s`} />
+                                <Row label="Fails after" value={`${monitor.retries} ${monitor.retries === 1 ? "try" : "tries"}`} />
+                                {monitor.type === "http" && monitor.target.method && <Row label="Method" value={monitor.target.method} />}
+                                {monitor.type === "http" && monitor.target.expected_status && (
+                                    <Row label="Expects" value={monitor.target.expected_status} />
+                                )}
+                                {monitor.target.keyword && <Row label="Body contains" value={monitor.target.keyword} />}
+                                {monitor.type === "tls" && <Row label="Warns at" value={`${monitor.target.tls_warn_days ?? 14} days`} />}
+                                <Row label="Last checked" value={relativeTime(monitor.state.last_check_at)} />
+                            </dl>
+                        </Panel>
+                    )}
 
                     <Panel title="Alerts">
                         {alertChannels.length === 0 ? (

web/app/(app)/monitors/new/page.tsx

@@ -24,6 +24,15 @@ export default function NewMonitorPage() {
             // survive the route change, which is the one thing an inline
             // banner on the form cannot do. The error stays on the form.
             toast.success(`Created ${m.name}. First check runs within its interval.`);
+            // The token is shown once. Hand it over in memory, never in the
+            // URL: the RSC fetch for a query string lands in proxy access logs.
+            if (m.heartbeat_token) {
+                try {
+                    sessionStorage.setItem(`vantage:hbtoken:${m.monitor_id}`, m.heartbeat_token);
+                } catch {
+                    // Storage blocked: the user can rotate to get a token.
+                }
+            }
             router.push(`/monitors/${m.monitor_id}`);
         },
     });

web/app/(app)/monitors/page.tsx

@@ -199,7 +199,15 @@ function MonitorRow({ monitor, rollups, incidents }: MonitorRowData) {
                         {monitor.type}
                     </span>
                 </span>
-                <span className="mt-1 block truncate font-mono text-[11.5px] text-text-tertiary">{targetSummary(monitor)}</span>
+                <span className="mt-1 block truncate font-mono text-[11.5px] text-text-tertiary">
+                    {monitor.type === "heartbeat"
+                        ? monitor.state.last_ping_at
+                            ? `Last ping ${relativeTime(monitor.state.last_ping_at)}`
+                            : "Waiting for first ping"
+                        : monitor.type === "metric"
+                          ? (monitor.state.message ?? "Awaiting first check")
+                          : targetSummary(monitor)}
+                </span>
             </div>
 
             <Tape slots={slots} live={monitor.enabled} />

web/components/monitors/HeartbeatUrlPanel.tsx

@@ -0,0 +1,35 @@
+"use client";
+
+import { heartbeatUrl } from "@/lib/api";
+import { Button, useToast } from "@/components/ui";
+
+/** Shown once, straight after create or rotate: the token is not retrievable later. */
+export function HeartbeatUrlPanel({ token }: { token: string }) {
+    const toast = useToast();
+    const url = heartbeatUrl(token);
+    const copy = async () => {
+        await navigator.clipboard.writeText(url);
+        toast.success("Ping URL copied");
+    };
+    return (
+        <div className="space-y-3 rounded border border-accent/40 bg-well p-4">
+            <p className="text-sm text-text-primary">
+                Copy this URL now. It is not shown again, rotate the token if you lose it.
+            </p>
+            <div className="flex items-center gap-2">
+                <code className="flex-1 overflow-x-auto rounded bg-surface px-2 py-1.5 font-mono text-xs">{url}</code>
+                <Button size="sm" onClick={copy}>Copy</Button>
+            </div>
+            <pre className="overflow-x-auto rounded bg-surface p-3 font-mono text-[11px] leading-relaxed text-text-secondary">
+{`# success
+curl -fsS -m 10 --retry 3 ${url}
+# mark start, to measure duration
+curl -fsS -m 10 ${url}/start
+# report failure with output
+your-job 2>&1 | tail -c 1024 | curl -fsS -m 10 --data-binary @- ${url}/fail
+# or keep the token out of URLs (and your proxy logs)
+curl -fsS -m 10 -X POST -H "X-Vantage-Token: ${token}" ${window.location.origin}/public/hb`}
+            </pre>
+        </div>
+    );
+}

web/components/monitors/MetricServersTable.tsx

@@ -0,0 +1,69 @@
+"use client";
+
+import Link from "next/link";
+import { useQuery } from "@tanstack/react-query";
+import { api } from "@/lib/api";
+import { StatusChip, relativeTime } from "@/components/monitors/MonitorVisuals";
+
+/*
+ * Per-server state for a metric monitor's rule. Polls the same way the
+ * detail page's other queries do - a 30s refetchInterval - rather than a
+ * bespoke subscription for one panel.
+ */
+
+const order: Record<string, number> = { down: 0, pending: 1, up: 2 };
+
+export function MetricServersTable({ monitorId }: { monitorId: string }) {
+    const { data, isLoading } = useQuery({
+        queryKey: ["monitors", monitorId, "servers"],
+        queryFn: () => api.monitorServers(monitorId),
+        refetchInterval: 30_000,
+    });
+
+    const rows = [...(data ?? [])].sort((a, b) => (order[a.status] ?? 3) - (order[b.status] ?? 3));
+
+    if (isLoading) {
+        return (
+            <div className="flex justify-center py-10">
+                <div className="h-6 w-6 animate-spin rounded-full border-2 border-border border-t-accent" />
+            </div>
+        );
+    }
+
+    if (rows.length === 0) {
+        return <p className="px-5 py-12 text-center text-sm text-text-secondary">No servers match this rule&apos;s tags.</p>;
+    }
+
+    return (
+        <div className="overflow-x-auto">
+            <table className="w-full text-left text-[13px]">
+                <thead>
+                    <tr className="border-b border-border-soft text-[10px] uppercase tracking-[0.14em] text-text-tertiary">
+                        <th className="px-5 py-2.5 font-medium">Server</th>
+                        <th className="px-5 py-2.5 font-medium">Status</th>
+                        <th className="px-5 py-2.5 font-medium">Value</th>
+                        <th className="px-5 py-2.5 font-medium">Since</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {rows.map((row) => (
+                        <tr key={row.server_id} className="border-t border-border-soft first:border-t-0">
+                            <td className="px-5 py-2.5">
+                                <Link href={`/servers/${row.server_id}`} className="text-text-primary hover:text-accent hover:underline">
+                                    {row.hostname ?? row.server_id}
+                                </Link>
+                            </td>
+                            <td className="px-5 py-2.5">
+                                <StatusChip status={row.status} />
+                            </td>
+                            <td className="px-5 py-2.5 font-mono text-text-secondary">{row.message ?? "-"}</td>
+                            <td className="px-5 py-2.5 font-mono text-text-tertiary">
+                                {row.breach_since ? relativeTime(row.breach_since) : "-"}
+                            </td>
+                        </tr>
+                    ))}
+                </tbody>
+            </table>
+        </div>
+    );
+}

web/components/monitors/MonitorForm.tsx

@@ -3,9 +3,10 @@
 import { useState } from "react";
 import { useQuery } from "@tanstack/react-query";
 import Link from "next/link";
-import { api, Monitor, MonitorInput, MonitorType } from "@/lib/api";
+import { api, MetricKind, Monitor, MonitorInput, MonitorType } from "@/lib/api";
 import { Button } from "@/components/ui";
-import { formatDuration } from "@/components/monitors/MonitorVisuals";
+import { formatDuration, metricCopy } from "@/components/monitors/MonitorVisuals";
+import { TagRestriction } from "@/components/apikeys/TagRestriction";
 
 /*
  * The form is grouped the way the detail page reads it back: what is checked,
@@ -17,11 +18,13 @@ import { formatDuration } from "@/components/monitors/MonitorVisuals";
 const inputClass =
     "w-full rounded-lg border border-border bg-surface-2 px-3 py-2 text-sm text-text-primary placeholder:text-text-tertiary focus:border-accent/50 focus:outline-none focus:ring-1 focus:ring-accent/30";
 
-const typeCopy: Record<MonitorType, { title: string; blurb: string; target: string }> = {
+const typeCopy: Partial<Record<MonitorType, { title: string; blurb: string; target: string }>> = {
     http: { title: "HTTP", blurb: "Requests a URL and checks the response", target: "URL" },
     tcp: { title: "TCP", blurb: "Opens a socket on a host and port", target: "Host and port" },
     icmp: { title: "Ping", blurb: "Pings a host and measures round trip", target: "Host" },
     tls: { title: "TLS", blurb: "Reads a certificate and counts days left", target: "Host and port" },
+    heartbeat: { title: "Heartbeat", blurb: "Your job calls a URL; alert when it stops.", target: "Ping URL" },
+    metric: { title: "Server metric", blurb: "Alert on disk, memory, load, units or reboots across tagged servers.", target: "Rule" },
 };
 
 function Section({ title, hint, children }: { title: string; hint?: string; children: React.ReactNode }) {
@@ -106,6 +109,15 @@ export function MonitorForm({
     const [runner, setRunner] = useState(initial?.runner ?? "server");
     const [enabled, setEnabled] = useState(initial?.enabled ?? true);
     const [channelIds, setChannelIds] = useState<string[]>(initial?.channel_ids ?? []);
+    const [periodMin, setPeriodMin] = useState(Math.round((initial?.target.period_sec ?? 3600) / 60));
+    const [graceMin, setGraceMin] = useState(Math.round((initial?.target.grace_sec ?? 300) / 60));
+    const [metric, setMetric] = useState<MetricKind>(initial?.target.metric ?? "disk_pct");
+    const [threshold, setThreshold] = useState<number>(initial?.target.threshold ?? 90);
+    const [mount, setMount] = useState(initial?.target.mount ?? "");
+    // for_sec 0 is omitted from JSON, so an existing monitor without it means
+    // "alert immediately"; only a new monitor gets the 5 minute default.
+    const [forMin, setForMin] = useState(Math.round((initial ? (initial.for_sec ?? 0) : 300) / 60));
+    const [selector, setSelector] = useState<Record<string, string>>(initial?.target.selector ?? {});
 
     const { data: servers } = useQuery({ queryKey: ["servers"], queryFn: () => api.listServers() });
     /* The group is free text, so the existing groups are offered as suggestions
@@ -130,13 +142,27 @@ export function MonitorForm({
             target.tls_warn_days = tlsWarnDays;
         } else if (type === "icmp") {
             target.host = host;
+        } else if (type === "heartbeat") {
+            target.period_sec = periodMin * 60;
+            target.grace_sec = graceMin * 60;
+        } else if (type === "metric") {
+            target.metric = metric;
+            target.selector = selector;
+            if (metricCopy[metric].unit) target.threshold = threshold;
+            if (metricCopy[metric].mount && mount) target.mount = mount;
         } else {
             target.host = host;
             target.port = port;
         }
-        onSubmit({ name, group: group.trim(), type, target, interval_sec: intervalSec, retries, runner, enabled, channel_ids: channelIds });
+        const input: MonitorInput = { name, group: group.trim(), type, target, interval_sec: intervalSec, retries, runner, enabled, channel_ids: channelIds };
+        if (type === "metric") input.for_sec = forMin * 60;
+        onSubmit(input);
     }
 
+    /* The server rejects a type change to or from heartbeat or metric, so an
+       edit of one of those locks the picker rather than letting the operator
+       hit a save error. */
+    const typePickerDisabled = !!initial && (initial.type === "heartbeat" || initial.type === "metric");
     const runnerName = runner === "server" ? "the control plane" : servers?.find((s) => s.server_id === runner)?.hostname || "an agent";
     /* Retries are consecutive failures, so the delay before "down" is one
        interval per remaining try after the first. Say it in minutes, because
@@ -146,7 +172,7 @@ export function MonitorForm({
     return (
         <form onSubmit={handleSubmit} className="flex flex-col gap-5">
             <div className="grid grid-cols-1 items-start gap-5 lg:grid-cols-[minmax(0,1fr)_minmax(320px,400px)]">
-                <Section title="Check" hint={typeCopy[type].target}>
+                <Section title="Check" hint={typeCopy[type]?.target}>
                     <div className="grid grid-cols-1 gap-4 sm:grid-cols-[minmax(0,1.6fr)_minmax(0,1fr)]">
                         <Field label="Name" help="Shown in the fleet list and in every alert this check sends.">
                             <input className={inputClass} value={name} onChange={(e) => setName(e.target.value)} placeholder="Billing API" required />
@@ -177,7 +203,8 @@ export function MonitorForm({
                                     type="button"
                                     onClick={() => setType(t)}
                                     aria-pressed={type === t}
-                                    className={`rounded-lg border px-3 py-2.5 text-left transition-colors ${
+                                    disabled={typePickerDisabled}
+                                    className={`rounded-lg border px-3 py-2.5 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-50 ${
                                         type === t
                                             ? "border-accent bg-accent/10"
                                             : "border-border bg-surface-2 hover:border-accent/40"
@@ -186,9 +213,9 @@ export function MonitorForm({
                                     <span
                                         className={`block font-mono text-[11px] uppercase tracking-[0.1em] ${type === t ? "text-accent" : "text-text-secondary"}`}
                                     >
-                                        {typeCopy[t].title}
+                                        {typeCopy[t]?.title}
                                     </span>
-                                    <span className="mt-1 block text-[11px] leading-snug text-text-tertiary">{typeCopy[t].blurb}</span>
+                                    <span className="mt-1 block text-[11px] leading-snug text-text-tertiary">{typeCopy[t]?.blurb}</span>
                                 </button>
                             ))}
                         </div>
@@ -268,50 +295,126 @@ export function MonitorForm({
                             />
                         </Field>
                     )}
-                </Section>
 
-                <div className="flex flex-col gap-5">
-                    <Section title="Schedule">
+                    {type === "heartbeat" && (
                         <div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
-                            <Field label="Run every" help="Seconds between checks. Minimum 10.">
+                            <Field label="Expected every (minutes)">
                                 <input
                                     type="number"
                                     className={`${inputClass} font-mono tabular-nums`}
-                                    value={intervalSec}
-                                    onChange={(e) => setIntervalSec(Number(e.target.value))}
-                                    min={10}
-                                />
-                            </Field>
-                            <Field label="Fails after" help="Consecutive failures before an incident opens.">
-                                <input
-                                    type="number"
-                                    className={`${inputClass} font-mono tabular-nums`}
-                                    value={retries}
-                                    onChange={(e) => setRetries(Number(e.target.value))}
+                                    value={periodMin}
+                                    onChange={(e) => setPeriodMin(Number(e.target.value))}
                                     min={1}
                                 />
                             </Field>
+                            <Field label="Grace (minutes)">
+                                <input
+                                    type="number"
+                                    className={`${inputClass} font-mono tabular-nums`}
+                                    value={graceMin}
+                                    onChange={(e) => setGraceMin(Number(e.target.value))}
+                                    min={0}
+                                />
+                            </Field>
                         </div>
+                    )}
 
-                        <Field
-                            label="Runs from"
-                            help="Pick an agent for anything only reachable from inside that network. Everything else runs centrally."
-                        >
-                            <select className={inputClass} value={runner} onChange={(e) => setRunner(e.target.value)}>
-                                <option value="server">Control plane</option>
-                                {servers?.map((s) => (
-                                    <option key={s.server_id} value={s.server_id}>
-                                        Agent · {s.hostname}
-                                    </option>
-                                ))}
-                            </select>
-                        </Field>
+                    {type === "metric" && (
+                        <>
+                            <Field label="Metric">
+                                <select className={inputClass} value={metric} onChange={(e) => setMetric(e.target.value as MetricKind)}>
+                                    {(Object.keys(metricCopy) as MetricKind[]).map((k) => (
+                                        <option key={k} value={k}>
+                                            {metricCopy[k].label}
+                                        </option>
+                                    ))}
+                                </select>
+                            </Field>
 
-                        <p className="rounded-lg bg-well px-4 py-3 font-mono text-[11.5px] leading-relaxed text-text-secondary">
-                            Checked every {intervalSec} s from {runnerName}. Reported down after {retries}{" "}
-                            {retries === 1 ? "failure" : "consecutive failures"} - roughly {downAfter}.
-                        </p>
-                    </Section>
+                            <div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
+                                {metricCopy[metric].unit && (
+                                    <Field label={`Threshold${metricCopy[metric].unit ? ` (${metricCopy[metric].unit})` : ""}`}>
+                                        <input
+                                            type="number"
+                                            className={`${inputClass} font-mono tabular-nums`}
+                                            value={threshold}
+                                            onChange={(e) => setThreshold(Number(e.target.value))}
+                                        />
+                                    </Field>
+                                )}
+                                {metricCopy[metric].mount && (
+                                    <Field label="Mount">
+                                        <input
+                                            className={`${inputClass} font-mono`}
+                                            value={mount}
+                                            onChange={(e) => setMount(e.target.value)}
+                                            placeholder="Any mount"
+                                        />
+                                    </Field>
+                                )}
+                                <Field label="For (minutes)" help="How long the breach must hold before it alerts.">
+                                    <input
+                                        type="number"
+                                        className={`${inputClass} font-mono tabular-nums`}
+                                        value={forMin}
+                                        onChange={(e) => setForMin(Number(e.target.value))}
+                                        min={0}
+                                    />
+                                </Field>
+                            </div>
+
+                            <Field label="Servers">
+                                <TagRestriction selector={selector} onChange={setSelector} />
+                                <span className="mt-1.5 block text-xs text-text-tertiary">Leave empty to watch every server.</span>
+                            </Field>
+                        </>
+                    )}
+                </Section>
+
+                <div className="flex flex-col gap-5">
+                    {type !== "heartbeat" && type !== "metric" && (
+                        <Section title="Schedule">
+                            <div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
+                                <Field label="Run every" help="Seconds between checks. Minimum 10.">
+                                    <input
+                                        type="number"
+                                        className={`${inputClass} font-mono tabular-nums`}
+                                        value={intervalSec}
+                                        onChange={(e) => setIntervalSec(Number(e.target.value))}
+                                        min={10}
+                                    />
+                                </Field>
+                                <Field label="Fails after" help="Consecutive failures before an incident opens.">
+                                    <input
+                                        type="number"
+                                        className={`${inputClass} font-mono tabular-nums`}
+                                        value={retries}
+                                        onChange={(e) => setRetries(Number(e.target.value))}
+                                        min={1}
+                                    />
+                                </Field>
+                            </div>
+
+                            <Field
+                                label="Runs from"
+                                help="Pick an agent for anything only reachable from inside that network. Everything else runs centrally."
+                            >
+                                <select className={inputClass} value={runner} onChange={(e) => setRunner(e.target.value)}>
+                                    <option value="server">Control plane</option>
+                                    {servers?.map((s) => (
+                                        <option key={s.server_id} value={s.server_id}>
+                                            Agent · {s.hostname}
+                                        </option>
+                                    ))}
+                                </select>
+                            </Field>
+
+                            <p className="rounded-lg bg-well px-4 py-3 font-mono text-[11.5px] leading-relaxed text-text-secondary">
+                                Checked every {intervalSec} s from {runnerName}. Reported down after {retries}{" "}
+                                {retries === 1 ? "failure" : "consecutive failures"} - roughly {downAfter}.
+                            </p>
+                        </Section>
+                    )}
 
                     <Section title="Alerts" hint={channelIds.length > 0 ? `${channelIds.length} selected` : undefined}>
                         {!channels || channels.length === 0 ? (

web/components/monitors/MonitorVisuals.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { Incident, Monitor, MonitorSample, MonitorStatus, Rollup } from "@/lib/api";
+import { Incident, MetricKind, Monitor, MonitorSample, MonitorStatus, Rollup } from "@/lib/api";
 
 /*
  * Shared vocabulary for the monitors screens.
@@ -82,10 +82,25 @@ export function formatMs(ms: number | null): string {
     return `${Math.round(ms)} ms`;
 }
 
+/** Label and unit per metric kind, shared by the rule builder and the detail
+ *  page's summary line. `mount` marks a kind that also carries a mountpoint. */
+export const metricCopy: Record<MetricKind, { label: string; unit?: string; mount?: boolean }> = {
+    disk_pct: { label: "Disk used", unit: "%", mount: true },
+    disk_free_gb: { label: "Disk free below", unit: "GB", mount: true },
+    mem_pct: { label: "Memory used", unit: "%" },
+    load_per_core: { label: "Load per core", unit: "×" },
+    unit_failed: { label: "Systemd unit failed" },
+    container_unhealthy: { label: "Container unhealthy" },
+    reboot_pending_days: { label: "Reboot pending for", unit: "days" },
+    agent_offline_min: { label: "Agent offline for", unit: "minutes" },
+};
+
 export function targetSummary(m: Monitor): string {
     if (m.type === "http") return m.target.url ?? "";
     if (m.type === "tls") return `${m.target.host ?? ""}:${m.target.port || 443}`;
     if (m.type === "icmp") return m.target.host ?? "";
+    if (m.type === "heartbeat") return "Ping URL";
+    if (m.type === "metric") return m.target.metric ? metricCopy[m.target.metric].label : "";
     return `${m.target.host ?? ""}:${m.target.port ?? ""}`;
 }
 

web/e2e/heartbeat.spec.ts

@@ -0,0 +1,50 @@
+import { test, expect } from "@playwright/test";
+import { ensureOwner, signIn } from "./helpers";
+
+const BASE_URL = process.env.E2E_BASE_URL ?? "http://localhost:3000";
+
+test("heartbeat monitor: URL ping, /fail, header ping", async ({ page, request }) => {
+    const owner = await ensureOwner(request);
+    await signIn(page, owner.email, owner.password);
+
+    const create = await page.request.post(`${BASE_URL}/api/monitors`, {
+        data: {
+            name: `e2e heartbeat ${Date.now()}`,
+            type: "heartbeat",
+            target: { period_sec: 3600, grace_sec: 300 },
+            enabled: true,
+        },
+    });
+    expect(create.status()).toBe(201);
+    const monitor = await create.json();
+    expect(monitor.heartbeat_token).toBeTruthy();
+
+    const ping = await request.get(`${BASE_URL}/public/hb/${monitor.heartbeat_token}`);
+    expect(ping.status()).toBe(200);
+    expect(await ping.text()).toBe("OK");
+
+    let got = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}`)).json();
+    expect(got.state.status).toBe("up");
+
+    const fail = await request.post(`${BASE_URL}/public/hb/${monitor.heartbeat_token}/fail`, { data: "disk full" });
+    expect(fail.status()).toBe(200);
+
+    got = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}`)).json();
+    expect(got.state.status).toBe("down");
+    expect(got.state.message).toContain("disk full");
+
+    const incidents = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}/incidents`)).json();
+    expect(incidents.length).toBeGreaterThan(0);
+
+    await new Promise((r) => setTimeout(r, 1100)); // same kind (ping) as the first request; limit is 1/s per token and kind
+    const headerPing = await request.post(`${BASE_URL}/public/hb`, { headers: { "X-Vantage-Token": monitor.heartbeat_token } });
+    expect(headerPing.status()).toBe(200);
+    got = await (await page.request.get(`${BASE_URL}/api/monitors/${monitor.monitor_id}`)).json();
+    expect(got.state.status).toBe("up");
+
+    expect((await request.get(`${BASE_URL}/public/hb/not-a-real-token`)).status()).toBe(404);
+    expect((await request.post(`${BASE_URL}/public/hb`)).status()).toBe(404);
+
+    await page.goto(`${BASE_URL}/monitors/${monitor.monitor_id}`);
+    await expect(page.getByText(/last ping/i)).toBeVisible();
+});

web/e2e/helpers.ts

@@ -0,0 +1,36 @@
+import { type APIRequestContext, type Page } from "@playwright/test";
+
+export const BASE_URL = process.env.E2E_BASE_URL ?? "http://localhost:3000";
+
+let ownerEmail = process.env.E2E_OWNER_EMAIL ?? "";
+let ownerPassword = process.env.E2E_OWNER_PASSWORD ?? "";
+
+/** Ensures an owner account exists and returns its credentials, bootstrapping the instance if needed. */
+export async function ensureOwner(request: APIRequestContext): Promise<{ email: string; password: string }> {
+    if (ownerEmail && ownerPassword) return { email: ownerEmail, password: ownerPassword };
+
+    const status = await request.get(`${BASE_URL}/auth/bootstrap-status`);
+    const body = await status.json();
+    if (!body.needs_setup) {
+        throw new Error(
+            "Instance is already bootstrapped and E2E_OWNER_EMAIL/E2E_OWNER_PASSWORD were not set - " +
+                "cannot create the owner account this suite needs to provision fresh members per test.",
+        );
+    }
+
+    ownerEmail = `e2e-owner-${Date.now()}@vantage.test`;
+    ownerPassword = "correct horse battery staple 1";
+    const res = await request.post(`${BASE_URL}/auth/bootstrap`, {
+        data: { instance_name: "E2E", email: ownerEmail, password: ownerPassword },
+    });
+    if (!res.ok()) throw new Error(`bootstrap failed: ${res.status()} ${await res.text()}`);
+    return { email: ownerEmail, password: ownerPassword };
+}
+
+/** Fills the login form and submits it. */
+export async function signIn(page: Page, email: string, password: string) {
+    await page.goto("/login");
+    await page.getByLabel("Email").fill(email);
+    await page.getByLabel("Password").fill(password);
+    await page.getByRole("button", { name: "Sign In" }).click();
+}

web/e2e/mfa.spec.ts

@@ -1,5 +1,6 @@
 import { test, expect, type APIRequestContext, type BrowserContext, type Page } from "@playwright/test";
 import { authenticator } from "otplib";
+import { BASE_URL, ensureOwner, signIn } from "./helpers";
 
 /**
  * End-to-end coverage for Task 15 of the MFA design
@@ -17,33 +18,6 @@ import { authenticator } from "otplib";
  * at an already-bootstrapped instance's owner instead.
  */
 
-const BASE_URL = process.env.E2E_BASE_URL ?? "http://localhost:3000";
-
-let ownerEmail = process.env.E2E_OWNER_EMAIL ?? "";
-let ownerPassword = process.env.E2E_OWNER_PASSWORD ?? "";
-
-/** Ensures an owner account exists and returns its credentials, bootstrapping the instance if needed. */
-async function ensureOwner(request: APIRequestContext): Promise<{ email: string; password: string }> {
-    if (ownerEmail && ownerPassword) return { email: ownerEmail, password: ownerPassword };
-
-    const status = await request.get(`${BASE_URL}/auth/bootstrap-status`);
-    const body = await status.json();
-    if (!body.needs_setup) {
-        throw new Error(
-            "Instance is already bootstrapped and E2E_OWNER_EMAIL/E2E_OWNER_PASSWORD were not set - " +
-                "cannot create the owner account this suite needs to provision fresh members per test.",
-        );
-    }
-
-    ownerEmail = `e2e-owner-${Date.now()}@vantage.test`;
-    ownerPassword = "correct horse battery staple 1";
-    const res = await request.post(`${BASE_URL}/auth/bootstrap`, {
-        data: { instance_name: "MFA E2E", email: ownerEmail, password: ownerPassword },
-    });
-    if (!res.ok()) throw new Error(`bootstrap failed: ${res.status()} ${await res.text()}`);
-    return { email: ownerEmail, password: ownerPassword };
-}
-
 /** Creates a fresh, unique local member with no MFA enrolled, using an owner session. */
 async function createMember(request: APIRequestContext): Promise<{ email: string; password: string }> {
     const owner = await ensureOwner(request);
@@ -80,19 +54,12 @@ async function addVirtualAuthenticator(context: BrowserContext, page: Page): Pro
     return authenticatorId;
 }
 
-async function fillCredentials(page: Page, email: string, password: string) {
-    await page.goto("/login");
-    await page.getByLabel("Email").fill(email);
-    await page.getByLabel("Password").fill(password);
-    await page.getByRole("button", { name: "Sign In" }).click();
-}
-
 test.describe("MFA end-to-end", () => {
     test("TOTP sign-in", async ({ page, request }) => {
         const { email, password } = await createMember(request);
 
         // Sign in, land on the account security page, enrol TOTP.
-        await fillCredentials(page, email, password);
+        await signIn(page, email, password);
         await expect(page).toHaveURL("/");
 
         await page.goto("/account/security");
@@ -111,7 +78,7 @@ test.describe("MFA end-to-end", () => {
 
         // Sign out and sign back in - TOTP is now required.
         await page.request.post(`${BASE_URL}/auth/logout`);
-        await fillCredentials(page, email, password);
+        await signIn(page, email, password);
 
         await expect(page.getByText("Enter the 6-digit code")).toBeVisible();
         const nextCode = authenticator.generate(secret.trim());
@@ -125,7 +92,7 @@ test.describe("MFA end-to-end", () => {
         const { email, password } = await createMember(request);
         await addVirtualAuthenticator(context, page);
 
-        await fillCredentials(page, email, password);
+        await signIn(page, email, password);
         await expect(page).toHaveURL("/");
 
         await page.goto("/account/security");
@@ -138,7 +105,7 @@ test.describe("MFA end-to-end", () => {
         await expect(page.getByText(/Added .* · Last used/)).toBeVisible();
 
         await page.request.post(`${BASE_URL}/auth/logout`);
-        await fillCredentials(page, email, password);
+        await signIn(page, email, password);
 
         await expect(page.getByText("Enter the 6-digit code")).toBeVisible();
         await page.getByRole("button", { name: "Use passkey" }).click();
@@ -151,7 +118,7 @@ test.describe("MFA end-to-end", () => {
         await addVirtualAuthenticator(context, page);
 
         // Enrol a passkey first (typing the password once, during setup only).
-        await fillCredentials(page, email, password);
+        await signIn(page, email, password);
         await expect(page).toHaveURL("/");
         await page.goto("/account/security");
         await page.getByRole("button", { name: "Add a passkey" }).click();
@@ -180,7 +147,7 @@ test.describe("MFA end-to-end", () => {
         await request.post(`${BASE_URL}/auth/logout`);
 
         // The member, who has no MFA yet, must enrol before reaching the app.
-        await fillCredentials(page, email, password);
+        await signIn(page, email, password);
         await expect(page.getByText("This instance requires a second sign-in factor. Set one up to continue.")).toBeVisible();
 
         await page.getByRole("button", { name: "Use an authenticator app" }).click();
@@ -204,7 +171,7 @@ test.describe("MFA end-to-end", () => {
 
         // Enrol TOTP so the account has a factor - not exercised in this test,
         // since sign-in itself sets StepUpAt and the window has not gone stale.
-        await fillCredentials(page, email, password);
+        await signIn(page, email, password);
         await page.goto("/account/security");
         await page.getByRole("button", { name: "Set up" }).click();
         await page.getByRole("button", { name: "Use an authenticator app" }).click();

web/lib/api.ts

@@ -45,9 +45,13 @@ export interface Server {
     tags?: Record<string, string>;
 }
 
-export type MonitorType = "http" | "tcp" | "icmp" | "tls";
+export type MonitorType = "http" | "tcp" | "icmp" | "tls" | "heartbeat" | "metric";
 export type MonitorStatus = "up" | "down" | "pending";
 
+export type MetricKind =
+    | "disk_pct" | "disk_free_gb" | "mem_pct" | "load_per_core"
+    | "unit_failed" | "container_unhealthy" | "reboot_pending_days" | "agent_offline_min";
+
 export interface MonitorTarget {
     url?: string;
     host?: string;
@@ -57,6 +61,12 @@ export interface MonitorTarget {
     keyword?: string;
     tls_warn_days?: number;
     insecure?: boolean;
+    period_sec?: number;
+    grace_sec?: number;
+    selector?: Record<string, string>;
+    metric?: MetricKind;
+    threshold?: number;
+    mount?: string;
 }
 
 export interface MonitorState {
@@ -66,6 +76,8 @@ export interface MonitorState {
     message?: string;
     cert_expiry_at?: string;
     fails: number;
+    last_ping_at?: string;
+    started_at?: string;
 }
 
 export interface Monitor {
@@ -82,6 +94,9 @@ export interface Monitor {
     channel_ids?: string[];
     state: MonitorState;
     created_at: string;
+    for_sec?: number;
+    /** Plaintext heartbeat ping token. Only ever present on the create response. */
+    heartbeat_token?: string;
 }
 
 export interface MonitorInput {
@@ -94,6 +109,7 @@ export interface MonitorInput {
     retries: number;
     enabled: boolean;
     channel_ids?: string[];
+    for_sec?: number;
 }
 
 export interface Incident {
@@ -102,6 +118,19 @@ export interface Incident {
     started_at: string;
     resolved_at?: string;
     cause?: string;
+    server_id?: string;
+}
+
+/** Per-server state for a metric monitor's rule, one row per matching server. */
+export interface MonitorServerState {
+    monitor_id: string;
+    server_id: string;
+    hostname?: string;
+    status: MonitorStatus;
+    breach_since?: string;
+    value: number;
+    message?: string;
+    updated_at: string;
 }
 
 /** One check result. Kept for 48 hours, which is what the sub-hour views read. */
@@ -1018,6 +1047,10 @@ export const api = {
         return request<Incident[]>(`/monitors/${monitorId}/incidents`);
     },
 
+    monitorServers(monitorId: string): Promise<MonitorServerState[]> {
+        return request<MonitorServerState[]>(`/monitors/${monitorId}/servers`);
+    },
+
     getMonitorUptime(monitorId: string): Promise<Rollup[]> {
         return request<Rollup[]>(`/monitors/${monitorId}/uptime`);
     },
@@ -1026,6 +1059,10 @@ export const api = {
         return request<MonitorSample[]>(`/monitors/${monitorId}/samples?minutes=${minutes}`);
     },
 
+    rotateHeartbeatToken(monitorId: string) {
+        return request<{ heartbeat_token: string }>(`/monitors/${monitorId}/rotate-token`, { method: "POST" });
+    },
+
     listStatusPages(): Promise<StatusPage[]> {
         return request<StatusPage[]>("/status-pages");
     },
@@ -1441,6 +1478,11 @@ export const api = {
     },
 };
 
+/** The ping URL a job calls. Same origin as the app: /public is routed to the server everywhere. */
+export function heartbeatUrl(token: string): string {
+    return `${window.location.origin}/public/hb/${token}`;
+}
+
 export type LicenseState = "valid" | "expired" | "invalid";
 
 export interface LicenseInfo {