fix(server): per-org settings and ESO read token

The settings collection was a single global document, so every org
shared one SMTP config, alert config, retention policy and ESO read
token. GetSecretGroupDecryptedAny then flattened every org's secrets
for a group into one map, meaning any tenant's token read every
tenant's secrets.

- settings gains org_id; GetSettings/SaveSettings/RotateSecretsReadToken/
  GetWorkflowLogRetentionDays all take orgID
- VerifySecretsReadToken replaced by ResolveSecretsReadToken, which
  resolves the org from the presented token's hash; the ESO endpoint
  derives its org from the token rather than a session, since it is
  called machine-to-machine
- GetSecretGroupDecryptedAny deleted in favour of the org-scoped variant
- settings and token-rotation routes now require owner/admin
- offline sweep and log retention resolve org per server / per run
- migration 0002 stamps the legacy settings doc with the default org

Note: /api/settings now 403s for members; the web settings page needs a
matching role check.

server/internal/api/handlers.go

@@ -59,9 +59,13 @@ func RegisterRoutes(r *gin.Engine) {
 
 		apiGroup.GET("/audit", listAuditEvents)
 
-		apiGroup.GET("/settings", getSettings)
-		apiGroup.PUT("/settings", saveSettings)
-		apiGroup.POST("/settings/secrets-token", rotateSecretsToken)
+		settings := apiGroup.Group("/settings")
+		settings.Use(auth.RequireRole("owner", "admin"))
+		{
+			settings.GET("", getSettings)
+			settings.PUT("", saveSettings)
+			settings.POST("/secrets-token", rotateSecretsToken)
+		}
 
 		apiGroup.GET("/secrets", listSecretGroups)
 		apiGroup.POST("/secrets", createSecretGroup)
@@ -456,7 +460,7 @@ func listAuditEvents(c *gin.Context) {
 }
 
 func getSettings(c *gin.Context) {
-	s, err := services.GetSettings()
+	s, err := services.GetSettings(auth.OrgID(c))
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -474,7 +478,7 @@ func saveSettings(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
-	if err := services.SaveSettings(body.Alerts, body.Email, body.WorkflowLogRetentionDays); err != nil {
+	if err := services.SaveSettings(auth.OrgID(c), body.Alerts, body.Email, body.WorkflowLogRetentionDays); err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}

server/internal/api/secrets.go

@@ -19,19 +19,29 @@ func validName(s string) bool {
 	return s != "" && len(s) <= 128 && groupNamePattern.MatchString(s)
 }
 
-// secretsReadAuth validates the ESO bearer token on the public read endpoint.
+// ctxSecretsOrgKey carries the org resolved from the ESO bearer token.
+const ctxSecretsOrgKey = "km_secrets_org"
+
+// secretsReadAuth validates the ESO bearer token on the public read endpoint
+// and stashes the org the token belongs to.
+//
+// This is the one endpoint whose org does NOT come from the session or the
+// host: External Secrets Operator calls it machine-to-machine with no session,
+// so the token itself is the org-bearing credential.
 func secretsReadAuth() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		const prefix = "Bearer "
-		auth := c.GetHeader("Authorization")
-		if len(auth) <= len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {
+		authHeader := c.GetHeader("Authorization")
+		if len(authHeader) <= len(prefix) || !strings.EqualFold(authHeader[:len(prefix)], prefix) {
 			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing bearer token"})
 			return
 		}
-		if !services.VerifySecretsReadToken(auth[len(prefix):]) {
+		orgID, ok := services.ResolveSecretsReadToken(authHeader[len(prefix):])
+		if !ok {
 			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid token"})
 			return
 		}
+		c.Set(ctxSecretsOrgKey, orgID)
 		c.Next()
 	}
 }
@@ -41,7 +51,9 @@ func secretsReadAuth() gin.HandlerFunc {
 // (ESO treats 404 as "deleted").
 func esoGetGroup(c *gin.Context) {
 	group := c.Param("group")
-	values, err := services.GetSecretGroupDecryptedAny(group)
+	// Org comes from the bearer token (set by secretsReadAuth), not a session.
+	orgID := c.GetString(ctxSecretsOrgKey)
+	values, err := services.GetSecretGroupDecrypted(orgID, group)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "store error"})
 		return
@@ -179,7 +191,7 @@ func deleteSecretGroup(c *gin.Context) {
 }
 
 func rotateSecretsToken(c *gin.Context) {
-	token, err := services.RotateSecretsReadToken()
+	token, err := services.RotateSecretsReadToken(auth.OrgID(c))
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return

server/internal/models/settings.go

@@ -33,6 +33,7 @@ type SecretsSettings struct {
 
 type Settings struct {
 	ID      bson.ObjectID   `bson:"_id,omitempty" json:"-"`
+	OrgID   string          `bson:"org_id" json:"org_id"`
 	Alerts  AlertSettings   `bson:"alerts" json:"alerts"`
 	Email   EmailSettings   `bson:"email" json:"email"`
 	Secrets SecretsSettings `bson:"secrets" json:"secrets"`

server/internal/services/migrate.go

@@ -87,3 +87,38 @@ func RunMigrations() error {
 	_, err := db.Col("migrations").InsertOne(ctx, bson.M{"_id": marker, "applied_at": time.Now()})
 	return err
 }
+
+// MigrateSettingsOrg stamps the legacy global settings singleton with the
+// default org's ID. Without it an upgrade would orphan the existing SMTP
+// config, alert config, retention setting, and ESO read token. Idempotent via
+// a marker in the migrations collection.
+func MigrateSettingsOrg() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	const marker = "0002_settings_org_backfill"
+	if n, _ := db.Col("migrations").CountDocuments(ctx, bson.M{"_id": marker}); n > 0 {
+		return nil
+	}
+
+	n, _ := db.Col("settings").CountDocuments(ctx, bson.M{"org_id": bson.M{"$exists": false}})
+	if n > 0 {
+		var org models.Org
+		err := db.Col("orgs").FindOne(ctx, bson.M{"slug": "default"}).Decode(&org)
+		if err != nil {
+			org = models.Org{OrgID: uuid.NewString(), Name: "Default", Slug: "default", CreatedAt: time.Now()}
+			if _, err := db.Col("orgs").InsertOne(ctx, org); err != nil {
+				return err
+			}
+		}
+		if _, err := db.Col("settings").UpdateMany(ctx,
+			bson.M{"org_id": bson.M{"$exists": false}},
+			bson.M{"$set": bson.M{"org_id": org.OrgID}},
+		); err != nil {
+			return err
+		}
+	}
+
+	_, err := db.Col("migrations").InsertOne(ctx, bson.M{"_id": marker, "applied_at": time.Now()})
+	return err
+}

server/internal/services/secrets.go

@@ -105,9 +105,8 @@ func GetSecretGroup(orgID, group string) ([]models.Secret, error) {
 }
 
 // GetSecretGroupDecrypted returns a flat map of key → plaintext value for a
-// group. Used by the ESO read endpoint, which authenticates via a bearer
-// token rather than a session — org resolution for that path is a known gap,
-// tracked separately; the token is currently global rather than per-org.
+// group. Also used by the ESO read endpoint, which resolves its org from the
+// per-org bearer token rather than from a session.
 func GetSecretGroupDecrypted(orgID, group string) (map[string]string, error) {
 	docs, err := GetSecretGroup(orgID, group)
 	if err != nil {
@@ -124,36 +123,6 @@ func GetSecretGroupDecrypted(orgID, group string) (map[string]string, error) {
 	return result, nil
 }
 
-// GetSecretGroupDecryptedAny is the ESO-bearer-token read path: it has no
-// session/org context (the read token is currently global, not per-org), so
-// it looks up the group across all orgs. This mirrors pre-multi-tenant
-// behavior; scoping the ESO token to an org is tracked as a follow-up.
-func GetSecretGroupDecryptedAny(group string) (map[string]string, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	cursor, err := db.Col("secrets").Find(ctx, bson.M{"group": group},
-		options.Find().SetSort(bson.D{{Key: "key", Value: 1}}))
-	if err != nil {
-		return nil, err
-	}
-	defer cursor.Close(ctx)
-
-	var docs []models.Secret
-	if err := cursor.All(ctx, &docs); err != nil {
-		return nil, err
-	}
-	result := make(map[string]string, len(docs))
-	for _, doc := range docs {
-		val, err := decryptString(doc.EncryptedValue)
-		if err != nil {
-			return nil, fmt.Errorf("decrypt %s/%s: %w", group, doc.Key, err)
-		}
-		result[doc.Key] = val
-	}
-	return result, nil
-}
-
 // RevealSecret returns the decrypted value of a single key.
 func RevealSecret(orgID, group, key string) (string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

server/internal/services/servers.go

@@ -277,53 +277,60 @@ func StoreAvailableUpdates(serverID string, pkgs []models.PackageUpdate) error {
 }
 
 func MarkOfflineServers() error {
-	settings, _ := GetSettings()
-	thresholdMinutes := 5
-	if settings != nil && settings.Alerts.OfflineThresholdMinutes > 0 {
-		thresholdMinutes = settings.Alerts.OfflineThresholdMinutes
-	}
-	threshold := time.Duration(thresholdMinutes) * time.Minute
-
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	cutoff := time.Now().Add(-threshold)
-
-	// Find servers about to transition to offline so we can alert on them.
-	cursor, err := db.Col("servers").Find(ctx, bson.M{
-		"status":    "active",
-		"last_seen": bson.M{"$lt": cutoff},
-	})
+	// No session here, so the sweep runs per-org and each org's threshold and
+	// alert config come from that org's own settings doc.
+	orgIDs, err := ListOrgIDs()
 	if err != nil {
 		return err
 	}
-	defer cursor.Close(ctx)
 
-	var goingOffline []models.Server
-	if err := cursor.All(ctx, &goingOffline); err != nil {
-		return err
-	}
-
-	if len(goingOffline) == 0 {
-		return nil
-	}
-
-	for _, s := range goingOffline {
-		LogEvent(s.OrgID, "server.offline", "system", s.ServerID, "", fmt.Sprintf("%s (%s) went offline", s.Hostname, s.IPAddress))
-		if settings != nil && settings.Alerts.Enabled && settings.Alerts.WebhookURL != "" {
-			go SendOfflineWebhook(settings.Alerts.WebhookURL, s.Hostname, s.ServerID, s.IPAddress)
+	for _, orgID := range orgIDs {
+		settings, _ := GetSettings(orgID)
+		thresholdMinutes := 5
+		if settings != nil && settings.Alerts.OfflineThresholdMinutes > 0 {
+			thresholdMinutes = settings.Alerts.OfflineThresholdMinutes
 		}
-		if settings != nil && settings.Email.Enabled {
-			go SendOfflineEmail(settings.Email, s.Hostname, s.ServerID, s.IPAddress)
-		}
-	}
+		cutoff := time.Now().Add(-time.Duration(thresholdMinutes) * time.Minute)
 
-	_, err = db.Col("servers").UpdateMany(ctx,
-		bson.M{
+		filter := bson.M{
+			"org_id":    orgID,
 			"status":    "active",
 			"last_seen": bson.M{"$lt": cutoff},
-		},
-		bson.M{"$set": bson.M{"status": "offline"}},
-	)
-	return err
+		}
+
+		// Find servers about to transition to offline so we can alert on them.
+		cursor, err := db.Col("servers").Find(ctx, filter)
+		if err != nil {
+			return err
+		}
+		var goingOffline []models.Server
+		err = cursor.All(ctx, &goingOffline)
+		cursor.Close(ctx)
+		if err != nil {
+			return err
+		}
+		if len(goingOffline) == 0 {
+			continue
+		}
+
+		for _, s := range goingOffline {
+			LogEvent(s.OrgID, "server.offline", "system", s.ServerID, "", fmt.Sprintf("%s (%s) went offline", s.Hostname, s.IPAddress))
+			if settings != nil && settings.Alerts.Enabled && settings.Alerts.WebhookURL != "" {
+				go SendOfflineWebhook(settings.Alerts.WebhookURL, s.Hostname, s.ServerID, s.IPAddress)
+			}
+			if settings != nil && settings.Email.Enabled {
+				go SendOfflineEmail(settings.Email, s.Hostname, s.ServerID, s.IPAddress)
+			}
+		}
+
+		if _, err := db.Col("servers").UpdateMany(ctx, filter,
+			bson.M{"$set": bson.M{"status": "offline"}},
+		); err != nil {
+			return err
+		}
+	}
+	return nil
 }

server/internal/services/settings.go

@@ -34,14 +34,44 @@ var defaultSettings = models.Settings{
 	},
 }
 
-func GetSettings() (*models.Settings, error) {
+// EnsureSettingsIndexes creates the per-org uniqueness constraints on settings.
+// Pre-multi-tenant deployments had a single global settings doc and no indexes;
+// drop any legacy index if a live DB still carries one.
+func EnsureSettingsIndexes() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if err := db.Col("settings").Indexes().DropOne(ctx, "secrets.read_token_hash_1"); err != nil && !isIndexNotFound(err) {
+		return err
+	}
+
+	if _, err := db.Col("settings").Indexes().CreateOne(ctx, mongo.IndexModel{
+		Keys:    bson.D{{Key: "org_id", Value: 1}},
+		Options: options.Index().SetUnique(true),
+	}); err != nil {
+		return err
+	}
+
+	// Partial so the many settings docs with no ESO token set don't collide on
+	// a missing field.
+	_, err := db.Col("settings").Indexes().CreateOne(ctx, mongo.IndexModel{
+		Keys: bson.D{{Key: "secrets.read_token_hash", Value: 1}},
+		Options: options.Index().SetUnique(true).SetPartialFilterExpression(
+			bson.M{"secrets.read_token_hash": bson.M{"$type": "string"}},
+		),
+	})
+	return err
+}
+
+func GetSettings(orgID string) (*models.Settings, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
 	var s models.Settings
-	err := db.Col("settings").FindOne(ctx, bson.M{}).Decode(&s)
+	err := db.Col("settings").FindOne(ctx, bson.M{"org_id": orgID}).Decode(&s)
 	if err == mongo.ErrNoDocuments {
 		cp := defaultSettings
+		cp.OrgID = orgID
 		return &cp, nil
 	}
 	if err != nil {
@@ -58,7 +88,7 @@ func hashToken(token string) string {
 
 // RotateSecretsReadToken generates a new ESO read token, stores its SHA-256
 // hash, and returns the plaintext token exactly once.
-func RotateSecretsReadToken() (string, error) {
+func RotateSecretsReadToken(orgID string) (string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
@@ -69,11 +99,14 @@ func RotateSecretsReadToken() (string, error) {
 	token := hex.EncodeToString(raw)
 
 	_, err := db.Col("settings").UpdateOne(ctx,
-		bson.M{},
-		bson.M{"$set": bson.M{
-			"secrets.read_token_hash": hashToken(token),
-			"secrets.rotated_at":      time.Now(),
-		}},
+		bson.M{"org_id": orgID},
+		bson.M{
+			"$set": bson.M{
+				"secrets.read_token_hash": hashToken(token),
+				"secrets.rotated_at":      time.Now(),
+			},
+			"$setOnInsert": bson.M{"org_id": orgID},
+		},
 		options.UpdateOne().SetUpsert(true),
 	)
 	if err != nil {
@@ -82,25 +115,33 @@ func RotateSecretsReadToken() (string, error) {
 	return token, nil
 }
 
-// VerifySecretsReadToken reports whether the supplied token matches the stored
-// hash, using a constant-time comparison.
-func VerifySecretsReadToken(token string) bool {
+// ResolveSecretsReadToken looks the presented token's hash up directly and
+// returns the owning org. This is the ESO machine-to-machine path: the org is
+// carried by the token itself, since there is no session to scope it.
+func ResolveSecretsReadToken(token string) (string, bool) {
 	if token == "" {
-		return false
+		return "", false
 	}
-	s, err := GetSettings()
-	if err != nil || s.Secrets.ReadTokenHash == "" {
-		return false
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	var s models.Settings
+	err := db.Col("settings").FindOne(ctx, bson.M{"secrets.read_token_hash": hashToken(token)}).Decode(&s)
+	if err != nil || s.Secrets.ReadTokenHash == "" || s.OrgID == "" {
+		return "", false
 	}
 	expected, err := hex.DecodeString(s.Secrets.ReadTokenHash)
 	if err != nil {
-		return false
+		return "", false
 	}
 	got := sha256.Sum256([]byte(token))
-	return subtle.ConstantTimeCompare(expected, got[:]) == 1
+	if subtle.ConstantTimeCompare(expected, got[:]) != 1 {
+		return "", false
+	}
+	return s.OrgID, true
 }
 
-func SaveSettings(alerts models.AlertSettings, email models.EmailSettings, retentionDays *int) error {
+func SaveSettings(orgID string, alerts models.AlertSettings, email models.EmailSettings, retentionDays *int) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
@@ -116,8 +157,8 @@ func SaveSettings(alerts models.AlertSettings, email models.EmailSettings, reten
 		set["workflow_log_retention_days"] = *retentionDays
 	}
 	_, err := db.Col("settings").UpdateOne(ctx,
-		bson.M{},
-		bson.M{"$set": set},
+		bson.M{"org_id": orgID},
+		bson.M{"$set": set, "$setOnInsert": bson.M{"org_id": orgID}},
 		options.UpdateOne().SetUpsert(true),
 	)
 	return err
@@ -125,8 +166,8 @@ func SaveSettings(alerts models.AlertSettings, email models.EmailSettings, reten
 
 // GetWorkflowLogRetentionDays returns the log retention in days: 30 when unset,
 // 0 for keep-forever, or the configured value.
-func GetWorkflowLogRetentionDays() (int, error) {
-	s, err := GetSettings()
+func GetWorkflowLogRetentionDays(orgID string) (int, error) {
+	s, err := GetSettings(orgID)
 	if err != nil {
 		return 30, err
 	}

server/internal/services/steplogs.go

@@ -164,54 +164,72 @@ func StartLogSweeper() {
 	}()
 }
 
+// sweepLogs walks the run-log dirs on disk. Log dirs are keyed by run ID, not
+// by org, and this runs with no session — so retention is resolved per run from
+// the owning org of that run's doc, with the per-org values cached for the
+// sweep. Runs whose doc is gone fall back to the default retention.
 func sweepLogs() {
-	days := retentionDays()
-	if days <= 0 {
-		return
-	}
-	cutoff := time.Now().AddDate(0, 0, -days)
 	base := WorkflowLogDir()
 	entries, err := os.ReadDir(base)
 	if err != nil {
 		return
 	}
+	cache := map[string]int{}
+	now := time.Now()
+
 	for _, e := range entries {
 		if !e.IsDir() {
 			continue
 		}
 		runID := e.Name()
 		dir := filepath.Join(base, runID)
-		if runExpired(runID, dir, cutoff) {
+
+		orgID, finishedAt, found := runRetentionInfo(runID)
+		if found && finishedAt == nil {
+			continue // still running / never finished — keep
+		}
+
+		days, ok := cache[orgID]
+		if !ok {
+			days = defaultRetentionDays
+			if orgID != "" {
+				if v, err := GetWorkflowLogRetentionDays(orgID); err == nil {
+					days = v
+				}
+			}
+			cache[orgID] = days
+		}
+		if days <= 0 {
+			continue // keep forever
+		}
+		cutoff := now.AddDate(0, 0, -days)
+
+		if found {
+			if finishedAt.Before(cutoff) {
+				_ = os.RemoveAll(dir)
+			}
+			continue
+		}
+		// run doc gone: use dir mtime
+		if fi, e := os.Stat(dir); e == nil && fi.ModTime().Before(cutoff) {
 			_ = os.RemoveAll(dir)
 		}
 	}
 }
 
-// runExpired is true when the run finished before cutoff (falling back to dir
-// mtime when the run doc is gone).
-func runExpired(runID, dir string, cutoff time.Time) bool {
+const defaultRetentionDays = 30
+
+// runRetentionInfo returns the owning org and finish time of a run, and whether
+// the run doc still exists.
+func runRetentionInfo(runID string) (string, *time.Time, bool) {
 	ctx, cancel := wfCtx()
 	defer cancel()
 	var run struct {
+		OrgID      string     `bson:"org_id"`
 		FinishedAt *time.Time `bson:"finished_at"`
 	}
-	err := db.Col("workflow_runs").FindOne(ctx, bson.M{"run_id": runID}).Decode(&run)
-	if err == nil {
-		if run.FinishedAt == nil {
-			return false // still running / never finished — keep
-		}
-		return run.FinishedAt.Before(cutoff)
+	if err := db.Col("workflow_runs").FindOne(ctx, bson.M{"run_id": runID}).Decode(&run); err != nil {
+		return "", nil, false
 	}
-	// run doc gone: use dir mtime
-	if fi, e := os.Stat(dir); e == nil {
-		return fi.ModTime().Before(cutoff)
-	}
-	return false
-}
-
-func retentionDays() int {
-	if v, err := GetWorkflowLogRetentionDays(); err == nil {
-		return v
-	}
-	return 30
+	return run.OrgID, run.FinishedAt, true
 }