fix: Fixes to running on kubernetes

server/cmd/main.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/api"
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/auth"
+	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/bus"
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db"
 	grpcserver "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/grpc"
 	"gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/monitorsched"
@@ -18,7 +20,21 @@ import (
 func main() {
 	mongoURI := getEnv("MONGO_URI", "mongodb://localhost:27017")
 
-	if os.Getenv("GRPC_HOST") == "" {
+	// Two flags, so the schema work can be lifted out of the serving pods.
+	//
+	// Under Docker Compose neither is set and nothing changes: one process
+	// migrates and then serves. Under Kubernetes with more than one replica
+	// that is unsafe — every pod would run MigrateOrgToInstance at once, and
+	// renaming collections while a sibling reads them is not a race anyone
+	// wins. The chart therefore runs a pre-upgrade Job with MIGRATE_ONLY and
+	// starts the Deployment with SKIP_MIGRATIONS.
+	migrateOnly := boolEnv("VANTAGE_MIGRATE_ONLY")
+	skipMigrations := boolEnv("VANTAGE_SKIP_MIGRATIONS")
+
+	// Not required in migrate-only mode: that process never serves gRPC, and
+	// demanding it would put an agent-facing address in a Job that has no
+	// business knowing one.
+	if !migrateOnly && os.Getenv("GRPC_HOST") == "" {
 		log.Fatal("GRPC_HOST is required (host:port agents dial for gRPC)")
 	}
 
@@ -28,6 +44,24 @@ func main() {
 	}
 	log.Println("connected to MongoDB")
 
+	if migrateOnly || !skipMigrations {
+		runSchemaSetup()
+	} else {
+		log.Println("VANTAGE_SKIP_MIGRATIONS set: assuming migrations ran elsewhere")
+	}
+
+	if migrateOnly {
+		log.Println("VANTAGE_MIGRATE_ONLY set: schema setup complete, exiting")
+		return
+	}
+
+	serve()
+}
+
+// runSchemaSetup performs every write that must happen exactly once before the
+// application serves: migrations, index builders and default step seeding. It
+// is fatal on anything that would leave the schema half-moved.
+func runSchemaSetup() {
 	// Migrations 0001 to 0003 still speak the pre-rename shape (orgs, org_id),
 	// so they must run before 0004 renames everything underneath them.
 	if err := services.RunMigrations(); err != nil {
@@ -87,9 +121,9 @@ func main() {
 		}
 	}
 
-	services.StartLogSweeper()
-	services.StartAuditSweeper()
+}
 
+func serve() {
 	redisAddr := getEnv("REDIS_ADDR", "localhost:6379")
 	redisUser := os.Getenv("REDIS_USERNAME")
 	redisPass := os.Getenv("REDIS_PASSWORD")
@@ -98,15 +132,16 @@ func main() {
 	}
 	log.Println("connected to Redis")
 
-	go func() {
-		ticker := time.NewTicker(2 * time.Minute)
-		defer ticker.Stop()
-		for range ticker.C {
-			if err := services.MarkOfflineServers(); err != nil {
-				log.Printf("mark offline error: %v", err)
-			}
-		}
-	}()
+	// The bus carries agent commands and step results between replicas. It is
+	// not optional even on a single-replica deployment: dispatch takes the same
+	// path either way, so the code exercised in production is the code
+	// exercised everywhere.
+	if err := bus.Init(redisAddr, redisUser, redisPass); err != nil {
+		log.Fatalf("failed to connect the message bus: %v", err)
+	}
+	log.Printf("message bus ready as node %s", bus.NodeID())
+
+	ctx := context.Background()
 
 	go func() {
 		if err := grpcserver.StartGRPC(9090); err != nil {
@@ -114,9 +149,33 @@ func main() {
 		}
 	}()
 
-	monitorsched.Start(context.Background())
+	// Everything below runs on exactly one replica at a time.
+	//
+	// These are cluster-singleton jobs, not per-pod work: N replicas would mean
+	// every monitor check firing N times, every incident notification delivered
+	// to the customer N times, every retention sweep deleting concurrently, and
+	// N reapers racing to purge the same instance. They share one lock rather
+	// than holding four, because they are one role — housekeeping — and
+	// splitting them would only spread that role across pods for no benefit.
+	bus.RunAsLeader(ctx, "housekeeping", func(jobCtx context.Context) {
+		services.StartLogSweeper(jobCtx)
+		services.StartAuditSweeper(jobCtx)
+		services.StartReaper(jobCtx)
+		monitorsched.Start(jobCtx)
 
-	services.StartReaper(context.Background())
+		ticker := time.NewTicker(2 * time.Minute)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-jobCtx.Done():
+				return
+			case <-ticker.C:
+				if err := services.MarkOfflineServers(); err != nil {
+					log.Printf("mark offline error: %v", err)
+				}
+			}
+		}
+	})
 
 	r := gin.New()
 	r.Use(gin.Recovery())
@@ -149,3 +208,14 @@ func getEnv(key, fallback string) string {
 	}
 	return fallback
 }
+
+// boolEnv reads a flag env var. Anything other than a recognised truthy value
+// is false, so a typo leaves the safe default (migrate here, serve here) rather
+// than silently skipping schema setup.
+func boolEnv(key string) bool {
+	switch strings.ToLower(strings.TrimSpace(os.Getenv(key))) {
+	case "1", "true", "yes", "on":
+		return true
+	}
+	return false
+}