Compare commits

...
4 Commits
12 changed files with 497 additions and 43 deletions
+2
View File
@@ -31,6 +31,8 @@ func main() {
log.Printf("warning: failed to ensure workflow indexes: %v", err)
}
services.StartLogSweeper()
redisAddr := getEnv("REDIS_ADDR", "localhost:6379")
if err := auth.InitRedis(redisAddr); err != nil {
log.Fatalf("failed to connect to Redis: %v", err)
+4 -3
View File
@@ -450,14 +450,15 @@ func getSettings(c *gin.Context) {
func saveSettings(c *gin.Context) {
var body struct {
Alerts models.AlertSettings `json:"alerts"`
Email models.EmailSettings `json:"email"`
Alerts models.AlertSettings `json:"alerts"`
Email models.EmailSettings `json:"email"`
WorkflowLogRetentionDays *int `json:"workflow_log_retention_days"`
}
if err := c.ShouldBindJSON(&body); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if err := services.SaveSettings(body.Alerts, body.Email); err != nil {
if err := services.SaveSettings(body.Alerts, body.Email, body.WorkflowLogRetentionDays); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
+112
View File
@@ -3,7 +3,11 @@ package api
import (
"fmt"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/mrhid6/vantage/server/internal/models"
@@ -26,6 +30,114 @@ func registerWorkflowRoutes(g *gin.RouterGroup) {
g.GET("/runs/:runId", getRun)
g.POST("/runs/:runId/cancel", cancelRun)
g.GET("/runs/:runId/servers/:serverId/logs", getServerRunLog)
g.GET("/runs/:runId/servers/:serverId/logs/stream", streamServerRunLog)
}
var uuidLike = regexp.MustCompile(`^[a-zA-Z0-9-]{1,64}$`)
func getServerRunLog(c *gin.Context) {
runID, serverID := c.Param("runId"), c.Param("serverId")
if !uuidLike.MatchString(runID) || !uuidLike.MatchString(serverID) {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
path := services.ServerRunLogPath(runID, serverID)
b, err := os.ReadFile(path)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "no logs"})
return
}
c.Data(http.StatusOK, "text/plain; charset=utf-8", b)
}
func streamServerRunLog(c *gin.Context) {
runID, serverID := c.Param("runId"), c.Param("serverId")
if !uuidLike.MatchString(runID) || !uuidLike.MatchString(serverID) {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"})
return
}
path := services.ServerRunLogPath(runID, serverID)
c.Writer.Header().Set("Content-Type", "text/event-stream")
c.Writer.Header().Set("Cache-Control", "no-cache")
c.Writer.Header().Set("Connection", "keep-alive")
c.Writer.Header().Set("X-Accel-Buffering", "no")
flusher, ok := c.Writer.(http.Flusher)
if !ok {
c.JSON(http.StatusInternalServerError, gin.H{"error": "stream unsupported"})
return
}
var offset int64
sendNew := func() {
f, err := os.Open(path)
if err != nil {
return // file may not exist yet; keep waiting
}
defer f.Close()
if _, err := f.Seek(offset, 0); err != nil {
return
}
buf := make([]byte, 32*1024)
for {
n, _ := f.Read(buf)
if n <= 0 {
break
}
offset += int64(n)
// SSE data frame; split on newlines to keep frames well-formed.
for _, line := range splitSSE(buf[:n]) {
_, _ = c.Writer.WriteString("data: " + line + "\n")
}
_, _ = c.Writer.WriteString("\n")
flusher.Flush()
}
}
ctx := c.Request.Context()
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
sendNew()
if serverRunTerminal(runID, serverID) {
sendNew() // final drain
_, _ = c.Writer.WriteString("event: done\ndata: end\n\n")
flusher.Flush()
return
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// serverRunTerminal reports whether the given server-run has reached a terminal status.
func serverRunTerminal(runID, serverID string) bool {
r, err := services.GetRun(runID)
if err != nil {
return true
}
for _, sr := range r.ServerRuns {
if sr.ServerID == serverID {
switch sr.Status {
case "success", "failed", "skipped", "cancelled":
return true
}
return false
}
}
return true
}
// splitSSE turns a raw byte slice into SSE-safe payload lines (newlines become
// separate data lines; carriage returns stripped).
func splitSSE(b []byte) []string {
s := strings.ReplaceAll(string(b), "\r", "")
return strings.Split(s, "\n")
}
func listSteps(c *gin.Context) {
+7
View File
@@ -132,6 +132,13 @@ func (s *vantageServer) CommandStream(stream pb.Vantage_CommandStreamServer) err
if m.StepResult != nil {
services.StepResults.Deliver(m.StepResult)
}
if m.StepOutput != nil {
if m.StepOutput.Eof {
services.StepLogs.Close(m.StepOutput.CommandId)
} else {
services.StepLogs.Append(m.StepOutput.CommandId, m.StepOutput.Data)
}
}
}
}()
+2
View File
@@ -36,4 +36,6 @@ type Settings struct {
Alerts AlertSettings `bson:"alerts" json:"alerts"`
Email EmailSettings `bson:"email" json:"email"`
Secrets SecretsSettings `bson:"secrets" json:"secrets"`
// WorkflowLogRetentionDays: nil = default 30, 0 = keep forever, N = N days.
WorkflowLogRetentionDays *int `bson:"workflow_log_retention_days,omitempty" json:"workflow_log_retention_days,omitempty"`
}
+1 -2
View File
@@ -68,8 +68,7 @@ type StepRun struct {
Status string `bson:"status" json:"status"` // queued|running|success|failed|skipped
Attempts int `bson:"attempts" json:"attempts"`
ExitCode int `bson:"exit_code" json:"exit_code"`
Stdout string `bson:"stdout" json:"stdout"`
Stderr string `bson:"stderr" json:"stderr"`
LogOffset int64 `bson:"log_offset" json:"log_offset"`
OutputEnv map[string]string `bson:"output_env" json:"output_env"`
StartedAt *time.Time `bson:"started_at,omitempty" json:"started_at,omitempty"`
FinishedAt *time.Time `bson:"finished_at,omitempty" json:"finished_at,omitempty"`
+19 -2
View File
@@ -100,7 +100,7 @@ func VerifySecretsReadToken(token string) bool {
return subtle.ConstantTimeCompare(expected, got[:]) == 1
}
func SaveSettings(alerts models.AlertSettings, email models.EmailSettings) error {
func SaveSettings(alerts models.AlertSettings, email models.EmailSettings, retentionDays *int) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
@@ -111,14 +111,31 @@ func SaveSettings(alerts models.AlertSettings, email models.EmailSettings) error
email.SMTPPort = 587
}
set := bson.M{"alerts": alerts, "email": email}
if retentionDays != nil {
set["workflow_log_retention_days"] = *retentionDays
}
_, err := db.Col("settings").UpdateOne(ctx,
bson.M{},
bson.M{"$set": bson.M{"alerts": alerts, "email": email}},
bson.M{"$set": set},
options.UpdateOne().SetUpsert(true),
)
return err
}
// GetWorkflowLogRetentionDays returns the log retention in days: 30 when unset,
// 0 for keep-forever, or the configured value.
func GetWorkflowLogRetentionDays() (int, error) {
s, err := GetSettings()
if err != nil {
return 30, err
}
if s.WorkflowLogRetentionDays == nil {
return 30, nil
}
return *s.WorkflowLogRetentionDays, nil
}
func SendOfflineWebhook(webhookURL, hostname, serverID, ipAddress string) {
payload := map[string]any{
"event": "server.offline",
+210
View File
@@ -0,0 +1,210 @@
package services
import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/mrhid6/vantage/server/internal/db"
"go.mongodb.org/mongo-driver/v2/bson"
)
// WorkflowLogDir returns the base directory for workflow step logs, creating it.
func WorkflowLogDir() string {
dir := os.Getenv("VANTAGE_WORKFLOW_LOG_DIR")
if dir == "" {
dir = filepath.Join("data", "workflow-logs")
}
_ = os.MkdirAll(dir, 0700)
return dir
}
// ServerRunLogPath is the per-server-run log file path.
func ServerRunLogPath(runID, serverID string) string {
return filepath.Join(WorkflowLogDir(), runID, serverID+".log")
}
// AppendMarker appends a line to the server-run log and returns the byte offset
// at which the write began (used as a step's log_offset).
func AppendMarker(runID, serverID, line string) (int64, error) {
path := ServerRunLogPath(runID, serverID)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return 0, err
}
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return 0, err
}
defer f.Close()
off, _ := f.Seek(0, 2) // current end = offset before write
if _, err := f.WriteString(line); err != nil {
return off, err
}
return off, nil
}
// ---- streamed chunk writer, boundary-safe secret masking ----
type stepLogWriter struct {
mu sync.Mutex
f *os.File
carry []byte
secrets []string
maxSecret int
}
type stepLogRegistry struct {
mu sync.Mutex
writers map[string]*stepLogWriter
}
var StepLogs = &stepLogRegistry{writers: make(map[string]*stepLogWriter)}
// Open opens (append) the server-run file for a step's streamed chunks.
func (r *stepLogRegistry) Open(commandID, path string, secrets []string) error {
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
max := 0
for _, s := range secrets {
if len(s) > max {
max = len(s)
}
}
w := &stepLogWriter{f: f, secrets: secrets, maxSecret: max}
r.mu.Lock()
r.writers[commandID] = w
r.mu.Unlock()
return nil
}
func (r *stepLogRegistry) get(commandID string) *stepLogWriter {
r.mu.Lock()
defer r.mu.Unlock()
return r.writers[commandID]
}
// Append masks and writes a chunk, holding back the last maxSecret-1 bytes so a
// secret split across a chunk boundary is still masked on the next append/close.
func (r *stepLogRegistry) Append(commandID string, data []byte) {
w := r.get(commandID)
if w == nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
if len(w.secrets) == 0 || w.maxSecret <= 1 {
_, _ = w.f.Write(data)
return
}
buf := append(w.carry, data...)
hold := w.maxSecret - 1
if len(buf) <= hold {
w.carry = buf
return
}
flush := buf[:len(buf)-hold]
w.carry = append([]byte{}, buf[len(buf)-hold:]...)
_, _ = w.f.Write(maskBytes(flush, w.secrets))
}
// Close flushes the carry (masked) and closes the file.
func (r *stepLogRegistry) Close(commandID string) {
r.mu.Lock()
w := r.writers[commandID]
delete(r.writers, commandID)
r.mu.Unlock()
if w == nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
if len(w.carry) > 0 {
_, _ = w.f.Write(maskBytes(w.carry, w.secrets))
w.carry = nil
}
_ = w.f.Close()
}
func maskBytes(b []byte, secrets []string) []byte {
s := string(b)
for _, v := range secrets {
if v == "" {
continue
}
s = strings.ReplaceAll(s, v, "***")
}
return []byte(s)
}
// ---- retention sweeper ----
// StartLogSweeper sweeps expired run-log dirs hourly (and once now).
func StartLogSweeper() {
go func() {
sweepLogs()
t := time.NewTicker(time.Hour)
defer t.Stop()
for range t.C {
sweepLogs()
}
}()
}
func sweepLogs() {
days := retentionDays()
if days <= 0 {
return
}
cutoff := time.Now().AddDate(0, 0, -days)
base := WorkflowLogDir()
entries, err := os.ReadDir(base)
if err != nil {
return
}
for _, e := range entries {
if !e.IsDir() {
continue
}
runID := e.Name()
dir := filepath.Join(base, runID)
if runExpired(runID, dir, cutoff) {
_ = os.RemoveAll(dir)
}
}
}
// runExpired is true when the run finished before cutoff (falling back to dir
// mtime when the run doc is gone).
func runExpired(runID, dir string, cutoff time.Time) bool {
ctx, cancel := wfCtx()
defer cancel()
var run struct {
FinishedAt *time.Time `bson:"finished_at"`
}
err := db.Col("workflow_runs").FindOne(ctx, bson.M{"run_id": runID}).Decode(&run)
if err == nil {
if run.FinishedAt == nil {
return false // still running / never finished — keep
}
return run.FinishedAt.Before(cutoff)
}
// run doc gone: use dir mtime
if fi, e := os.Stat(dir); e == nil {
return fi.ModTime().Before(cutoff)
}
return false
}
func retentionDays() int {
if v, err := GetWorkflowLogRetentionDays(); err == nil {
return v
}
return 30
}
+28 -13
View File
@@ -195,40 +195,48 @@ func runServer(runID string, srvIdx int, steps []models.ResolvedStep, serverID s
cmdEnv[k] = v
}
// Write the step marker to the server-run log and remember the offset so
// the UI can slice this step's output later.
marker := fmt.Sprintf("\n===== step %d: %s =====\n", step.Order, step.Name)
offset, _ := AppendMarker(runID, serverID, marker)
logPath := ServerRunLogPath(runID, serverID)
secretsSlice := secretValues(secretVals)
commandID := uuid.New().String()
for attempts < maxAttempts {
attempts++
res = dispatchAndWait(serverID, &pb.RunStepCmd{
// Open a fresh writer per attempt; the agent's eof closes it, and the
// defensive Close below covers a missing result.
_ = StepLogs.Open(commandID, logPath, secretsSlice)
res = dispatchAndWait(serverID, commandID, &pb.RunStepCmd{
Interpreter: step.Interpreter,
Script: step.Script,
Env: cmdEnv,
TimeoutSeconds: 0,
})
StepLogs.Close(commandID) // idempotent; no-op if eof already closed it
if res != nil && res.ExitCode == 0 {
break
}
}
// Mask secret values before persisting.
stdout, stderr := "", ""
exit := 1
outEnv := map[string]string{} // masked copy, safe to persist
outEnv := map[string]string{} // masked copy, safe to persist
if res != nil {
stdout = maskSecrets(res.Stdout, allSecrets)
stderr = maskSecrets(res.Stderr, allSecrets)
exit = res.ExitCode
for k, v := range res.OutputEnv {
runEnv[k] = v // real, unmasked value threads forward to later steps
outEnv[k] = maskSecrets(v, allSecrets)
}
} else {
stderr = "[vantage] agent did not return a result"
_, _ = AppendMarker(runID, serverID, "[vantage] agent did not return a result\n")
}
status := "success"
if exit != 0 {
status = "failed"
}
finishStep(runID, serverID, i, status, attempts, exit, stdout, stderr, outEnv)
finishStep(runID, serverID, i, status, attempts, exit, offset, outEnv)
if exit != 0 {
switch step.OnFailure {
@@ -264,8 +272,7 @@ func runServer(runID string, srvIdx int, steps []models.ResolvedStep, serverID s
// dispatchAndWait registers a waiter, dispatches the step, and blocks for the
// result or a timeout.
func dispatchAndWait(serverID string, cmd *pb.RunStepCmd) *pb.StepResult {
commandID := uuid.New().String()
func dispatchAndWait(serverID, commandID string, cmd *pb.RunStepCmd) *pb.StepResult {
ch := StepResults.Await(commandID)
if err := DispatchRunStep(serverID, commandID, cmd); err != nil {
StepResults.Cancel(commandID)
@@ -336,19 +343,27 @@ func startStep(runID, serverID string, order int, status string) {
})
}
func finishStep(runID, serverID string, order int, status string, attempts, exit int, stdout, stderr string, outEnv map[string]string) {
func finishStep(runID, serverID string, order int, status string, attempts, exit int, logOffset int64, outEnv map[string]string) {
now := time.Now()
updateStep(runID, serverID, order, bson.M{
"server_runs.$[s].steps.$[t].status": status,
"server_runs.$[s].steps.$[t].attempts": attempts,
"server_runs.$[s].steps.$[t].exit_code": exit,
"server_runs.$[s].steps.$[t].stdout": stdout,
"server_runs.$[s].steps.$[t].stderr": stderr,
"server_runs.$[s].steps.$[t].log_offset": logOffset,
"server_runs.$[s].steps.$[t].output_env": outEnv,
"server_runs.$[s].steps.$[t].finished_at": now,
})
}
// secretValues returns just the values of a secret map, for masking log output.
func secretValues(m map[string]string) []string {
out := make([]string, 0, len(m))
for _, v := range m {
out = append(out, v)
}
return out
}
func markRemainingSkipped(runID, serverID string, fromOrder int) {
ctx, cancel := wfCtx()
defer cancel()
+33 -2
View File
@@ -168,6 +168,9 @@ export default function SettingsPage() {
const [toAddrs, setToAddrs] = useState(""); // comma-separated in UI
const [useTLS, setUseTLS] = useState(false);
// Workflow log retention (days). 0 = keep forever.
const [logRetentionDays, setLogRetentionDays] = useState(30);
const [saved, setSaved] = useState(false);
useEffect(() => {
@@ -183,11 +186,15 @@ export default function SettingsPage() {
setFromAddr(settings.email?.from_addr ?? "");
setToAddrs((settings.email?.to_addrs ?? []).join(", "));
setUseTLS(settings.email?.use_tls ?? false);
setLogRetentionDays(settings.workflow_log_retention_days ?? 30);
}, [settings]);
const { mutate: save, isPending } = useMutation({
mutationFn: (payload: { alerts: AlertSettings; email: EmailSettings }) =>
api.saveSettings(payload),
mutationFn: (payload: {
alerts: AlertSettings;
email: EmailSettings;
workflow_log_retention_days?: number | null;
}) => api.saveSettings(payload),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ["settings"] });
setSaved(true);
@@ -218,6 +225,7 @@ export default function SettingsPage() {
to_addrs: toList,
use_tls: useTLS,
},
workflow_log_retention_days: logRetentionDays,
});
}
@@ -374,6 +382,29 @@ export default function SettingsPage() {
</div>
</Card>
{/* Workflow logs */}
<Card>
<CardHeader>
<CardTitle>Workflow Logs</CardTitle>
</CardHeader>
<p className="mb-5 text-sm text-text-secondary">
How long to keep workflow run logs on the server before they are
automatically deleted.
</p>
<Field
label="Log retention (days)"
hint="0 = keep forever. Applies to per-run step output logs."
>
<input
type="number"
min={0}
value={logRetentionDays}
onChange={(e) => setLogRetentionDays(Number(e.target.value))}
className="w-32 rounded-lg border border-border bg-surface-2 px-3 py-2 text-sm text-text-primary focus:border-accent/50 focus:outline-none focus:ring-1 focus:ring-accent/30"
/>
</Field>
</Card>
<div className="flex items-center gap-3">
<Button type="submit" variant="primary" loading={isPending}>
{saved ? "Saved!" : "Save Settings"}
+60 -18
View File
@@ -1,6 +1,7 @@
"use client";
import { useParams } from "next/navigation";
import { useEffect, useRef, useState } from "react";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { api, ServerRun, StepRun } from "@/lib/api";
import { Button, Badge, Card } from "@/components/ui";
@@ -20,6 +21,50 @@ function StatusBadge({ status }: { status: string }) {
return <Badge variant={statusVariant[status] ?? "neutral"}>{status}</Badge>;
}
function ServerLog({
runId,
serverId,
status,
}: {
runId: string;
serverId: string;
status: string;
}) {
const [text, setText] = useState("");
const preRef = useRef<HTMLPreElement>(null);
const running = status === "running";
useEffect(() => {
if (running) {
const es = new EventSource(api.serverRunLogStreamUrl(runId, serverId), {
withCredentials: true,
});
es.onmessage = (e) => setText((t) => t + e.data + "\n");
es.addEventListener("done", () => es.close());
es.onerror = () => es.close();
return () => es.close();
}
// terminal: fetch the whole file once
api
.getServerRunLog(runId, serverId)
.then(setText)
.catch(() => setText(""));
}, [running, runId, serverId]);
useEffect(() => {
preRef.current?.scrollTo(0, preRef.current.scrollHeight);
}, [text]);
return (
<pre
ref={preRef}
className="mt-3 max-h-80 overflow-auto whitespace-pre-wrap rounded bg-black/40 p-2 font-mono text-xs text-text-secondary"
>
{text || (running ? "Waiting for output…" : "No output.")}
</pre>
);
}
export default function RunDetail() {
const { runId } = useParams<{ runId: string }>();
const queryClient = useQueryClient();
@@ -65,32 +110,29 @@ export default function RunDetail() {
</div>
<div className="space-y-2">
{sr.steps.map((st: StepRun) => (
<details
<div
key={st.order}
className="rounded-lg border border-border bg-surface-2 p-2"
className="flex items-center justify-between gap-2 rounded-lg border border-border bg-surface-2 p-2"
>
<summary className="flex cursor-pointer items-center justify-between gap-2">
<span className="text-sm text-text-primary">{st.name}</span>
<span className="flex items-center gap-2">
<span className="text-xs text-text-secondary">
attempts: {st.attempts}
{st.status === "failed" ? ` · exit ${st.exit_code}` : ""}
</span>
<StatusBadge status={st.status} />
<span className="text-sm text-text-primary">{st.name}</span>
<span className="flex items-center gap-2">
<span className="text-xs text-text-secondary">
attempts: {st.attempts}
{st.status === "failed" ? ` · exit ${st.exit_code}` : ""}
</span>
</summary>
{(st.stdout || st.stderr) && (
<pre className="mt-2 max-h-64 overflow-auto rounded bg-black/40 p-2 font-mono text-xs text-text-secondary">
{st.stdout}
{st.stderr ? `\n${st.stderr}` : ""}
</pre>
)}
</details>
<StatusBadge status={st.status} />
</span>
</div>
))}
{sr.steps.length === 0 && (
<p className="text-xs text-text-secondary">No steps yet.</p>
)}
</div>
<ServerLog
runId={run.run_id}
serverId={sr.server_id}
status={sr.status}
/>
</Card>
))}
{run.server_runs.length === 0 && (
+19 -3
View File
@@ -95,6 +95,7 @@ export interface Settings {
alerts: AlertSettings;
email: EmailSettings;
secrets: SecretsSettings;
workflow_log_retention_days?: number | null;
}
export interface SecretGroupSummary {
@@ -171,8 +172,7 @@ export interface StepRun {
status: string;
attempts: number;
exit_code: number;
stdout: string;
stderr: string;
log_offset: number;
output_env: Record<string, string>;
started_at?: string;
finished_at?: string;
@@ -290,7 +290,11 @@ export const api = {
return request<Settings>("/settings");
},
saveSettings(settings: { alerts: AlertSettings; email: EmailSettings }): Promise<{ saved: boolean }> {
saveSettings(settings: {
alerts: AlertSettings;
email: EmailSettings;
workflow_log_retention_days?: number | null;
}): Promise<{ saved: boolean }> {
return request<{ saved: boolean }>("/settings", {
method: "PUT",
body: JSON.stringify(settings),
@@ -460,4 +464,16 @@ export const api = {
cancelRun(runId: string): Promise<void> {
return request<void>(`/runs/${runId}/cancel`, { method: "POST" });
},
async getServerRunLog(runId: string, serverId: string): Promise<string> {
const res = await fetch(`/api/runs/${runId}/servers/${serverId}/logs`, {
credentials: "include",
});
if (!res.ok) throw new Error("no logs");
return res.text();
},
serverRunLogStreamUrl(runId: string, serverId: string): string {
return `/api/runs/${runId}/servers/${serverId}/logs/stream`;
},
};