From a28157dcf85b41ae1060fc935c0e70feb8cfd362 Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Tue, 21 Jul 2026 14:05:19 +0100 Subject: [PATCH] feat(server): store inventory and handle ReportInventory RPC --- ...7-21-adhoc-steps-import-export-defaults.md | 1162 ----------------- ...hoc-steps-import-export-defaults-design.md | 267 ---- server/internal/grpc/server.go | 11 + server/internal/services/inventory.go | 52 + 4 files changed, 63 insertions(+), 1429 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-21-adhoc-steps-import-export-defaults.md delete mode 100644 docs/superpowers/specs/2026-07-21-adhoc-steps-import-export-defaults-design.md create mode 100644 server/internal/services/inventory.go diff --git a/docs/superpowers/plans/2026-07-21-adhoc-steps-import-export-defaults.md b/docs/superpowers/plans/2026-07-21-adhoc-steps-import-export-defaults.md deleted file mode 100644 index bbcc570..0000000 --- a/docs/superpowers/plans/2026-07-21-adhoc-steps-import-export-defaults.md +++ /dev/null @@ -1,1162 +0,0 @@ -# Ad-hoc Steps, Import/Export, Default Steps & Auto-Outputs Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Let workflows carry inline ad-hoc steps, import/export steps as portable JSON, seed default steps from a bind-mounted dir, and auto-derive `declared_outputs` from the script. - -**Architecture:** Go + Gin + MongoDB backend (`server/`), Next.js + TanStack Query frontend (`web/`). Backend changes are pure-function-first (scanner, slug, validation, inline resolve) so they unit-test without a DB, wired into existing service CRUD. Frontend has no test harness — its tasks build/typecheck + manual-verify. - -**Tech Stack:** Go 1.x, gin-gonic, mongo-driver/v2, google/uuid; Next.js (app router), React, TanStack Query, TypeScript. - -## Global Constraints - -- Step JSON envelope: `"kind": "vantage.step/v1"` exactly. -- Env var for defaults dir: `VANTAGE_DEFAULT_STEPS_DIR`, fallback `filepath.Join("data", "default-steps")`, created with `0700` (mirror `WorkflowLogDir`). -- Output env sentinel in scripts: the token `WORKFLOW_ENV` (matches `$WORKFLOW_ENV` bash / `$env:WORKFLOW_ENV` powershell). -- A `WorkflowStepRef` is valid iff **exactly one** of `step_id` / `inline` is set. -- `declared_outputs` is always server-derived; client-sent values are ignored everywhere. -- Default steps are keyed by `{ slug, source: "default" }`; re-sync overwrites their content; `source: "user"` steps are never touched by the seeder. -- Go tests run from `server/`: `go test ./internal/services/ -run -v`. -- Web verify from `web/`: `npm run build`. - ---- - -### Task 1: Model changes + workflow validation - -**Files:** -- Modify: `server/internal/models/workflow.go` -- Create: `server/internal/services/validate.go` -- Test: `server/internal/services/validate_test.go` - -**Interfaces:** -- Produces: `models.WorkflowStep` gains `Source string`, `Slug string`. `models.WorkflowStepRef` gains `Inline *models.WorkflowStep`. `services.ValidateWorkflow(w models.Workflow) error`. - -- [ ] **Step 1: Add fields to models** - -In `server/internal/models/workflow.go`, add to `WorkflowStep` (after `SecretRefs`): - -```go - Source string `bson:"source" json:"source"` // "user" | "default" - Slug string `bson:"slug,omitempty" json:"slug,omitempty"` -``` - -Change `WorkflowStepRef.StepID` tag to omitempty and add `Inline`: - -```go -type WorkflowStepRef struct { - StepID string `bson:"step_id,omitempty" json:"step_id,omitempty"` - Inline *WorkflowStep `bson:"inline,omitempty" json:"inline,omitempty"` - Order int `bson:"order" json:"order"` - OnFailure string `bson:"on_failure" json:"on_failure"` - MaxRetries int `bson:"max_retries" json:"max_retries"` - Overrides *StepOverride `bson:"overrides,omitempty" json:"overrides,omitempty"` - Inputs map[string]string `bson:"inputs,omitempty" json:"inputs,omitempty"` -} -``` - -- [ ] **Step 2: Write the failing test** - -Create `server/internal/services/validate_test.go`: - -```go -package services - -import ( - "testing" - - "github.com/mrhid6/vantage/server/internal/models" -) - -func TestValidateWorkflow(t *testing.T) { - inline := &models.WorkflowStep{Name: "x", Interpreter: "bash", Script: "echo hi"} - cases := []struct { - name string - ref models.WorkflowStepRef - wantErr bool - }{ - {"library only", models.WorkflowStepRef{StepID: "abc"}, false}, - {"inline only", models.WorkflowStepRef{Inline: inline}, false}, - {"both set", models.WorkflowStepRef{StepID: "abc", Inline: inline}, true}, - {"neither set", models.WorkflowStepRef{}, true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := ValidateWorkflow(models.Workflow{Steps: []models.WorkflowStepRef{tc.ref}}) - if (err != nil) != tc.wantErr { - t.Fatalf("got err=%v want wantErr=%v", err, tc.wantErr) - } - }) - } -} -``` - -- [ ] **Step 3: Run test to verify it fails** - -Run: `go test ./internal/services/ -run TestValidateWorkflow -v` -Expected: FAIL — `undefined: ValidateWorkflow`. - -- [ ] **Step 4: Write minimal implementation** - -Create `server/internal/services/validate.go`: - -```go -package services - -import ( - "fmt" - - "github.com/mrhid6/vantage/server/internal/models" -) - -// ValidateWorkflow checks each step ref sets exactly one of step_id / inline. -func ValidateWorkflow(w models.Workflow) error { - for i, ref := range w.Steps { - hasLib := ref.StepID != "" - hasInline := ref.Inline != nil - if hasLib == hasInline { - return fmt.Errorf("step %d: exactly one of step_id or inline must be set", i) - } - } - return nil -} -``` - -- [ ] **Step 5: Run test to verify it passes** - -Run: `go test ./internal/services/ -run TestValidateWorkflow -v` -Expected: PASS. - -- [ ] **Step 6: Commit** - -```bash -git add server/internal/models/workflow.go server/internal/services/validate.go server/internal/services/validate_test.go -git commit -m "feat(server): inline step ref + workflow validation" -``` - ---- - -### Task 2: Output scanner + slug helpers - -**Files:** -- Create: `server/internal/services/stepscan.go` -- Test: `server/internal/services/stepscan_test.go` - -**Interfaces:** -- Produces: `services.DeriveOutputs(script string) []string`, `services.Slugify(name string) string`. - -- [ ] **Step 1: Write the failing test** - -Create `server/internal/services/stepscan_test.go`: - -```go -package services - -import ( - "reflect" - "testing" -) - -func TestDeriveOutputs(t *testing.T) { - script := `#!/bin/bash -echo "test=123" >> $WORKFLOW_ENV -echo "other=hi" >> "$WORKFLOW_ENV" -printf 'third=1\n' >> $WORKFLOW_ENV -echo "test=456" >> $WORKFLOW_ENV -echo "ignored=nope" -NORMAL=assignment -` - got := DeriveOutputs(script) - want := []string{"test", "other", "third"} - if !reflect.DeepEqual(got, want) { - t.Fatalf("got %v want %v", got, want) - } -} - -func TestDeriveOutputsPowershell(t *testing.T) { - script := `"result=ok" >> $env:WORKFLOW_ENV -Add-Content $env:WORKFLOW_ENV "count=5"` - got := DeriveOutputs(script) - want := []string{"result", "count"} - if !reflect.DeepEqual(got, want) { - t.Fatalf("got %v want %v", got, want) - } -} - -func TestDeriveOutputsNone(t *testing.T) { - got := DeriveOutputs("echo hello\nNOPE=1") - if len(got) != 0 { - t.Fatalf("got %v want empty", got) - } -} - -func TestSlugify(t *testing.T) { - if got := Slugify("Restart NGINX Service!"); got != "restart-nginx-service" { - t.Fatalf("got %q", got) - } -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `go test ./internal/services/ -run 'TestDeriveOutputs|TestSlugify' -v` -Expected: FAIL — `undefined: DeriveOutputs`. - -- [ ] **Step 3: Write minimal implementation** - -Create `server/internal/services/stepscan.go`: - -```go -package services - -import ( - "regexp" - "strings" -) - -// keyAssign matches an env-var assignment target: KEY= (captures KEY). -var keyAssign = regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)=`) - -// DeriveOutputs scans a step script and returns the output keys it writes to -// $WORKFLOW_ENV. Best-effort: only lines that reference WORKFLOW_ENV are -// considered. Deduplicated, first-seen order preserved. -func DeriveOutputs(script string) []string { - out := []string{} - seen := map[string]bool{} - for _, line := range strings.Split(script, "\n") { - if !strings.Contains(line, "WORKFLOW_ENV") { - continue - } - for _, m := range keyAssign.FindAllStringSubmatch(line, -1) { - key := m[1] - // Skip the sentinel itself (e.g. "WORKFLOW_ENV=..." assignments). - if key == "WORKFLOW_ENV" || key == "env" { - continue - } - if seen[key] { - continue - } - seen[key] = true - out = append(out, key) - } - } - return out -} - -var slugStrip = regexp.MustCompile(`[^a-z0-9]+`) - -// Slugify converts a step name into a stable kebab-case slug. -func Slugify(name string) string { - s := strings.ToLower(name) - s = slugStrip.ReplaceAllString(s, "-") - return strings.Trim(s, "-") -} -``` - -Note: the `env` skip keeps `$env:WORKFLOW_ENV` from yielding an `env` key when it appears as `env:WORKFLOW_ENV` — `env=` won't match there (no `=`), but the guard is cheap insurance. Verify the powershell test still expects only `result`/`count`. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `go test ./internal/services/ -run 'TestDeriveOutputs|TestSlugify' -v` -Expected: PASS (all four). - -- [ ] **Step 5: Commit** - -```bash -git add server/internal/services/stepscan.go server/internal/services/stepscan_test.go -git commit -m "feat(server): derive declared_outputs from script + slugify" -``` - ---- - -### Task 3: Wire auto-outputs into step CRUD; inline resolve - -**Files:** -- Modify: `server/internal/services/workflows.go` (`CreateStep`, `UpdateStep`, `CreateWorkflow`, `UpdateWorkflow`) -- Modify: `server/internal/services/workflow_runner.go` (`resolveSteps` + new helper) -- Test: `server/internal/services/resolve_test.go` - -**Interfaces:** -- Consumes: `DeriveOutputs`, `ValidateWorkflow` (Task 1-2). -- Produces: `resolveInlineStep(ref models.WorkflowStepRef) models.ResolvedStep`. - -- [ ] **Step 1: Derive outputs in CreateStep/UpdateStep** - -In `server/internal/services/workflows.go` `CreateStep`, before the InsertOne, replace the `DeclaredOutputs == nil` block with an unconditional derive and default source: - -```go - s.DeclaredOutputs = DeriveOutputs(s.Script) - if s.Source == "" { - s.Source = "user" - } -``` - -(Keep the existing nil-guards for `SecretRefs` and `DeclaredInputs`.) - -In `UpdateStep`, change the `$set` `"declared_outputs"` value from `s.DeclaredOutputs` to `DeriveOutputs(s.Script)`. - -- [ ] **Step 2: Normalise inline steps on workflow save** - -In `workflows.go`, add a helper and call it at the top of both `CreateWorkflow` and `UpdateWorkflow` (after the function receives `w`): - -```go -// normalizeInlineSteps derives outputs for inline steps and strips fields that -// only belong to library steps. -func normalizeInlineSteps(w *models.Workflow) { - for i := range w.Steps { - in := w.Steps[i].Inline - if in == nil { - continue - } - in.DeclaredOutputs = DeriveOutputs(in.Script) - in.StepID = "" - in.Slug = "" - in.Source = "" - in.CreatedAt = time.Time{} - in.UpdatedAt = time.Time{} - if in.SecretRefs == nil { - in.SecretRefs = []string{} - } - if in.DeclaredInputs == nil { - in.DeclaredInputs = []models.InputParam{} - } - } -} -``` - -In `CreateWorkflow`, after `w.Steps == nil` guard add: - -```go - if err := ValidateWorkflow(w); err != nil { - return nil, err - } - normalizeInlineSteps(&w) -``` - -In `UpdateWorkflow`, before the `UpdateOne` add: - -```go - if err := ValidateWorkflow(w); err != nil { - return err - } - normalizeInlineSteps(&w) -``` - -- [ ] **Step 3: Write the failing test for inline resolve** - -Create `server/internal/services/resolve_test.go`: - -```go -package services - -import ( - "testing" - - "github.com/mrhid6/vantage/server/internal/models" -) - -func TestResolveInlineStep(t *testing.T) { - ref := models.WorkflowStepRef{ - Order: 2, - OnFailure: "", - Inline: &models.WorkflowStep{ - Name: "adhoc", - Interpreter: "bash", - Script: "echo hi", - SecretRefs: []string{"TOKEN"}, - DeclaredInputs: []models.InputParam{ - {Name: "REGION", Default: "eu"}, - }, - }, - Inputs: map[string]string{"REGION": "us"}, - } - rs := resolveInlineStep(ref) - if rs.Name != "adhoc" || rs.Script != "echo hi" || rs.Order != 2 { - t.Fatalf("bad resolve: %+v", rs) - } - if rs.OnFailure != "stop" { - t.Fatalf("want default on_failure=stop, got %q", rs.OnFailure) - } - if rs.Inputs["REGION"] != "us" { - t.Fatalf("want input override us, got %q", rs.Inputs["REGION"]) - } - if len(rs.SecretRefs) != 1 || rs.SecretRefs[0] != "TOKEN" { - t.Fatalf("bad secret refs: %v", rs.SecretRefs) - } -} -``` - -- [ ] **Step 4: Run test to verify it fails** - -Run: `go test ./internal/services/ -run TestResolveInlineStep -v` -Expected: FAIL — `undefined: resolveInlineStep`. - -- [ ] **Step 5: Implement resolveInlineStep and branch resolveSteps** - -In `server/internal/services/workflow_runner.go`, add: - -```go -// resolveInlineStep freezes an ad-hoc (inline) step ref into a ResolvedStep. -func resolveInlineStep(ref models.WorkflowStepRef) models.ResolvedStep { - in := ref.Inline - inputs := map[string]string{} - for _, p := range in.DeclaredInputs { - if ref.Inputs != nil { - if v, ok := ref.Inputs[p.Name]; ok { - inputs[p.Name] = v - continue - } - } - inputs[p.Name] = p.Default - } - onFailure := ref.OnFailure - if onFailure == "" { - onFailure = "stop" - } - return models.ResolvedStep{ - Order: ref.Order, - Name: in.Name, - Interpreter: in.Interpreter, - Script: in.Script, - SecretRefs: in.SecretRefs, - OnFailure: onFailure, - MaxRetries: ref.MaxRetries, - Inputs: inputs, - } -} -``` - -In `resolveSteps`, at the top of the `for _, ref := range wf.Steps` loop, add before `lib, err := getStep(...)`: - -```go - if ref.Inline != nil { - out = append(out, resolveInlineStep(ref)) - continue - } -``` - -- [ ] **Step 6: Run tests to verify they pass** - -Run: `go test ./internal/services/ -run 'TestResolveInlineStep|TestValidateWorkflow|TestDeriveOutputs' -v` -Expected: PASS. Then `go build ./...` from `server/` — expect no errors. - -- [ ] **Step 7: Commit** - -```bash -git add server/internal/services/workflows.go server/internal/services/workflow_runner.go server/internal/services/resolve_test.go -git commit -m "feat(server): auto-derive outputs on save + resolve inline steps" -``` - ---- - -### Task 4: Export / import / parse steps - -**Files:** -- Create: `server/internal/services/stepio.go` -- Modify: `server/internal/api/workflows.go` (routes + handlers) -- Test: `server/internal/services/stepio_test.go` - -**Interfaces:** -- Consumes: `CreateStep`, `DeriveOutputs`. -- Produces: `services.StepDoc` struct, `services.ParseStepDoc(b []byte) (models.WorkflowStep, error)`, `services.ExportStepDoc(s models.WorkflowStep) StepDoc`, `services.ImportStepToLibrary(b []byte) (*models.WorkflowStep, error)`. - -- [ ] **Step 1: Write the failing test** - -Create `server/internal/services/stepio_test.go`: - -```go -package services - -import ( - "encoding/json" - "testing" - - "github.com/mrhid6/vantage/server/internal/models" -) - -func mkStep() models.WorkflowStep { - return models.WorkflowStep{ - StepID: "should-not-export", Source: "default", Name: "Restart", - Interpreter: "bash", Script: "echo x=1 >> $WORKFLOW_ENV", - SecretRefs: []string{"TOK"}, - } -} - -func TestParseStepDocValid(t *testing.T) { - raw := `{"kind":"vantage.step/v1","name":"Restart","interpreter":"bash", - "script":"echo x=1 >> $WORKFLOW_ENV","declared_outputs":["stale"], - "declared_inputs":[{"name":"A","default":"1"}],"secret_refs":["TOK"]}` - s, err := ParseStepDoc([]byte(raw)) - if err != nil { - t.Fatal(err) - } - if s.Name != "Restart" || s.Interpreter != "bash" { - t.Fatalf("bad parse: %+v", s) - } - // declared_outputs recomputed from script, ignoring the file's ["stale"]. - if len(s.DeclaredOutputs) != 1 || s.DeclaredOutputs[0] != "x" { - t.Fatalf("outputs should be derived, got %v", s.DeclaredOutputs) - } - if s.StepID != "" || s.Source != "" { - t.Fatalf("parse must not set id/source") - } -} - -func TestParseStepDocBadKind(t *testing.T) { - if _, err := ParseStepDoc([]byte(`{"kind":"nope","name":"x"}`)); err == nil { - t.Fatal("want error for bad kind") - } -} - -func TestParseStepDocBadJSON(t *testing.T) { - if _, err := ParseStepDoc([]byte(`{`)); err == nil { - t.Fatal("want error for bad json") - } -} - -func TestExportStepDocRoundTrip(t *testing.T) { - doc := ExportStepDoc(mkStep()) - b, _ := json.Marshal(doc) - s, err := ParseStepDoc(b) - if err != nil { - t.Fatal(err) - } - if s.Name != "Restart" || s.Interpreter != "bash" { - t.Fatalf("round trip lost data: %+v", s) - } -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `go test ./internal/services/ -run 'TestParseStepDoc|TestExportStepDoc' -v` -Expected: FAIL — `undefined: ParseStepDoc`. - -- [ ] **Step 3: Implement stepio.go** - -Create `server/internal/services/stepio.go`: - -```go -package services - -import ( - "encoding/json" - "fmt" - - "github.com/mrhid6/vantage/server/internal/models" -) - -const StepDocKind = "vantage.step/v1" - -// StepDoc is the portable, id-free representation of a step. -type StepDoc struct { - Kind string `json:"kind"` - Name string `json:"name"` - Description string `json:"description"` - Interpreter string `json:"interpreter"` - Script string `json:"script"` - DeclaredOutputs []string `json:"declared_outputs"` - DeclaredInputs []models.InputParam `json:"declared_inputs"` - SecretRefs []string `json:"secret_refs"` -} - -// ExportStepDoc builds a portable doc from a library step (ids/source stripped). -func ExportStepDoc(s models.WorkflowStep) StepDoc { - return StepDoc{ - Kind: StepDocKind, - Name: s.Name, - Description: s.Description, - Interpreter: s.Interpreter, - Script: s.Script, - DeclaredOutputs: s.DeclaredOutputs, - DeclaredInputs: s.DeclaredInputs, - SecretRefs: s.SecretRefs, - } -} - -// ParseStepDoc validates a v1 doc and returns a normalized (id-free) step with -// declared_outputs recomputed from the script. -func ParseStepDoc(b []byte) (models.WorkflowStep, error) { - var d StepDoc - if err := json.Unmarshal(b, &d); err != nil { - return models.WorkflowStep{}, fmt.Errorf("invalid step JSON: %w", err) - } - if d.Kind != StepDocKind { - return models.WorkflowStep{}, fmt.Errorf("unsupported kind %q (want %q)", d.Kind, StepDocKind) - } - if d.Name == "" || d.Interpreter == "" { - return models.WorkflowStep{}, fmt.Errorf("step name and interpreter are required") - } - if d.SecretRefs == nil { - d.SecretRefs = []string{} - } - if d.DeclaredInputs == nil { - d.DeclaredInputs = []models.InputParam{} - } - return models.WorkflowStep{ - Name: d.Name, - Description: d.Description, - Interpreter: d.Interpreter, - Script: d.Script, - DeclaredOutputs: DeriveOutputs(d.Script), - DeclaredInputs: d.DeclaredInputs, - SecretRefs: d.SecretRefs, - }, nil -} - -// ImportStepToLibrary parses a doc and persists it as a new user library step. -func ImportStepToLibrary(b []byte) (*models.WorkflowStep, error) { - s, err := ParseStepDoc(b) - if err != nil { - return nil, err - } - return CreateStep(s) -} - -// ExportStep loads a library step and marshals it to a portable doc. -func ExportStep(stepID string) ([]byte, error) { - ctx, cancel := wfCtx() - defer cancel() - s, err := getStep(ctx, stepID) - if err != nil { - return nil, err - } - return json.MarshalIndent(ExportStepDoc(*s), "", " ") -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `go test ./internal/services/ -run 'TestParseStepDoc|TestExportStepDoc' -v` -Expected: PASS. - -- [ ] **Step 5: Add API routes/handlers** - -In `server/internal/api/workflows.go`, add to `registerWorkflowRoutes` after the existing step routes: - -```go - g.GET("/steps/:id/export", exportStep) - g.POST("/steps/import", importStep) - g.POST("/steps/parse", parseStep) -``` - -Add handlers (place near the other step handlers): - -```go -func exportStep(c *gin.Context) { - b, err := services.ExportStep(c.Param("id")) - if err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) - return - } - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=step-%s.json", c.Param("id"))) - c.Data(http.StatusOK, "application/json", b) -} - -func importStep(c *gin.Context) { - body, err := io.ReadAll(c.Request.Body) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - out, err := services.ImportStepToLibrary(body) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - services.LogEvent("workflow.step_imported", actorFromCtx(c), "", out.StepID, fmt.Sprintf("step '%s' imported", out.Name)) - c.JSON(http.StatusCreated, out) -} - -// parseStep validates a step doc and returns the normalized step WITHOUT -// persisting — used by the editor to insert an imported ad-hoc (inline) step. -func parseStep(c *gin.Context) { - body, err := io.ReadAll(c.Request.Body) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - s, err := services.ParseStepDoc(body) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, s) -} -``` - -Add `"io"` to the import block of `workflows.go`. - -- [ ] **Step 6: Verify build** - -Run: `go build ./...` from `server/` -Expected: no errors. - -- [ ] **Step 7: Commit** - -```bash -git add server/internal/services/stepio.go server/internal/services/stepio_test.go server/internal/api/workflows.go -git commit -m "feat(server): step import/export/parse endpoints" -``` - ---- - -### Task 5: Default steps dir + seeder + boot + route - -**Files:** -- Create: `server/internal/services/defaults.go` -- Modify: `server/internal/services/workflows.go` (`EnsureWorkflowIndexes` — add slug index) -- Modify: `server/cmd/main.go` (boot seed) -- Modify: `server/internal/api/workflows.go` (route + handler) -- Test: `server/internal/services/defaults_test.go` - -**Interfaces:** -- Consumes: `ParseStepDoc`, `Slugify`. -- Produces: `services.DefaultStepsDir() string`, `services.SeedDefaultSteps() (created, updated int, err error)`. - -- [ ] **Step 1: Write the failing test (dir + parse-only pure parts)** - -Create `server/internal/services/defaults_test.go`: - -```go -package services - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDefaultStepsDirEnv(t *testing.T) { - dir := filepath.Join(t.TempDir(), "ds") - t.Setenv("VANTAGE_DEFAULT_STEPS_DIR", dir) - got := DefaultStepsDir() - if got != dir { - t.Fatalf("got %q want %q", got, dir) - } - if _, err := os.Stat(dir); err != nil { - t.Fatalf("dir not created: %v", err) - } -} - -func TestReadDefaultStepFiles(t *testing.T) { - dir := t.TempDir() - t.Setenv("VANTAGE_DEFAULT_STEPS_DIR", dir) - good := `{"kind":"vantage.step/v1","name":"Ping Host","interpreter":"bash","script":"ping -c1 x=1 >> $WORKFLOW_ENV"}` - os.WriteFile(filepath.Join(dir, "ping.json"), []byte(good), 0600) - os.WriteFile(filepath.Join(dir, "notes.txt"), []byte("ignore me"), 0600) - - steps, err := readDefaultStepFiles() - if err != nil { - t.Fatal(err) - } - if len(steps) != 1 { - t.Fatalf("want 1 step, got %d", len(steps)) - } - if steps[0].Slug != "ping-host" || steps[0].Source != "default" { - t.Fatalf("bad seed step: %+v", steps[0]) - } -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `go test ./internal/services/ -run 'TestDefaultStepsDir|TestReadDefaultStepFiles' -v` -Expected: FAIL — `undefined: DefaultStepsDir`. - -- [ ] **Step 3: Implement defaults.go** - -Create `server/internal/services/defaults.go`: - -```go -package services - -import ( - "os" - "path/filepath" - "time" - - "github.com/google/uuid" - "github.com/mrhid6/vantage/server/internal/db" - "github.com/mrhid6/vantage/server/internal/models" - "go.mongodb.org/mongo-driver/v2/bson" - "go.mongodb.org/mongo-driver/v2/mongo" - "go.mongodb.org/mongo-driver/v2/mongo/options" -) - -// DefaultStepsDir returns the directory holding default step JSON files. -func DefaultStepsDir() string { - dir := os.Getenv("VANTAGE_DEFAULT_STEPS_DIR") - if dir == "" { - dir = filepath.Join("data", "default-steps") - } - _ = os.MkdirAll(dir, 0700) - return dir -} - -// readDefaultStepFiles parses every *.json in the defaults dir into -// source=default library steps (with slug set). Non-json and invalid files are -// skipped silently; a slug is derived from the step name. -func readDefaultStepFiles() ([]models.WorkflowStep, error) { - matches, err := filepath.Glob(filepath.Join(DefaultStepsDir(), "*.json")) - if err != nil { - return nil, err - } - out := []models.WorkflowStep{} - for _, path := range matches { - b, err := os.ReadFile(path) - if err != nil { - continue - } - s, err := ParseStepDoc(b) - if err != nil { - continue - } - s.Source = "default" - s.Slug = Slugify(s.Name) - if s.Slug == "" { - continue - } - out = append(out, s) - } - return out, nil -} - -// SeedDefaultSteps upserts default steps from disk keyed on {slug, source}. -// Re-sync overwrites default-step content; user steps are never touched. -func SeedDefaultSteps() (created, updated int, err error) { - steps, err := readDefaultStepFiles() - if err != nil { - return 0, 0, err - } - ctx, cancel := wfCtx() - defer cancel() - col := db.Col("workflow_steps") - for _, s := range steps { - filter := bson.M{"slug": s.Slug, "source": "default"} - set := bson.M{ - "name": s.Name, - "description": s.Description, - "interpreter": s.Interpreter, - "script": s.Script, - "declared_outputs": s.DeclaredOutputs, - "declared_inputs": s.DeclaredInputs, - "secret_refs": s.SecretRefs, - "updated_at": time.Now(), - } - res, uerr := col.UpdateOne(ctx, filter, bson.M{ - "$set": set, - "$setOnInsert": bson.M{ - "step_id": uuid.New().String(), - "slug": s.Slug, - "source": "default", - "created_at": time.Now(), - }, - }, options.UpdateOne().SetUpsert(true)) - if uerr != nil { - return created, updated, uerr - } - if res.UpsertedCount > 0 { - created++ - } else if res.ModifiedCount > 0 { - updated++ - } - } - return created, updated, nil -} - -// used to silence unused import if getStep path not referenced here -var _ = mongo.ErrNoDocuments -``` - -(Remove the trailing `var _ = mongo.ErrNoDocuments` and the `mongo` import if the build reports `mongo` unused — it's only there in case the import is otherwise unreferenced.) - -- [ ] **Step 4: Run test to verify it passes** - -Run: `go test ./internal/services/ -run 'TestDefaultStepsDir|TestReadDefaultStepFiles' -v` -Expected: PASS. - -- [ ] **Step 5: Add slug index** - -In `server/internal/services/workflows.go` `EnsureWorkflowIndexes`, after the `workflow_steps` step_id index, add a partial unique index on slug for default steps: - -```go - if _, err := db.Col("workflow_steps").Indexes().CreateOne(ctx, mongo.IndexModel{ - Keys: bson.D{{Key: "slug", Value: 1}}, - Options: options.Index().SetUnique(true). - SetPartialFilterExpression(bson.M{"source": "default"}), - }); err != nil { - return err - } -``` - -- [ ] **Step 6: Boot seed in main.go** - -In `server/cmd/main.go`, after the `EnsureWorkflowIndexes()` block, add: - -```go - if created, updated, err := services.SeedDefaultSteps(); err != nil { - log.Printf("warning: failed to seed default steps: %v", err) - } else { - log.Printf("default steps seeded: %d created, %d updated", created, updated) - } -``` - -- [ ] **Step 7: Add seed-defaults route/handler** - -In `server/internal/api/workflows.go`, add route: - -```go - g.POST("/steps/seed-defaults", seedDefaults) -``` - -Handler: - -```go -func seedDefaults(c *gin.Context) { - created, updated, err := services.SeedDefaultSteps() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - services.LogEvent("workflow.defaults_synced", actorFromCtx(c), "", "", fmt.Sprintf("default steps synced: %d created, %d updated", created, updated)) - c.JSON(http.StatusOK, gin.H{"created": created, "updated": updated}) -} -``` - -- [ ] **Step 8: Verify build + full service test run** - -Run from `server/`: `go build ./...` then `go test ./internal/services/ -v` -Expected: build clean; all service tests PASS. - -- [ ] **Step 9: Commit** - -```bash -git add server/internal/services/defaults.go server/internal/services/defaults_test.go server/internal/services/workflows.go server/cmd/main.go server/internal/api/workflows.go -git commit -m "feat(server): default steps seed-on-boot + admin re-sync" -``` - ---- - -### Task 6: Frontend API client - -**Files:** -- Modify: `web/lib/api.ts` - -**Interfaces:** -- Produces: `WorkflowStep.source`/`slug` fields, `WorkflowStepRef.inline`, and `api.exportStepUrl`, `api.importStep`, `api.parseStep`, `api.seedDefaults`. - -- [ ] **Step 1: Extend types** - -In `web/lib/api.ts`, add to `WorkflowStep` interface: - -```ts - source?: "user" | "default"; - slug?: string; -``` - -Change `WorkflowStepRef` to make `step_id` optional and add `inline`: - -```ts -export interface WorkflowStepRef { - step_id?: string; - inline?: WorkflowStep; - order: number; - on_failure: "stop" | "continue" | "retry"; - max_retries: number; - overrides?: { script?: string; secret_refs?: string[] }; - inputs?: Record; -} -``` - -- [ ] **Step 2: Add API methods** - -In the `api` object, after `deleteStep`, add: - -```ts - exportStepUrl(stepId: string): string { - return `/api/steps/${stepId}/export`; - }, - - importStep(doc: unknown): Promise { - return request("/steps/import", { - method: "POST", - body: JSON.stringify(doc), - }); - }, - - parseStep(doc: unknown): Promise { - return request("/steps/parse", { - method: "POST", - body: JSON.stringify(doc), - }); - }, - - seedDefaults(): Promise<{ created: number; updated: number }> { - return request<{ created: number; updated: number }>("/steps/seed-defaults", { - method: "POST", - }); - }, -``` - -- [ ] **Step 3: Verify typecheck/build** - -Run from `web/`: `npm run build` -Expected: compiles (types only touched here). - -- [ ] **Step 4: Commit** - -```bash -git add web/lib/api.ts -git commit -m "feat(web): api client for step import/export/inline/defaults" -``` - ---- - -### Task 7: Steps list — export/import/sync + default badge + read-only outputs - -**Files:** -- Read first: `web/app/workflows/[id]/page.tsx`, `web/components/workflows/EditStepModal.tsx` -- Modify: whichever component renders the shared steps list (identified while reading) and `web/components/workflows/EditStepModal.tsx` - -**Interfaces:** -- Consumes: `api.exportStepUrl`, `api.importStep`, `api.seedDefaults`, `WorkflowStep.source`. - -- [ ] **Step 1: Read the current steps UI** - -Read `web/app/workflows/[id]/page.tsx` and `web/components/workflows/EditStepModal.tsx` fully. Identify: (a) where the shared step library list is rendered with its create/edit/delete buttons, (b) the `declared_outputs` editing control in `EditStepModal`. - -- [ ] **Step 2: Make declared_outputs read-only in EditStepModal** - -In `EditStepModal.tsx`, replace the editable declared-outputs input with a read-only display derived from the step (outputs are computed server-side on save). Show a hint. Concretely: remove the add/remove-output handlers and the mutable input; render the current `declared_outputs` as static chips/text with helper copy `Outputs are detected automatically from lines writing to $WORKFLOW_ENV.` Do not send `declared_outputs` in the create/update payload (the server ignores it regardless). - -- [ ] **Step 3: Add Export button per step** - -In the steps list, for each step add an Export control that triggers a download: - -```tsx - - - -``` - -- [ ] **Step 4: Add Import + Sync defaults controls** - -Above the steps list add a hidden file input and two buttons. Import reads the file text, `JSON.parse`, calls `api.importStep`, then invalidates the steps query. Sync calls `api.seedDefaults`, toasts `${created} created, ${updated} updated`, invalidates the steps query. Follow the file's existing mutation + `queryClient.invalidateQueries` + toast patterns discovered in Step 1. Example import handler: - -```tsx -async function onImportFile(e: React.ChangeEvent) { - const file = e.target.files?.[0]; - if (!file) return; - try { - const doc = JSON.parse(await file.text()); - await api.importStep(doc); - await queryClient.invalidateQueries({ queryKey: ["steps"] }); - } catch (err) { - // surface via the page's existing error/toast mechanism - } finally { - e.target.value = ""; - } -} -``` - -(Use the actual steps query key found in Step 1 in place of `["steps"]`.) - -- [ ] **Step 5: Default badge** - -For steps where `step.source === "default"`, render a small `default` badge next to the name, following the badge styling already used elsewhere in the page. - -- [ ] **Step 6: Verify build + manual check** - -Run from `web/`: `npm run build` — expect success. Manual: with the dev stack running, export a step (downloads JSON), re-import it (new step appears), click Sync defaults (toast shows counts), confirm default steps show the badge and their outputs render read-only. - -- [ ] **Step 7: Commit** - -```bash -git add web/app/workflows/ web/components/workflows/EditStepModal.tsx -git commit -m "feat(web): step export/import, sync defaults, auto outputs, default badge" -``` - ---- - -### Task 8: Workflow editor — add ad-hoc step + import to inline - -**Files:** -- Read first: `web/app/workflows/[id]/page.tsx`, `web/components/workflows/EditWorkflowModal.tsx` -- Modify: the workflow editing component that manages `workflow.steps` (the step ref list) - -**Interfaces:** -- Consumes: `api.parseStep`, `WorkflowStepRef.inline`, `WorkflowStep`. - -- [ ] **Step 1: Read the workflow step-ref editor** - -Read `web/components/workflows/EditWorkflowModal.tsx` and the relevant part of `web/app/workflows/[id]/page.tsx`. Identify where step refs are added (the "add step from library" control) and how a ref is appended to `workflow.steps` before save. - -- [ ] **Step 2: Add "Add ad-hoc step" control** - -Next to the existing add-from-library control, add an "Add ad-hoc step" button that appends a new ref with an `inline` object and no `step_id`: - -```ts -const newRef: WorkflowStepRef = { - inline: { - step_id: "", - name: "New ad-hoc step", - description: "", - interpreter: "bash", - script: "", - declared_outputs: [], - declared_inputs: [], - secret_refs: [], - }, - order: workflow.steps.length, - on_failure: "stop", - max_retries: 0, -}; -``` - -Render an inline edit form for `ref.inline` (name, interpreter select, script textarea) when a ref has `inline` set, editing the ref's inline object in local state. Library refs keep their existing override UI. Distinguish inline refs with an `ad-hoc` badge. - -- [ ] **Step 3: Add "Import ad-hoc from file"** - -Add a control that reads a JSON file, `JSON.parse`, calls `api.parseStep(doc)`, then appends the returned `WorkflowStep` as a new inline ref: - -```tsx -async function onImportInline(e: React.ChangeEvent) { - const file = e.target.files?.[0]; - if (!file) return; - try { - const doc = JSON.parse(await file.text()); - const step = await api.parseStep(doc); - appendRef({ inline: step, order: workflow.steps.length, on_failure: "stop", max_retries: 0 }); - } catch (err) { - // surface via existing error handling - } finally { - e.target.value = ""; - } -} -``` - -(`appendRef` = the file's existing add-step state updater found in Step 1.) - -- [ ] **Step 4: Ensure save sends inline refs** - -Confirm the workflow save payload serializes `steps` including `inline` (it will, since the ref object carries it). The server validates exactly-one-of and derives inline outputs. No client-side output entry for inline steps. - -- [ ] **Step 5: Verify build + manual check** - -Run from `web/`: `npm run build` — expect success. Manual: add an ad-hoc step to a workflow, save, reopen — inline step persists and is not in the shared library. Run the workflow — the ad-hoc step executes. Import a step JSON as an ad-hoc step and confirm it appends inline. - -- [ ] **Step 6: Commit** - -```bash -git add web/app/workflows/ web/components/workflows/EditWorkflowModal.tsx -git commit -m "feat(web): ad-hoc inline steps + import-to-inline in workflow editor" -``` - ---- - -## Self-Review Notes - -- **Spec coverage:** §1 model → Task 1; §2 resolve → Task 3; §3 import/export → Task 4 + Task 6/7; §4 defaults → Task 5 + Task 7; §5 auto-outputs → Task 2 + Task 3 (wiring) + Task 7 (read-only UI); §6 web → Tasks 6-8. All covered. -- **DeclaredOutputs client-ignored:** enforced in CreateStep/UpdateStep (Task 3), ParseStepDoc (Task 4), seeder (via ParseStepDoc, Task 5). -- **Exactly-one-of validation:** ValidateWorkflow (Task 1), called in Create/UpdateWorkflow (Task 3). -- **Naming consistency:** `DeriveOutputs`, `Slugify`, `ParseStepDoc`, `ExportStepDoc`, `ExportStep`, `ImportStepToLibrary`, `SeedDefaultSteps`, `DefaultStepsDir`, `resolveInlineStep`, `normalizeInlineSteps`, `ValidateWorkflow` — used identically across tasks. diff --git a/docs/superpowers/specs/2026-07-21-adhoc-steps-import-export-defaults-design.md b/docs/superpowers/specs/2026-07-21-adhoc-steps-import-export-defaults-design.md deleted file mode 100644 index bd28deb..0000000 --- a/docs/superpowers/specs/2026-07-21-adhoc-steps-import-export-defaults-design.md +++ /dev/null @@ -1,267 +0,0 @@ -# Ad-hoc Steps, Step Import/Export, and Default Steps — Design - -Date: 2026-07-21 - -## Summary - -Four related additions to the workflow step system: - -1. **Ad-hoc steps** — steps defined inline in a single workflow, not written to the - shared step library. -2. **Import/Export** — single-step portable JSON (`vantage.step/v1`). Import can - target the shared library or a workflow as an inline ad-hoc step. -3. **Default steps** — JSON files in a bind-mounted directory, seeded into the - library on boot and re-syncable on demand. Org-ready for a future SaaS plan. -4. **Auto-derived outputs** — `declared_outputs` is scanned from the script - (writes to `$WORKFLOW_ENV`) instead of being entered by hand. - -Existing model: shared steps live in the `workflow_steps` collection; a -`Workflow.Steps[]` is a list of `WorkflowStepRef` that reference a library step by -`step_id` and may carry `Overrides` + `Inputs`. `resolveSteps` freezes each ref -into a `ResolvedStep` snapshot at run time. - -## 1. Data model - -`server/internal/models/workflow.go`. - -### WorkflowStep - -Add a provenance field: - -```go -Source string `bson:"source" json:"source"` // "user" | "default" -Slug string `bson:"slug" json:"slug"` // kebab of name; stable key for default seeding -``` - -`Slug` is set for `source="default"` steps (used as the upsert key by the seeder). -For `source="user"` steps it may be empty. Existing steps default to -`source="user"` (absent field decodes to ""). - -### WorkflowStepRef - -Add an inline definition. A ref is **either** a library ref (`StepID` set) **or** -ad-hoc (`Inline` set). Never both. - -```go -type WorkflowStepRef struct { - StepID string `bson:"step_id,omitempty" json:"step_id"` - Inline *WorkflowStep `bson:"inline,omitempty" json:"inline,omitempty"` - Order int `bson:"order" json:"order"` - OnFailure string `bson:"on_failure" json:"on_failure"` - MaxRetries int `bson:"max_retries" json:"max_retries"` - Overrides *StepOverride `bson:"overrides,omitempty" json:"overrides,omitempty"` // library-ref only - Inputs map[string]string `bson:"inputs,omitempty" json:"inputs,omitempty"` -} -``` - -`Inline` reuses `WorkflowStep` (name, description, interpreter, script, -declared_inputs, declared_outputs, secret_refs). Its `ID`, `StepID`, `Slug`, -`Source`, and timestamps stay empty and are never persisted to `workflow_steps`. - -Validation on workflow create/update: for each ref, exactly one of `StepID` / -`Inline` must be set. `Overrides` is ignored when `Inline` is set. - -## 2. Resolve at run time - -`server/internal/services/workflow_runner.go`, `resolveSteps`. - -For each ref: - -- If `ref.Inline != nil`: build `ResolvedStep` from `ref.Inline` directly - (name/interpreter/script/secret_refs), apply `ref.Inputs` against - `Inline.DeclaredInputs` defaults. Skip `getStep`, skip `Overrides`. -- Else: current library path unchanged (load step, apply overrides). - -`ResolvedStep` output shape and the run snapshot are unchanged, so the runner and -the run-history UI need no changes. - -`DeleteStep` cascade is unaffected — ad-hoc refs carry no `step_id`, so they never -match the cascade query. - -## 3. Import / Export - -Portable single-step JSON, `kind: "vantage.step/v1"`: - -```json -{ - "kind": "vantage.step/v1", - "name": "...", - "description": "...", - "interpreter": "bash", - "script": "...", - "declared_inputs": [ { "name": "...", "default": "...", "description": "..." } ], - "declared_outputs": ["..."], - "secret_refs": ["NAME"] -} -``` - -Export strips `_id`, `step_id`, `slug`, `source`, and timestamps. -Secret refs are exported as names only. On import, dangling secret refs are kept -verbatim (not auto-created). - -### Service functions (`services/workflows.go`) - -- `ExportStep(stepID string) ([]byte, error)` — load library step, marshal to the - v1 shape. -- `ParseStepDoc(b []byte) (models.WorkflowStep, error)` — validate `kind`, decode - into a `WorkflowStep` (no id/source). Shared by both import targets. -- `ImportStepToLibrary(b []byte) (*models.WorkflowStep, error)` — `ParseStepDoc` - then `CreateStep` (fresh `step_id`, `source="user"`). - -Import-to-inline needs no new service fn: the web editor calls `ParseStepDoc`'s -API equivalent (see routes) and drops the returned step object into a new -`WorkflowStepRef.Inline` in the workflow it's editing, then saves the workflow -normally. - -### Routes (`api/workflows.go`) - -- `GET /api/steps/:id/export` — returns JSON as a downloadable attachment - (`Content-Disposition`). -- `POST /api/steps/import` — body is the v1 JSON; imports to library; returns the - created step. (Used by the "import to library" flow.) -- `POST /api/steps/parse` — body is the v1 JSON; validates and returns the - normalized step object **without** persisting. Used by "import to inline" so the - editor can insert it as an ad-hoc ref. (Keeps parsing/validation server-side.) - -Audit: `workflow.step_imported` logged on library import. - -## 4. Default steps (seed + admin re-sync) - -Mirrors the existing `WorkflowLogDir` pattern — a bind-mounted directory, no Go -`embed`. - -### Directory - -```go -// DefaultStepsDir returns the directory holding default step JSON files, creating it. -func DefaultStepsDir() string { - dir := os.Getenv("VANTAGE_DEFAULT_STEPS_DIR") - if dir == "" { - dir = filepath.Join("data", "default-steps") - } - _ = os.MkdirAll(dir, 0700) - return dir -} -``` - -Compose already bind-mounts `./data:/data`. Set -`VANTAGE_DEFAULT_STEPS_DIR=/data/default-steps` for explicitness (optional). -Operator drops `*.json` (`vantage.step/v1`) files into that folder. - -### Seeder - -`SeedDefaultSteps() (created, updated int, err error)`: - -1. Glob `DefaultStepsDir()/*.json`. -2. For each file: `ParseStepDoc`, compute `slug = kebab(name)`. -3. Upsert into `workflow_steps` keyed on `{ slug, source: "default" }`: - - absent → insert with fresh `step_id`, `source="default"`, `slug`. (`created++`) - - present → `$set` name/description/interpreter/script/declared_*/secret_refs + - `updated_at`. (`updated++`) - -**Override rule (confirmed):** re-sync is authoritative for `source="default"` -steps and overwrites their content, reverting any user edits to those steps. -`source="user"` steps are never touched by the seeder, even on a slug collision -(the seeder query is scoped to `source: "default"`). - -Add a partial unique index on `slug` where `source == "default"` (or enforce -uniqueness in the seeder loop) to keep default slugs unambiguous. - -### Boot - -Call `SeedDefaultSteps()` from server startup after `EnsureWorkflowIndexes()` -(alongside index setup in `server/cmd/main.go`). Log the created/updated counts; -a seed error is logged but non-fatal (server still boots). - -### Route - -- `POST /api/steps/seed-defaults` (admin) — runs `SeedDefaultSteps()`, returns - `{ "created": n, "updated": m }`. Audit `workflow.defaults_synced`. - -### Org readiness - -Signature stays global today. When Orgs land, `SeedDefaultSteps(orgID)` seeds -per-org and the upsert key becomes `{ org_id, slug, source }`. No schema churn -blocks that later change. - -## 5. Auto-derived outputs - -Today `WorkflowStep.DeclaredOutputs` is entered by hand and consumed only by the -UI (no runtime reads it — outputs are captured at run time by `parseEnvFile` on -the agent). Replace manual entry with a server-side scan of the script. - -At run time the agent exposes an env file path in `$WORKFLOW_ENV` (bash) / -`$env:WORKFLOW_ENV` (powershell); a step emits an output by appending a -`KEY=value` line to it, e.g. `echo "test=123" >> $WORKFLOW_ENV`. - -### Scanner - -`services.DeriveOutputs(script string) []string`: - -- Scan line by line. For each line that references `WORKFLOW_ENV`, extract every - `KEY=` assignment target on that line, where `KEY` matches - `[A-Za-z_][A-Za-z0-9_]*`. -- Covers the common forms across both interpreters (line mentions `WORKFLOW_ENV` - and contains `KEY=...`): - - `echo "test=123" >> $WORKFLOW_ENV` - - `echo "test=123" >> "$WORKFLOW_ENV"` - - `printf 'k=v\n' >> $WORKFLOW_ENV` - - `"k=v" >> $env:WORKFLOW_ENV` / `Add-Content $env:WORKFLOW_ENV "k=v"` -- Deduplicate, preserve first-seen order. Best-effort heuristic — false positives - are acceptable (they only widen the documented output list); it never affects - what the agent actually captures. - -### Wiring - -- `CreateStep` and `UpdateStep` set `DeclaredOutputs = DeriveOutputs(s.Script)`, - ignoring any client-sent value. -- Inline ad-hoc steps: `DeriveOutputs` is applied when the workflow is saved (for - each `ref.Inline`), so inline outputs are derived too. -- `ParseStepDoc` (import) also derives outputs, so `declared_outputs` in an - imported/exported file is informational and always recomputed on import. -- `SeedDefaultSteps` derives outputs the same way when upserting. - -`DeclaredOutputs` stays in the model and JSON (still shown in the UI and used to -wire step-to-step input references), it is just no longer user-authored. - -### Web - -The step editor's "declared outputs" input becomes a read-only, auto-populated -display (derived from the script, refreshed on save / on script edit). No manual -add/remove. - -## 6. Web - -`web/app/workflows/[id]/page.tsx` and the steps list page. - -- **Steps list:** per-row **Export** (downloads JSON) and top-level **Import** - (file picker → `POST /api/steps/import` → library). **Sync defaults** admin - button → `POST /api/steps/seed-defaults`, toast the counts. `source="default"` - rows get a "default" badge. -- **Workflow editor — Add step:** existing "add from library" plus **Add ad-hoc - step** (inline mini-form: name, interpreter, script, optional inputs) stored as - a `WorkflowStepRef.Inline`. Also **Import ad-hoc from file** → `POST - /api/steps/parse` → inserts the returned step as a new inline ref. -- Ad-hoc rows in the editor are visually distinguished from library refs (badge) - and are editable in place; library refs keep the existing override UI. - -## Testing - -- `resolveSteps`: inline ref resolves without touching the library; inputs apply - from `Inline.DeclaredInputs` defaults and ref overrides; library path unchanged. -- Workflow validation: rejects a ref with both `StepID` and `Inline`, and one with - neither. -- Export → import round-trips to an equivalent library step with a new `step_id`. -- `ParseStepDoc` rejects a wrong/missing `kind`. -- `SeedDefaultSteps`: insert-then-update idempotency; user steps untouched; - user-edited default step reverted on re-sync; counts correct. -- `DeleteStep` cascade ignores ad-hoc refs. -- `DeriveOutputs`: extracts keys from each interpreter form above, dedupes, - preserves order, ignores lines not referencing `WORKFLOW_ENV`; create/update/ - import/seed all populate `declared_outputs` from it and ignore client input. - -## Out of scope - -- Whole-workflow export/import. -- Auto-creating secret refs on import. -- Multi-tenant Org model (design is forward-compatible only). diff --git a/server/internal/grpc/server.go b/server/internal/grpc/server.go index 9a96d4f..2a0e7f8 100644 --- a/server/internal/grpc/server.go +++ b/server/internal/grpc/server.go @@ -95,6 +95,17 @@ func (s *vantageServer) ReportUpdates(ctx context.Context, req *pb.ReportUpdates return &pb.ReportUpdatesResponse{}, nil } +func (s *vantageServer) ReportInventory(ctx context.Context, req *pb.InventoryReport) (*pb.InventoryReportResponse, error) { + srv, err := services.ValidateAgentToken(req.ServerId, req.AgentToken) + if err != nil { + return nil, status.Errorf(codes.Unauthenticated, "invalid agent token") + } + if err := services.StoreInventory(srv.ServerID, req); err != nil { + log.Printf("store inventory for %s: %v", srv.ServerID, err) + } + return &pb.InventoryReportResponse{}, nil +} + func (s *vantageServer) CommandStream(stream pb.Vantage_CommandStreamServer) error { // First message authenticates the agent and signals readiness. msg, err := stream.Recv() diff --git a/server/internal/services/inventory.go b/server/internal/services/inventory.go new file mode 100644 index 0000000..adf96ad --- /dev/null +++ b/server/internal/services/inventory.go @@ -0,0 +1,52 @@ +package services + +import ( + "context" + "time" + + "github.com/mrhid6/vantage/server/internal/db" + "github.com/mrhid6/vantage/server/internal/grpc/pb" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// StoreInventory upserts the latest inventory snapshot onto the server document. +// Metrics fields update every call; static fields only when r.IncludeStatic. +func StoreInventory(serverID string, r *pb.InventoryReport) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + now := time.Now() + set := bson.M{"inventory.metrics_at": now} + if r.CPU != nil { + set["inventory.cpu.usage_pct"] = r.CPU.UsagePct + set["inventory.cpu.load1"] = r.CPU.Load1 + } + if r.Memory != nil { + set["inventory.memory.used_bytes"] = r.Memory.UsedBytes + } + set["inventory.swap_used_bytes"] = r.SwapUsed + + if r.IncludeStatic { + set["inventory.static_at"] = now + set["inventory.swap_total_bytes"] = r.SwapTotal + set["inventory.kernel"] = r.Kernel + if r.CPU != nil { + set["inventory.cpu.model"] = r.CPU.Model + set["inventory.cpu.cores"] = r.CPU.Cores + } + if r.Memory != nil { + set["inventory.memory.total_bytes"] = r.Memory.TotalBytes + } + parts := make([]bson.M, 0, len(r.Partitions)) + for _, p := range r.Partitions { + parts = append(parts, bson.M{ + "device": p.Device, "mountpoint": p.Mountpoint, "fstype": p.Fstype, + "total_bytes": p.TotalBytes, "used_bytes": p.UsedBytes, + }) + } + set["inventory.partitions"] = parts + } + + _, err := db.Col("servers").UpdateOne(ctx, bson.M{"server_id": serverID}, bson.M{"$set": set}) + return err +}