feat(runs): mask encoded and multi-line secrets in run logs

Mask base64 and URL-encoded forms of secret values, each line of a
multi-line secret, and secrets loaded by earlier steps of the run.
Replace longer values first so overlapping secrets mask cleanly.

server/internal/services/secret_mask_test.go

@@ -0,0 +1,27 @@
+package services
+
+import (
+	"encoding/base64"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestMaskSecretsEncodedAndMultiline(t *testing.T) {
+	pem := "-----BEGIN KEY-----\nAbCdEfGh12345678\n-----END KEY-----"
+	secrets := map[string]string{"TOKEN": "p@ss w0rd/x+y", "KEY": pem, "SHORT": "ab"}
+	for _, in := range []string{
+		"raw p@ss w0rd/x+y",
+		"b64 " + base64.StdEncoding.EncodeToString([]byte("p@ss w0rd/x+y")),
+		"url " + url.QueryEscape("p@ss w0rd/x+y"),
+		"AbCdEfGh12345678",
+	} {
+		out := maskSecrets(in, secrets)
+		if !strings.Contains(out, "***") || strings.Contains(out, "w0rd") || strings.Contains(out, "AbCd") {
+			t.Errorf("not masked: %q -> %q", in, out)
+		}
+	}
+	if out := maskSecrets("about", secrets); out != "about" {
+		t.Errorf("short secret masked ordinary text: %q", out)
+	}
+}

server/internal/services/workflow_runner.go

@@ -1,8 +1,11 @@
 package services
 
 import (
+	"encoding/base64"
 	"fmt"
+	"net/url"
 	"os"
+	"sort"
 	"strings"
 	"time"
 
@@ -17,6 +20,10 @@ import (
 
 const stepDispatchGrace = 15 * time.Second
 
+// minMaskLen skips masking values so short that replacing them would shred
+// ordinary output, such as a one-character line of a multi-line secret.
+const minMaskLen = 4
+
 // TriggerWorkflow starts a run of workflow workflowID.
 //
 // tokenScope is the acting credential's tag restriction, nil meaning
@@ -245,7 +252,7 @@ func runServer(instanceID, runID string, srvIdx int, steps []models.ResolvedStep
 
 		marker := fmt.Sprintf("===== step %d/%d: %s (%s) =====", step.Order+1, len(steps), step.Name, step.Interpreter)
 		offset, _ := AppendMarker(runID, serverID, marker)
-		secretsSlice := secretValues(secretVals)
+		secretsSlice := secretValues(allSecrets)
 
 		commandID := uuid.New().String()
 		for attempts < maxAttempts {
@@ -378,13 +385,7 @@ func resolveSecrets(instanceID string, refs []string) map[string]string {
 }
 
 func maskSecrets(s string, secrets map[string]string) string {
-	for _, v := range secrets {
-		if v == "" {
-			continue
-		}
-		s = strings.ReplaceAll(s, v, "***")
-	}
-	return s
+	return string(maskBytes([]byte(s), secretValues(secrets)))
 }
 
 func setServerRun(runID string, srvIdx int, set bson.M) {
@@ -423,11 +424,40 @@ func finishStep(runID, serverID string, order int, status string, attempts, exit
 	})
 }
 
+// secretValues returns every form of the secrets worth masking: the raw value,
+// its base64 and URL-encoded forms, and each line of a multi-line value, since
+// logs are masked a line at a time and a PEM key would otherwise never match.
+// Longest first, so a shorter value never breaks up a longer one before it is
+// replaced.
 func secretValues(m map[string]string) []string {
-	out := make([]string, 0, len(m))
+	seen := map[string]bool{}
+	add := func(v string) {
+		if len(strings.TrimSpace(v)) >= minMaskLen {
+			seen[v] = true
+		}
+	}
 	for _, v := range m {
+		for _, f := range []string{v,
+			base64.StdEncoding.EncodeToString([]byte(v)),
+			base64.RawStdEncoding.EncodeToString([]byte(v)),
+			base64.URLEncoding.EncodeToString([]byte(v)),
+			base64.RawURLEncoding.EncodeToString([]byte(v)),
+			url.QueryEscape(v),
+			url.PathEscape(v),
+		} {
+			add(f)
+		}
+		if strings.Contains(v, "\n") {
+			for _, line := range strings.Split(v, "\n") {
+				add(strings.TrimRight(line, "\r"))
+			}
+		}
+	}
+	out := make([]string, 0, len(seen))
+	for v := range seen {
 		out = append(out, v)
 	}
+	sort.Slice(out, func(i, j int) bool { return len(out[i]) > len(out[j]) })
 	return out
 }