From 5dda3b5c4a0ef5907f04e1fd90ae8939015b8d1f Mon Sep 17 00:00:00 2001 From: mrhid6 Date: Thu, 6 Aug 2026 14:33:46 +0100 Subject: [PATCH] feat: vulnerability scanning pipeline, matcher, scheduler and API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes tasks 10-15 and fixes what was outstanding: - vulndb.Pull implemented with oras-go, streaming the ~50MB layer and staging both files before replacing either, so a failed pull leaves the previous database intact rather than a half-written one. - db.go: Vulnerability.Severity is a string, not trivy Severity, so the int conversion did not compile. Severity now resolves vendor (highest when vendors disagree) then NVD then unknown, and CVSS is read too. - findings.go: added sweepFixedFindings plus the fleet query, severity counts, rescan flag and accept/unaccept the API needs. - vulnrules.go: added rule CRUD and the digest builder. ResolveTargets returns []models.Server, not []string, so filterByServers was wrong. - api/vulnerabilities.go was an empty file while handlers.go registered twelve routes against it; written, grouped by CVE. - shared/mail: added the missing sender. The templates were orphaned and the HTML one was a copy of the text one, defining "subject" (which html/template would escape) and emitting no markup. render.go parses every template in init(), so a bad one panics server, admin and sitesvc at boot — go build never runs init(), which is why nothing complained. - notify: digests dispatch through their own path so SMTP gets the digest template rather than arriving dressed as a monitor alert. --- go.work.sum | 50 +++ server/cmd/main.go | 7 + server/go.mod | 4 + server/go.sum | 8 + server/internal/api/handlers.go | 13 + server/internal/api/vulnerabilities.go | 304 ++++++++++++++ server/internal/notify/dispatch.go | 6 +- server/internal/notify/vuln.go | 83 ++++ server/internal/services/findings.go | 420 ++++++++++++++++++++ server/internal/services/vulnrules.go | 363 +++++++++++++++++ server/internal/vulndb/db.go | 129 ++++++ server/internal/vulndb/match.go | 81 ++++ server/internal/vulndb/pull.go | 186 +++++++++ server/internal/vulnsched/sched.go | 238 +++++++++++ shared/mail/templates/vuln_digest.html.tmpl | 13 + shared/mail/templates/vuln_digest.txt.tmpl | 12 + shared/mail/vuln.go | 41 ++ shared/models/settings.go | 5 + 18 files changed, 1962 insertions(+), 1 deletion(-) create mode 100644 server/internal/api/vulnerabilities.go create mode 100644 server/internal/notify/vuln.go create mode 100644 server/internal/services/findings.go create mode 100644 server/internal/services/vulnrules.go create mode 100644 server/internal/vulndb/db.go create mode 100644 server/internal/vulndb/match.go create mode 100644 server/internal/vulndb/pull.go create mode 100644 server/internal/vulnsched/sched.go create mode 100644 shared/mail/templates/vuln_digest.html.tmpl create mode 100644 shared/mail/templates/vuln_digest.txt.tmpl create mode 100644 shared/mail/vuln.go diff --git a/go.work.sum b/go.work.sum index 6011376..5bdf3f4 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,22 +1,69 @@ cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/Intevation/gval v1.3.0/go.mod h1:xmGyGpP5be12EL0P12h+dqiYG8qn2j3PJxIgkoOHO5o= +github.com/Intevation/jsonpath v0.2.1/go.mod h1:WnZ8weMmwAx/fAO3SutjYFU+v7DFreNYnibV7CiaYIw= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986/go.mod h1:NT+jyeCzXk6vXR5MTkdn4z64TgGfE5HMLC8qfj5unl8= +github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce/go.mod h1:HXgVzOPvXhVGLJs4ZKO817idqr/xhwsTcj17CLYY74s= +github.com/aquasecurity/go-npm-version v0.0.1/go.mod h1:hxbJZtKlO4P8sZ9nztizR6XLoE33O+BkPmuYQ4ACyz0= +github.com/aquasecurity/go-pep440-version v0.0.1/go.mod h1:3naPe+Bp6wi3n4l5iBFCZgS0JG8vY6FT0H4NGhFJ+i4= +github.com/aquasecurity/go-version v0.0.1/go.mod h1:s1UU6/v2hctXcOa3OLwfj5d9yoXHa3ahf+ipSwEvGT0= +github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= +github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/goccy/go-yaml v1.19.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gocsaf/csaf/v3 v3.1.1/go.mod h1:EpUCrQg69i+Y66MphmQvVbcj333GFLjXOYHg1zoXVso= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/josephburnett/jd/v2 v2.3.0/go.mod h1:0I5+gbo7y8diuajJjm79AF44eqTheSJy1K7DSbIUFAQ= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/masahiro331/go-mvn-version v0.0.0-20250131095131-f4974fa13b8a/go.mod h1:jZ3F25l7DbD7l7DcA8aj7eo1EZ84nbzcQHBB4lCSrI8= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= +github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= +github.com/pandatix/go-cvss v0.6.2/go.mod h1:jDXYlQBZrc8nvrMUVVvTG8PhmuShOnKrxP53nOFkt8Q= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.50.0 h1:XrG0xOeHs+4FQ8gJR97zDz5uOFMW7OwFWiFVzqopKgY= +github.com/samber/lo v1.50.0/go.mod h1:RjZyNk6WSnUFRKK6EyOhsRJMqft3G+pg7dCWHQCWvsc= +github.com/samber/oops v1.18.1 h1:qjhZbqbdyhWBKntkY8sxrDNKA8b4c5VHlmI1rli7X7M= +github.com/samber/oops v1.18.1/go.mod h1:xYqvimigkKV70HyLXiBZJFpIWi2CGcc6Xx7eV+2HycI= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20251125195548-87e1e737ad39/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= @@ -32,10 +79,13 @@ golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/server/cmd/main.go b/server/cmd/main.go index d5d81be..92a174a 100644 --- a/server/cmd/main.go +++ b/server/cmd/main.go @@ -24,6 +24,7 @@ import ( grpcserver "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/grpc" "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/monitorsched" "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/services" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/vulnsched" "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/workflowsched" "github.com/gin-gonic/gin" ) @@ -195,6 +196,12 @@ func serve() { LogEvent: services.LogEvent, }) + vulnsched.Start(jobCtx, vulnsched.Deps{ + LogEvent: services.LogEvent, + SendDigest: services.SendVulnDigest, + }) + services.StartVulnSweeper(jobCtx) + ticker := time.NewTicker(2 * time.Minute) defer ticker.Stop() for { diff --git a/server/go.mod b/server/go.mod index 5cfd081..bffcdcc 100644 --- a/server/go.mod +++ b/server/go.mod @@ -15,11 +15,15 @@ require ( ) require ( + github.com/aquasecurity/trivy-db v0.0.0-20260713131703-4be526083c54 // indirect github.com/hyperboloide/lk v0.0.0-20251220053519-b291812e3216 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20241115132648-6f4aee6ccd23 // indirect github.com/knqyf263/go-rpm-version v0.0.0-20240918084003-2afd7dc6a38f // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + oras.land/oras-go/v2 v2.6.2 // indirect ) require ( diff --git a/server/go.sum b/server/go.sum index 103fb7b..4b698e9 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,3 +1,5 @@ +github.com/aquasecurity/trivy-db v0.0.0-20260713131703-4be526083c54 h1:4CZNoDkNfcuACevZeDraACGmP1+L0nKkRY52+jV8k1M= +github.com/aquasecurity/trivy-db v0.0.0-20260713131703-4be526083c54/go.mod h1:iIEV2oGuZScvfyX2SMIn78iVMNnepgo0QuJJh/srgVI= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -70,6 +72,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -170,4 +176,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo= +oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/server/internal/api/handlers.go b/server/internal/api/handlers.go index 3dc48da..8be3694 100644 --- a/server/internal/api/handlers.go +++ b/server/internal/api/handlers.go @@ -119,6 +119,19 @@ func RegisterRoutes(r *gin.Engine) { providers.POST("/:id/ack-notice", ackAuthProviderNotice) } apiGroup.GET("/auth/presets", auth.RequireRole("owner", "admin"), listAuthPresets) + + apiGroup.GET("/vulnerabilities", listVulnerabilities) + apiGroup.GET("/vulnerabilities/summary", vulnerabilitySummary) + apiGroup.POST("/vulnerabilities/rescan", auth.RequireRole("owner", "admin"), rescanVulnerabilities) + apiGroup.POST("/vulnerabilities/:id/accept", auth.RequireRole("owner", "admin"), acceptFinding) + apiGroup.DELETE("/vulnerabilities/:id/accept", auth.RequireRole("owner", "admin"), unacceptFinding) + apiGroup.GET("/servers/:id/vulnerabilities", listServerVulnerabilities) + apiGroup.GET("/servers/:id/packages", getServerPackages) + apiGroup.GET("/packages/search", searchPackages) + apiGroup.GET("/vuln-rules", listVulnRules) + apiGroup.POST("/vuln-rules", auth.RequireRole("owner", "admin"), createVulnRule) + apiGroup.PUT("/vuln-rules/:id", auth.RequireRole("owner", "admin"), updateVulnRule) + apiGroup.DELETE("/vuln-rules/:id", auth.RequireRole("owner", "admin"), deleteVulnRule) } } diff --git a/server/internal/api/vulnerabilities.go b/server/internal/api/vulnerabilities.go new file mode 100644 index 0000000..90532e3 --- /dev/null +++ b/server/internal/api/vulnerabilities.go @@ -0,0 +1,304 @@ +package api + +import ( + "errors" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/auth" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/services" + "github.com/gin-gonic/gin" +) + +// vulnGroup is one CVE across every server it affects. +// +// The board groups by CVE rather than listing findings flat: the same CVE on +// forty servers is one decision, and a flat list makes it look like forty. +type vulnGroup struct { + CVEID string `json:"cve_id"` + Severity string `json:"severity"` + Title string `json:"title,omitempty"` + ServerCount int `json:"server_count"` + Findings []models.VulnFinding `json:"findings"` +} + +func listVulnerabilities(c *gin.Context) { + findings, err := services.ListInstanceFindings(auth.InstanceID(c), services.FindingFilter{ + Severity: c.Query("severity"), + State: c.DefaultQuery("state", models.FindingOpen), + ServerID: c.Query("server"), + Tags: tagsFromQuery(c), + }) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, groupByCVE(findings)) +} + +// groupByCVE collapses findings into one row per CVE, most severe first. +func groupByCVE(findings []models.VulnFinding) []vulnGroup { + index := map[string]*vulnGroup{} + order := []string{} + + for _, f := range findings { + g, ok := index[f.CVEID] + if !ok { + g = &vulnGroup{CVEID: f.CVEID, Severity: f.Severity, Title: f.Title} + index[f.CVEID] = g + order = append(order, f.CVEID) + } + // Several servers can disagree on severity when their distributions + // rate the same CVE differently. The highest is shown, because that is + // the one deciding whether anyone acts. + if models.SeverityRank(f.Severity) > models.SeverityRank(g.Severity) { + g.Severity = f.Severity + } + g.Findings = append(g.Findings, f) + } + + out := make([]vulnGroup, 0, len(order)) + for _, id := range order { + g := index[id] + servers := map[string]bool{} + for _, f := range g.Findings { + servers[f.ServerID] = true + } + g.ServerCount = len(servers) + out = append(out, *g) + } + + sort.SliceStable(out, func(i, j int) bool { + ri, rj := models.SeverityRank(out[i].Severity), models.SeverityRank(out[j].Severity) + if ri != rj { + return ri > rj + } + return out[i].ServerCount > out[j].ServerCount + }) + return out +} + +// tagsFromQuery reads repeated tag=key:value parameters. +func tagsFromQuery(c *gin.Context) map[string]string { + out := map[string]string{} + for _, raw := range c.QueryArray("tag") { + k, v, ok := strings.Cut(raw, ":") + if !ok || k == "" { + continue + } + out[k] = v + } + if len(out) == 0 { + return nil + } + return out +} + +func vulnerabilitySummary(c *gin.Context) { + counts, err := services.CountOpenFindingsBySeverity(auth.InstanceID(c)) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + resp := gin.H{"counts": counts} + + // Database freshness travels with the counts rather than living in + // settings: a fleet scanned against a three-week-old database must say so + // wherever its findings are read, not somewhere the reader has to go and + // look for it. + if meta, err := services.GetVulnDBMeta(); err == nil && meta != nil { + resp["db_version"] = meta.DBVersion + resp["pulled_at"] = meta.PulledAt + resp["last_full_scan_at"] = meta.LastFullScanAt + if meta.LastError != "" { + resp["last_error"] = meta.LastError + } + } + + c.JSON(http.StatusOK, resp) +} + +func rescanVulnerabilities(c *gin.Context) { + instanceID := auth.InstanceID(c) + + n, err := services.MarkInstanceForRescan(instanceID) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + services.LogEvent(instanceID, "vuln.rescan", actorFromCtx(c), "", "", + "queued "+strconv.FormatInt(n, 10)+" server(s) for rescan") + c.JSON(http.StatusOK, gin.H{"queued": n}) +} + +type acceptFindingRequest struct { + Reason string `json:"reason"` + Until time.Time `json:"until"` +} + +func acceptFinding(c *gin.Context) { + var req acceptFindingRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body"}) + return + } + + // Both rejected deliberately. An acceptance with no reason is a dismissal + // nobody can audit, and one already expired is a permanent dismissal + // wearing an expiry — the graveyard the expiry exists to prevent. + if strings.TrimSpace(req.Reason) == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "a reason is required"}) + return + } + if req.Until.IsZero() || !req.Until.After(time.Now()) { + c.JSON(http.StatusBadRequest, gin.H{"error": "until must be a future date"}) + return + } + + instanceID := auth.InstanceID(c) + actor := actorFromCtx(c) + + f, err := services.AcceptFinding(instanceID, c.Param("id"), actor, req.Reason, req.Until) + if err != nil { + writeFindingError(c, err) + return + } + + services.LogEvent(instanceID, "vuln.accepted", actor, f.ServerID, "", + f.CVEID+" on "+f.PackageName+" accepted until "+req.Until.Format(time.RFC3339)+": "+req.Reason) + c.JSON(http.StatusOK, f) +} + +func unacceptFinding(c *gin.Context) { + instanceID := auth.InstanceID(c) + actor := actorFromCtx(c) + + f, err := services.UnacceptFinding(instanceID, c.Param("id")) + if err != nil { + writeFindingError(c, err) + return + } + + services.LogEvent(instanceID, "vuln.unaccepted", actor, f.ServerID, "", + f.CVEID+" on "+f.PackageName+" returned to open") + c.JSON(http.StatusOK, f) +} + +func writeFindingError(c *gin.Context, err error) { + if errors.Is(err, services.ErrFindingNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "finding not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) +} + +func listServerVulnerabilities(c *gin.Context) { + findings, err := services.ListFindings(c.Request.Context(), auth.InstanceID(c), c.Param("id")) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if findings == nil { + findings = []models.VulnFinding{} + } + c.JSON(http.StatusOK, findings) +} + +func getServerPackages(c *gin.Context) { + sp, err := services.ListPackages(auth.InstanceID(c), c.Param("id")) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if sp == nil { + // Not a 404: an agent that has not reported yet is the normal state for + // the first hour after install, and is a different thing from a bad + // server id. + c.JSON(http.StatusOK, gin.H{"reported": false}) + return + } + c.JSON(http.StatusOK, sp) +} + +func searchPackages(c *gin.Context) { + name := c.Query("name") + if name == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "name is required"}) + return + } + hits, err := services.SearchPackages(auth.InstanceID(c), name) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, hits) +} + +func listVulnRules(c *gin.Context) { + rules, err := services.ListVulnRules(auth.InstanceID(c)) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, rules) +} + +func createVulnRule(c *gin.Context) { + var r models.VulnAlertRule + if err := c.ShouldBindJSON(&r); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body"}) + return + } + + instanceID := auth.InstanceID(c) + created, err := services.CreateVulnRule(instanceID, &r) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + services.LogEvent(instanceID, "vuln.rule_created", actorFromCtx(c), "", "", "rule "+created.Name) + c.JSON(http.StatusCreated, created) +} + +func updateVulnRule(c *gin.Context) { + var r models.VulnAlertRule + if err := c.ShouldBindJSON(&r); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body"}) + return + } + + instanceID := auth.InstanceID(c) + if err := services.UpdateVulnRule(instanceID, c.Param("id"), &r); err != nil { + if errors.Is(err, services.ErrVulnRuleNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "rule not found"}) + return + } + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + services.LogEvent(instanceID, "vuln.rule_updated", actorFromCtx(c), "", "", "rule "+r.Name) + c.JSON(http.StatusOK, gin.H{"status": "updated"}) +} + +func deleteVulnRule(c *gin.Context) { + instanceID := auth.InstanceID(c) + if err := services.DeleteVulnRule(instanceID, c.Param("id")); err != nil { + if errors.Is(err, services.ErrVulnRuleNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "rule not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + services.LogEvent(instanceID, "vuln.rule_deleted", actorFromCtx(c), "", "", "rule "+c.Param("id")) + c.JSON(http.StatusOK, gin.H{"status": "deleted"}) +} diff --git a/server/internal/notify/dispatch.go b/server/internal/notify/dispatch.go index 39f0c46..bc63971 100644 --- a/server/internal/notify/dispatch.go +++ b/server/internal/notify/dispatch.go @@ -26,7 +26,11 @@ func (e Event) title() string { verb = "is DOWN" } var s string - if e.Type == TypeServer { + if e.Type == TypeVuln { + // A digest is not a transition. MonitorName already carries the whole + // headline ("12 new critical across 4 servers"), so no verb applies. + s = fmt.Sprintf("[Vantage] %s", e.MonitorName) + } else if e.Type == TypeServer { if e.NewStatus == models.StatusDown { verb = "went offline" } else { diff --git a/server/internal/notify/vuln.go b/server/internal/notify/vuln.go new file mode 100644 index 0000000..5e5b953 --- /dev/null +++ b/server/internal/notify/vuln.go @@ -0,0 +1,83 @@ +package notify + +import ( + "fmt" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "gitea.hostxtra.co.uk/mrhid6/vantage/shared/mail" +) + +// TypeVuln marks an event whose subject is a batch of new vulnerability +// findings rather than a state transition. It exists so title() does not +// describe a digest as something going "DOWN". +const TypeVuln = "vulnerability" + +// VulnDigest is one batch of newly opened findings, ready to send. +// +// One per rule per scan, never one per finding: a database refresh can open +// several hundred at once, and a message each would rate-limit the webhook or +// get the channel muted — either way the alerts stop being read. +type VulnDigest struct { + InstanceName string + RuleName string + Summary string + TopSeverity string + Count int + Rows []mail.VulnDigestRow + More int + DBAge string +} + +// DispatchVulnDigest delivers a digest over one channel. +// +// SMTP gets its own template so a vulnerability digest does not arrive dressed +// as a monitor alert. The other four transports carry short text, so they reuse +// the existing Event path rather than growing a second payload shape per +// transport. +func DispatchVulnDigest(ch models.NotificationChannel, d VulnDigest) error { + if ch.Type == models.ChannelSMTP { + to := ch.Config["to"] + sender := mail.Sender{ + Host: ch.Config["host"], + Port: ch.Config["port"], + From: ch.Config["from"], + Username: ch.Config["username"], + Password: ch.Config["password"], + } + if !sender.Enabled() || sender.Port == "" || to == "" { + return fmt.Errorf("smtp: missing host/port/from/to") + } + return sender.SendVulnDigest(to, mail.VulnDigest{ + InstanceName: d.InstanceName, + Count: d.Count, + TopSeverity: d.TopSeverity, + Summary: d.Summary, + Rows: d.Rows, + More: d.More, + DBAge: d.DBAge, + }) + } + + return Dispatch(ch, Event{ + MonitorName: d.Summary, + Type: TypeVuln, + Message: vulnLines(d), + }) +} + +// vulnLines renders the finding list for the text-only transports, capped by +// whatever the caller already put in Rows. +func vulnLines(d VulnDigest) string { + var s string + for _, r := range d.Rows { + fix := "no fix published" + if r.FixedIn != "" { + fix = "fixed in " + r.FixedIn + } + s += fmt.Sprintf("\n• %s (%s) — %s on %s, %s", r.CVEID, r.Severity, r.PackageName, r.ServerName, fix) + } + if d.More > 0 { + s += fmt.Sprintf("\n…and %d more.", d.More) + } + return s +} diff --git a/server/internal/services/findings.go b/server/internal/services/findings.go new file mode 100644 index 0000000..a93ff42 --- /dev/null +++ b/server/internal/services/findings.go @@ -0,0 +1,420 @@ +package services + +import ( + "context" + "errors" + "log" + "time" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/vulndb" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +// FindingDiff is what one server's scan changes. +type FindingDiff struct { + Upserts []models.VulnFinding + FixedIDs []bson.ObjectID + ReopenIDs []bson.ObjectID + // NewlyOpened is what the digest reports: findings that were not open + // before this scan. A finding that was already open must not re-alert every + // tick, or the digest becomes noise and stops being read. + NewlyOpened []models.VulnFinding +} + +func findingKey(cveID, pkg string) string { return cveID + "\x00" + pkg } + +// DiffFindings computes the state changes for one server's scan. +// +// Pure by design: no database, no clock of its own. The ordering below is +// load-bearing — see the comment above the second loop. +func DiffFindings(existing []models.VulnFinding, results []vulndb.Result, now time.Time) FindingDiff { + var d FindingDiff + + byKey := make(map[string]models.VulnFinding, len(existing)) + for _, f := range existing { + byKey[findingKey(f.CVEID, f.PackageName)] = f + } + + seen := make(map[string]bool, len(results)) + for _, r := range results { + key := findingKey(r.CVEID, r.PackageName) + seen[key] = true + + prev, had := byKey[key] + + f := models.VulnFinding{ + CVEID: r.CVEID, + PackageName: r.PackageName, + Installed: r.Installed, + FixedIn: r.FixedIn, + Severity: r.Severity, + State: models.FindingOpen, + FirstSeen: now, + LastSeen: now, + } + + if had { + f.ID = prev.ID + // Preserved, never overwritten: an upsert that moves first_seen + // forward makes every finding look discovered today. + f.FirstSeen = prev.FirstSeen + + // A live acceptance survives the scan untouched: it is suppressed + // from counts and alerts until its expiry, then reopens on its own. + if prev.State == models.FindingAccepted && prev.Accepted != nil { + if now.Before(prev.Accepted.Until) { + continue + } + d.ReopenIDs = append(d.ReopenIDs, prev.ID) + continue + } + + if prev.State != models.FindingOpen { + d.NewlyOpened = append(d.NewlyOpened, f) + } + } else { + d.NewlyOpened = append(d.NewlyOpened, f) + } + + d.Upserts = append(d.Upserts, f) + } + + // Anything we hold that this scan did not produce is fixed. This runs AFTER + // the loop above, and the ordering matters: a finding that is both absent + // and past its acceptance expiry must settle as fixed rather than reopening + // on a package that no longer carries it. + for _, f := range existing { + if seen[findingKey(f.CVEID, f.PackageName)] { + continue + } + if f.State == models.FindingFixed { + continue + } + d.FixedIDs = append(d.FixedIDs, f.ID) + } + + return d +} + +// ErrFindingNotFound is returned for a finding that does not exist in this +// instance. Callers turn it into a 404 — never a 403, which would confirm the +// finding exists in someone else's instance. +var ErrFindingNotFound = errors.New("finding not found") + +// FindingFilter narrows a fleet-wide finding query. An empty field is no +// filter. +type FindingFilter struct { + Severity string + State string + ServerID string + Tags map[string]string +} + +// ListInstanceFindings returns findings across the whole fleet. +func ListInstanceFindings(instanceID string, f FindingFilter) ([]models.VulnFinding, error) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + filter := bson.M{"instance_id": instanceID} + if f.State != "" { + filter["state"] = f.State + } + if f.Severity != "" { + filter["severity"] = f.Severity + } + if f.ServerID != "" { + filter["server_id"] = f.ServerID + } + + // The tag selector resolves through ResolveTargets, the single answer to + // which servers a selector touches. A second matcher here could disagree + // with what a workflow means by env:prod. + if len(f.Tags) > 0 { + servers, err := ResolveTargets(instanceID, nil, f.Tags) + if err != nil { + return nil, err + } + ids := make([]string, 0, len(servers)) + for _, s := range servers { + ids = append(ids, s.ServerID) + } + if len(ids) == 0 { + return []models.VulnFinding{}, nil + } + filter["server_id"] = bson.M{"$in": ids} + } + + cur, err := db.Col("vuln_findings").Find(ctx, filter) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + + out := []models.VulnFinding{} + if err := cur.All(ctx, &out); err != nil { + return nil, err + } + return out, nil +} + +// CountOpenFindingsBySeverity powers the summary tiles. Accepted findings are +// excluded: they are suppressed from counts until their expiry, which is the +// whole point of accepting one. +func CountOpenFindingsBySeverity(instanceID string) (map[string]int, error) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + cur, err := db.Col("vuln_findings").Aggregate(ctx, []bson.M{ + {"$match": bson.M{"instance_id": instanceID, "state": models.FindingOpen}}, + {"$group": bson.M{"_id": "$severity", "n": bson.M{"$sum": 1}}}, + }) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + + var rows []struct { + Severity string `bson:"_id"` + N int `bson:"n"` + } + if err := cur.All(ctx, &rows); err != nil { + return nil, err + } + + counts := map[string]int{} + for _, r := range rows { + counts[r.Severity] = r.N + } + return counts, nil +} + +// MarkInstanceForRescan flags every server in an instance for rescanning and +// returns how many were flagged. +// +// It does not scan. vulnsched picks the flags up on its next tick, which keeps +// matching on the leader and means this endpoint cannot become a second +// scanning path. +func MarkInstanceForRescan(instanceID string) (int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + res, err := db.Col("server_packages").UpdateMany(ctx, + bson.M{"instance_id": instanceID}, + bson.M{"$set": bson.M{"scan_pending": true}}, + ) + if err != nil { + return 0, err + } + return res.ModifiedCount, nil +} + +// AcceptFinding suppresses a finding until a date, with a reason. +// +// The expiry is mandatory at the API layer. A finding reopens on its own when +// it passes, which is what stops the accepted list becoming where risk goes to +// be forgotten. +func AcceptFinding(instanceID, findingID, actor, reason string, until time.Time) (*models.VulnFinding, error) { + id, err := bson.ObjectIDFromHex(findingID) + if err != nil { + return nil, ErrFindingNotFound + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var f models.VulnFinding + err = db.Col("vuln_findings").FindOneAndUpdate(ctx, + bson.M{"_id": id, "instance_id": instanceID}, + bson.M{"$set": bson.M{ + "state": models.FindingAccepted, + "accepted": models.Acceptance{ + By: actor, + Reason: reason, + Until: until, + At: time.Now(), + }, + }}, + options.FindOneAndUpdate().SetReturnDocument(options.After), + ).Decode(&f) + if err == mongo.ErrNoDocuments { + return nil, ErrFindingNotFound + } + if err != nil { + return nil, err + } + return &f, nil +} + +// UnacceptFinding returns an accepted finding to open before its expiry. +func UnacceptFinding(instanceID, findingID string) (*models.VulnFinding, error) { + id, err := bson.ObjectIDFromHex(findingID) + if err != nil { + return nil, ErrFindingNotFound + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var f models.VulnFinding + err = db.Col("vuln_findings").FindOneAndUpdate(ctx, + bson.M{"_id": id, "instance_id": instanceID}, + bson.M{ + "$set": bson.M{"state": models.FindingOpen}, + "$unset": bson.M{"accepted": ""}, + }, + options.FindOneAndUpdate().SetReturnDocument(options.After), + ).Decode(&f) + if err == mongo.ErrNoDocuments { + return nil, ErrFindingNotFound + } + if err != nil { + return nil, err + } + return &f, nil +} + +// ListFindings returns every finding held for one server. +func ListFindings(ctx context.Context, instanceID, serverID string) ([]models.VulnFinding, error) { + cur, err := db.Col("vuln_findings").Find(ctx, bson.M{ + "instance_id": instanceID, + "server_id": serverID, + }) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + + var out []models.VulnFinding + if err := cur.All(ctx, &out); err != nil { + return nil, err + } + return out, nil +} + +// ApplyFindingDiff writes a diff. Thin on purpose — the logic worth reading +// twice is all in DiffFindings. +func ApplyFindingDiff(ctx context.Context, instanceID, serverID string, d FindingDiff, now time.Time) error { + col := db.Col("vuln_findings") + + for _, f := range d.Upserts { + _, err := col.UpdateOne(ctx, + bson.M{ + "instance_id": instanceID, + "server_id": serverID, + "cve_id": f.CVEID, + "package_name": f.PackageName, + }, + bson.M{ + "$set": bson.M{ + "installed_version": f.Installed, + "fixed_in": f.FixedIn, + "severity": f.Severity, + "state": models.FindingOpen, + "last_seen": now, + }, + // first_seen is written only on insert, so a rescan cannot move + // it forward. + "$setOnInsert": bson.M{ + "instance_id": instanceID, + "server_id": serverID, + "cve_id": f.CVEID, + "package_name": f.PackageName, + "first_seen": f.FirstSeen, + }, + "$unset": bson.M{"fixed_at": "", "accepted": ""}, + }, + options.UpdateOne().SetUpsert(true), + ) + if err != nil { + return err + } + } + + if len(d.FixedIDs) > 0 { + if _, err := col.UpdateMany(ctx, + bson.M{"_id": bson.M{"$in": d.FixedIDs}}, + bson.M{"$set": bson.M{"state": models.FindingFixed, "fixed_at": now}}, + ); err != nil { + return err + } + } + + if len(d.ReopenIDs) > 0 { + if _, err := col.UpdateMany(ctx, + bson.M{"_id": bson.M{"$in": d.ReopenIDs}}, + bson.M{"$set": bson.M{"state": models.FindingOpen, "last_seen": now}, "$unset": bson.M{"accepted": ""}}, + ); err != nil { + return err + } + } + + return nil +} + +// StartVulnSweeper deletes old FIXED findings. Open and accepted findings are +// never swept at any setting: retention is about history, and an unresolved +// vulnerability is not history. +func StartVulnSweeper(ctx context.Context) { + go func() { + ticker := time.NewTicker(6 * time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + sweepFixedFindings(ctx) + } + } + }() +} + +// defaultVulnRetentionDays is what an unset setting means. A pointer field and +// this constant together give absent-means-90 and 0-means-forever, the same +// shape as workflow log retention. +const defaultVulnRetentionDays = 90 + +// sweepFixedFindings deletes fixed findings past each instance's retention. +// +// Only "fixed" is ever swept. An open or accepted finding is not history, it is +// an outstanding decision, and deleting one on a timer would quietly shrink the +// fleet's risk picture. +func sweepFixedFindings(ctx context.Context) { + instanceIDs, err := ListInstanceIDs() + if err != nil { + log.Printf("vuln sweeper: list instances: %v", err) + return + } + + for _, instanceID := range instanceIDs { + if ctx.Err() != nil { + return + } + + days := defaultVulnRetentionDays + if s, err := GetSettings(instanceID); err == nil && s != nil && s.VulnFindingRetentionDays != nil { + days = *s.VulnFindingRetentionDays + } + if days <= 0 { + continue // 0 means keep forever + } + + cutoff := time.Now().AddDate(0, 0, -days) + res, err := db.Col("vuln_findings").DeleteMany(ctx, bson.M{ + "instance_id": instanceID, + "state": models.FindingFixed, + "fixed_at": bson.M{"$lt": cutoff}, + }) + if err != nil { + log.Printf("vuln sweeper: delete for %s: %v", instanceID, err) + continue + } + if res.DeletedCount > 0 { + log.Printf("vuln sweeper: removed %d fixed findings for %s", res.DeletedCount, instanceID) + } + } +} \ No newline at end of file diff --git a/server/internal/services/vulnrules.go b/server/internal/services/vulnrules.go new file mode 100644 index 0000000..f6f0e7a --- /dev/null +++ b/server/internal/services/vulnrules.go @@ -0,0 +1,363 @@ +package services + +import ( + "context" + "fmt" + "log" + "sort" + "time" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/notify" + "gitea.hostxtra.co.uk/mrhid6/vantage/shared/mail" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" +) + +// digestRowLimit caps how many findings a single digest lists by name. The +// remainder is summarised as a count: a webhook payload holding six hundred +// rows is not a notification, it is a report nobody reads in a chat client. +const digestRowLimit = 20 + +// ErrVulnRuleNotFound is returned for a rule that does not exist in this +// instance. Callers turn it into a 404. +var ErrVulnRuleNotFound = fmt.Errorf("vulnerability alert rule not found") + +func ListVulnRules(instanceID string) ([]models.VulnAlertRule, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cur, err := db.Col("vuln_alert_rules").Find(ctx, bson.M{"instance_id": instanceID}) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + + rules := []models.VulnAlertRule{} + if err := cur.All(ctx, &rules); err != nil { + return nil, err + } + return rules, nil +} + +func CreateVulnRule(instanceID string, r *models.VulnAlertRule) (*models.VulnAlertRule, error) { + if err := validateVulnRule(instanceID, r); err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + r.ID = bson.NewObjectID() + r.InstanceID = instanceID + r.CreatedAt = time.Now() + r.UpdatedAt = r.CreatedAt + + if _, err := db.Col("vuln_alert_rules").InsertOne(ctx, r); err != nil { + return nil, err + } + return r, nil +} + +func UpdateVulnRule(instanceID, ruleID string, r *models.VulnAlertRule) error { + if err := validateVulnRule(instanceID, r); err != nil { + return err + } + + id, err := bson.ObjectIDFromHex(ruleID) + if err != nil { + return ErrVulnRuleNotFound + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + res, err := db.Col("vuln_alert_rules").UpdateOne(ctx, + bson.M{"_id": id, "instance_id": instanceID}, + bson.M{"$set": bson.M{ + "name": r.Name, + "enabled": r.Enabled, + "min_severity": r.MinSeverity, + "tags": r.Tags, + "channel_ids": r.ChannelIDs, + "updated_at": time.Now(), + }}, + ) + if err != nil { + return err + } + if res.MatchedCount == 0 { + return ErrVulnRuleNotFound + } + return nil +} + +func DeleteVulnRule(instanceID, ruleID string) error { + id, err := bson.ObjectIDFromHex(ruleID) + if err != nil { + return ErrVulnRuleNotFound + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + res, err := db.Col("vuln_alert_rules").DeleteOne(ctx, bson.M{"_id": id, "instance_id": instanceID}) + if err != nil { + return err + } + if res.DeletedCount == 0 { + return ErrVulnRuleNotFound + } + return nil +} + +// validateVulnRule rejects a rule that could never fire, and one naming a +// channel from another instance. The channel check reuses validateChannelIDs so +// there is one answer to "is this channel mine". +func validateVulnRule(instanceID string, r *models.VulnAlertRule) error { + if r.Name == "" { + return fmt.Errorf("name is required") + } + switch r.MinSeverity { + case models.SeverityUnknown, models.SeverityLow, models.SeverityMedium, + models.SeverityHigh, models.SeverityCritical: + default: + return fmt.Errorf("min_severity %q is not a severity", r.MinSeverity) + } + if len(r.ChannelIDs) == 0 { + return fmt.Errorf("at least one channel is required") + } + return validateChannelIDs(instanceID, r.ChannelIDs) +} + +// SendVulnDigest delivers one message per rule per tick — never one per +// finding. See vulnsched for why the tick is the batch boundary. +func SendVulnDigest(instanceID string, newly []models.VulnFinding) { + rules, err := ListVulnRules(instanceID) + if err != nil { + log.Printf("vuln digest: list rules: %v", err) + return + } + + for _, rule := range rules { + if !rule.Enabled { + continue + } + + matched := filterBySeverity(newly, rule.MinSeverity) + if len(matched) == 0 { + continue + } + + if len(rule.Tags) > 0 { + // ResolveTargets is already the single answer to which servers a + // selector touches. A rule that disagreed with a workflow about + // what env:prod means would be worse than no filter at all. + allowed, err := ResolveTargets(instanceID, nil, rule.Tags) + if err != nil { + log.Printf("vuln digest: resolve targets: %v", err) + continue + } + matched = filterByServers(matched, allowed) + if len(matched) == 0 { + continue + } + } + + dispatchVulnDigest(instanceID, rule, matched) + } +} + +func filterBySeverity(findings []models.VulnFinding, min string) []models.VulnFinding { + floor := models.SeverityRank(min) + out := make([]models.VulnFinding, 0, len(findings)) + for _, f := range findings { + if models.SeverityRank(f.Severity) >= floor { + out = append(out, f) + } + } + return out +} + +// filterByServers keeps findings on servers the rule's tag selector matched. +// ResolveTargets answers in whole server documents, so the IDs are lifted here. +func filterByServers(findings []models.VulnFinding, allowed []models.Server) []models.VulnFinding { + set := make(map[string]bool, len(allowed)) + for _, s := range allowed { + set[s.ServerID] = true + } + out := make([]models.VulnFinding, 0, len(findings)) + for _, f := range findings { + if set[f.ServerID] { + out = append(out, f) + } + } + return out +} + +// dispatchVulnDigest builds one digest and sends it over each of the rule's +// channels. +func dispatchVulnDigest(instanceID string, rule models.VulnAlertRule, findings []models.VulnFinding) { + channels, err := GetChannels(instanceID, rule.ChannelIDs) + if err != nil { + log.Printf("vuln digest: load channels for rule %s: %v", rule.Name, err) + return + } + + digest := buildVulnDigest(instanceID, rule, findings) + + for _, ch := range channels { + if !ch.Enabled { + continue + } + go func(c models.NotificationChannel) { + if err := notify.DispatchVulnDigest(c, digest); err != nil { + log.Printf("vuln digest: dispatch to %s (%s): %v", c.Name, c.Type, err) + } + }(ch) + } +} + +// buildVulnDigest turns a batch of findings into one message. +// +// Findings are ordered most severe first so the capped list shows the ones that +// matter rather than whichever the scan happened to produce first. +func buildVulnDigest(instanceID string, rule models.VulnAlertRule, findings []models.VulnFinding) notify.VulnDigest { + sorted := make([]models.VulnFinding, len(findings)) + copy(sorted, findings) + sort.SliceStable(sorted, func(i, j int) bool { + return models.SeverityRank(sorted[i].Severity) > models.SeverityRank(sorted[j].Severity) + }) + + counts := map[string]int{} + servers := map[string]bool{} + for _, f := range sorted { + counts[f.Severity]++ + servers[f.ServerID] = true + } + + names := serverNames(instanceID) + + shown := sorted + more := 0 + if len(shown) > digestRowLimit { + more = len(shown) - digestRowLimit + shown = shown[:digestRowLimit] + } + + rows := make([]mail.VulnDigestRow, 0, len(shown)) + for _, f := range shown { + name := names[f.ServerID] + if name == "" { + name = f.ServerID + } + rows = append(rows, mail.VulnDigestRow{ + CVEID: f.CVEID, + Severity: f.Severity, + PackageName: f.PackageName, + ServerName: name, + FixedIn: f.FixedIn, + }) + } + + top := models.SeverityUnknown + if len(sorted) > 0 { + top = sorted[0].Severity + } + + instanceName := instanceID + if inst, err := GetInstance(instanceID); err == nil && inst != nil && inst.Name != "" { + instanceName = inst.Name + } + + return notify.VulnDigest{ + InstanceName: instanceName, + RuleName: rule.Name, + Summary: summariseCounts(counts, len(servers)), + TopSeverity: top, + Count: len(sorted), + Rows: rows, + More: more, + DBAge: vulnDBAge(), + } +} + +// summariseCounts renders "12 new critical, 4 new high across 6 servers". +func summariseCounts(counts map[string]int, serverCount int) string { + order := []string{ + models.SeverityCritical, models.SeverityHigh, + models.SeverityMedium, models.SeverityLow, models.SeverityUnknown, + } + parts := "" + for _, sev := range order { + if counts[sev] == 0 { + continue + } + if parts != "" { + parts += ", " + } + parts += fmt.Sprintf("%d new %s", counts[sev], sev) + } + if parts == "" { + parts = "new findings" + } + plural := "servers" + if serverCount == 1 { + plural = "server" + } + return fmt.Sprintf("%s across %d %s", parts, serverCount, plural) +} + +// serverNames maps server IDs to display names for one instance. A digest that +// named raw UUIDs would be unreadable in a chat client. +func serverNames(instanceID string) map[string]string { + out := map[string]string{} + servers, err := ListServers(instanceID) + if err != nil { + log.Printf("vuln digest: list servers: %v", err) + return out + } + for _, s := range servers { + out[s.ServerID] = s.Hostname + } + return out +} + +// vulnDBAge renders how long ago the vulnerability database was pulled. +// +// It is on every digest deliberately: a fleet scanned against a three-week-old +// database must say so rather than let the reader assume freshness. +func vulnDBAge() string { + meta, err := GetVulnDBMeta() + if err != nil || meta == nil || meta.PulledAt.IsZero() { + return "an unknown time" + } + d := time.Since(meta.PulledAt) + switch { + case d < time.Hour: + return fmt.Sprintf("%d minutes", int(d.Minutes())) + case d < 48*time.Hour: + return fmt.Sprintf("%d hours", int(d.Hours())) + default: + return fmt.Sprintf("%d days", int(d.Hours()/24)) + } +} + +// GetVulnDBMeta reads the deployment-wide vulnerability database metadata. +// It carries no instance_id: the database is a property of the deployment, not +// of a tenant. +func GetVulnDBMeta() (*models.VulnDBMeta, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var meta models.VulnDBMeta + err := db.Col("vulndb_meta").FindOne(ctx, bson.M{}).Decode(&meta) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &meta, nil +} diff --git a/server/internal/vulndb/db.go b/server/internal/vulndb/db.go new file mode 100644 index 0000000..dc1dfa7 --- /dev/null +++ b/server/internal/vulndb/db.go @@ -0,0 +1,129 @@ +package vulndb + +import ( + "strings" + + trivydb "github.com/aquasecurity/trivy-db/pkg/db" + trivytypes "github.com/aquasecurity/trivy-db/pkg/types" +) + +// Advisory is one fixed-version statement for one source package. +type Advisory struct { + CVEID string + // FixedVersion empty means no vendor fix has been published. It is a real + // state, not an absence of data, and callers must treat it as vulnerable. + FixedVersion string + Severity string +} + +// VulnInfo is the CVE's own metadata, shared across every server it affects. +type VulnInfo struct { + Title string + Severity string + CVSSScore float64 + References []string +} + +// Store reads a pulled trivy-db. +type Store struct { + cfg trivydb.Config +} + +// Open opens the database in dir. trivy-db expects the directory, not the file: +// it appends "trivy.db" itself. +func Open(dir string) (*Store, error) { + if err := trivydb.Init(dir); err != nil { + return nil, err + } + return &Store{cfg: trivydb.Config{}}, nil +} + +func (s *Store) Close() error { return trivydb.Close() } + +// Advisories returns every advisory for a source package in a bucket. +func (s *Store) Advisories(bucket, srcName string) ([]Advisory, error) { + raw, err := s.cfg.GetAdvisories(bucket, srcName) + if err != nil { + return nil, err + } + out := make([]Advisory, 0, len(raw)) + for _, a := range raw { + out = append(out, Advisory{ + CVEID: a.VulnerabilityID, + FixedVersion: a.FixedVersion, + // Advisory.Severity is trivy's numeric Severity type, unlike + // Vulnerability.Severity which is a string. They are genuinely + // different types in trivy-db, not an inconsistency here. + Severity: severityFromLevel(a.Severity), + }) + } + return out, nil +} + +// Vulnerability returns a CVE's shared metadata. +func (s *Store) Vulnerability(cveID string) (VulnInfo, error) { + v, err := s.cfg.GetVulnerability(cveID) + if err != nil { + return VulnInfo{}, err + } + return VulnInfo{ + Title: v.Title, + Severity: resolveSeverity(v), + CVSSScore: topCVSS(v), + References: v.References, + }, nil +} + +// resolveSeverity picks a CVE's severity: vendor, then NVD, then unknown. +// +// Never invented. This will surface as "why is this critical CVE marked low": +// Debian and Red Hat routinely downgrade an NVD score because the vulnerable +// path is not reachable in their build, and their rating is the accurate one +// for that package. Where several vendors disagree the highest wins, because +// under-reporting a vulnerability is the worse mistake. +func resolveSeverity(v trivytypes.Vulnerability) string { + best := 0 + for _, sev := range v.VendorSeverity { + if int(sev) > best { + best = int(sev) + } + } + if best > 0 { + return severityFromLevel(trivytypes.Severity(best)) + } + + // Vulnerability.Severity is the deprecated NVD-derived string. Used only as + // the fallback, which is exactly what it is good for. + if s := strings.ToLower(strings.TrimSpace(v.Severity)); s != "" && s != "unknown" { + return s + } + return "unknown" +} + +// topCVSS returns the highest V3 score any source published, or 0. +func topCVSS(v trivytypes.Vulnerability) float64 { + var top float64 + for _, c := range v.CVSS { + if c.V3Score > top { + top = c.V3Score + } + } + return top +} + +// severityFromLevel maps trivy-db's numeric severity onto our lowercase +// strings. The names are fixed by models.Severity* and must stay in step. +func severityFromLevel(n trivytypes.Severity) string { + switch int(n) { + case 4: + return "critical" + case 3: + return "high" + case 2: + return "medium" + case 1: + return "low" + default: + return "unknown" + } +} diff --git a/server/internal/vulndb/match.go b/server/internal/vulndb/match.go new file mode 100644 index 0000000..32d317d --- /dev/null +++ b/server/internal/vulndb/match.go @@ -0,0 +1,81 @@ +package vulndb + +import ( + "fmt" + "log" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" +) + +// AdvisorySource is the advisory lookup the matcher needs. *Store satisfies it. +// The seam keeps the matching logic independent of how the database is opened. +type AdvisorySource interface { + Advisories(bucket, srcName string) ([]Advisory, error) +} + +// Result is one vulnerable package on one server, before it becomes a finding. +type Result struct { + CVEID string + PackageName string // the BINARY package, which is what is installed + Installed string + FixedIn string + Severity string +} + +// Match returns every advisory that the installed packages do not satisfy. +// +// Vulnerable means: no fix has been published, or the installed version sorts +// strictly before the fixed version under the distribution's own ordering. +// Equal is NOT vulnerable — that is the backported-fix case, where a +// distribution patches in place without changing the upstream version, and +// treating it as vulnerable reports a patched fleet as exposed. +func Match(src AdvisorySource, os models.OSRelease, pkgs []models.InstalledPackage) ([]Result, error) { + bucket, err := Bucket(os.Family, os.VersionID) + if err != nil { + return nil, err + } + + var out []Result + for _, p := range pkgs { + // Debian and Ubuntu advisories are keyed on the source package: one + // advisory against "openssl" covers libssl3, openssl and libssl-dev. + srcName := p.SourceName + if srcName == "" { + srcName = p.Name + } + + advs, err := src.Advisories(bucket, srcName) + if err != nil { + return nil, fmt.Errorf("advisories for %s: %w", srcName, err) + } + + for _, a := range advs { + // No published fix. Vulnerable, and the finding most in need of + // acceptance, since there is nothing to patch. + if a.FixedVersion == "" { + out = append(out, Result{ + CVEID: a.CVEID, PackageName: p.Name, + Installed: p.Version, Severity: a.Severity, + }) + continue + } + + older, err := LessThan(os.Family, p.Version, a.FixedVersion) + if err != nil { + // Skip this one advisory rather than failing the whole server: + // one unparseable version must not blind us to every other CVE + // on the host. Log it — a silent skip is a silent false + // negative, which is the direction that hurts. + log.Printf("vulndb: compare %s %s vs %s: %v", p.Name, p.Version, a.FixedVersion, err) + continue + } + if older { + out = append(out, Result{ + CVEID: a.CVEID, PackageName: p.Name, + Installed: p.Version, FixedIn: a.FixedVersion, Severity: a.Severity, + }) + } + } + } + return out, nil +} diff --git a/server/internal/vulndb/pull.go b/server/internal/vulndb/pull.go new file mode 100644 index 0000000..39b1e78 --- /dev/null +++ b/server/internal/vulndb/pull.go @@ -0,0 +1,186 @@ +package vulndb + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/registry" + "oras.land/oras-go/v2/registry/remote" +) + +// DefaultRef is the published trivy-db OCI artifact, rebuilt every six hours. +const DefaultRef = "ghcr.io/aquasecurity/trivy-db:2" + +// SupportedSchema is the trivy-db schema version this code understands. +// +// A different version is refused rather than parsed on the assumption it is +// close enough. Mis-reading the schema would not fail loudly — it would return +// no advisories, which is indistinguishable from a clean fleet. +const SupportedSchema = 2 + +// dbFileName and metaFileName are the two files inside the artifact layer. +const ( + dbFileName = "trivy.db" + metaFileName = "metadata.json" +) + +// Ref returns the artifact reference, honouring VANTAGE_TRIVY_DB_REF so an +// air-gapped deployment can mirror the artifact into its own registry, and so +// a busy deployment can avoid the anonymous ghcr rate limit. +func Ref() string { + if v := os.Getenv("VANTAGE_TRIVY_DB_REF"); v != "" { + return v + } + return DefaultRef +} + +// Disabled reports whether the puller and scheduler are switched off entirely. +// Findings already written are still served, and still marked stale. +func Disabled() bool { + return strings.EqualFold(os.Getenv("VANTAGE_VULNDB_DISABLED"), "true") +} + +// dbMetadata is the subset of trivy-db's metadata.json we read. +type dbMetadata struct { + Version int `json:"Version"` +} + +// Pull fetches the trivy-db artifact into dir and returns its schema version. +// +// It extracts into a staging directory and only moves the files into place once +// both are present and the schema has been accepted. A pull that fails partway +// therefore leaves the previous database untouched rather than a half-written +// one that Open would happily accept and scan against. +func Pull(ctx context.Context, dir string) (int, error) { + ref := Ref() + + parsed, err := registry.ParseReference(ref) + if err != nil { + return 0, fmt.Errorf("parse reference %q: %w", ref, err) + } + + repo, err := remote.NewRepository(ref) + if err != nil { + return 0, fmt.Errorf("open repository %q: %w", ref, err) + } + + // The tag or digest half of the reference; the repository already knows the + // registry and path. + target := parsed.Reference + if target == "" { + target = "latest" + } + + _, manifestBytes, err := oras.FetchBytes(ctx, repo, target, oras.DefaultFetchBytesOptions) + if err != nil { + return 0, fmt.Errorf("fetch manifest %s: %w", ref, err) + } + + var man ocispec.Manifest + if err := json.Unmarshal(manifestBytes, &man); err != nil { + return 0, fmt.Errorf("decode manifest %s: %w", ref, err) + } + if len(man.Layers) == 0 { + return 0, fmt.Errorf("artifact %s has no layers", ref) + } + + // Streamed rather than buffered: the layer is ~50MB and there is no reason + // to hold it in memory on the way to disk. + rc, err := repo.Blobs().Fetch(ctx, man.Layers[0]) + if err != nil { + return 0, fmt.Errorf("fetch layer: %w", err) + } + defer rc.Close() + + staging, err := os.MkdirTemp(dir, ".staging-") + if err != nil { + return 0, fmt.Errorf("staging dir: %w", err) + } + defer os.RemoveAll(staging) + + if err := extractTarGz(rc, staging); err != nil { + return 0, fmt.Errorf("extract layer: %w", err) + } + + metaBytes, err := os.ReadFile(filepath.Join(staging, metaFileName)) + if err != nil { + return 0, fmt.Errorf("read %s: %w", metaFileName, err) + } + var meta dbMetadata + if err := json.Unmarshal(metaBytes, &meta); err != nil { + return 0, fmt.Errorf("decode %s: %w", metaFileName, err) + } + if meta.Version != SupportedSchema { + return 0, fmt.Errorf("trivy-db schema %d is not supported (want %d)", meta.Version, SupportedSchema) + } + + if _, err := os.Stat(filepath.Join(staging, dbFileName)); err != nil { + return 0, fmt.Errorf("artifact has no %s: %w", dbFileName, err) + } + + // Both files present and the schema accepted, so it is safe to replace. + for _, name := range []string{dbFileName, metaFileName} { + src := filepath.Join(staging, name) + dst := filepath.Join(dir, name) + if err := os.Remove(dst); err != nil && !os.IsNotExist(err) { + return 0, fmt.Errorf("remove old %s: %w", name, err) + } + if err := os.Rename(src, dst); err != nil { + return 0, fmt.Errorf("install %s: %w", name, err) + } + } + + return meta.Version, nil +} + +// extractTarGz writes the artifact layer into dir. Paths are flattened and +// checked so a crafted archive cannot write outside dir. +func extractTarGz(r io.Reader, dir string) error { + gz, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gz.Close() + + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if hdr.Typeflag != tar.TypeReg { + continue + } + name := filepath.Base(hdr.Name) // flatten; the archive is two files + if name == "." || name == ".." || name == "" { + continue + } + dst := filepath.Join(dir, name) + if !strings.HasPrefix(dst, filepath.Clean(dir)+string(os.PathSeparator)) { + return fmt.Errorf("archive entry escapes destination: %q", hdr.Name) + } + f, err := os.Create(dst) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + } +} diff --git a/server/internal/vulnsched/sched.go b/server/internal/vulnsched/sched.go new file mode 100644 index 0000000..b1d2eeb --- /dev/null +++ b/server/internal/vulnsched/sched.go @@ -0,0 +1,238 @@ +// Package vulnsched owns the vulnerability scan loop. +// +// It runs inside bus.RunAsLeader("housekeeping", …) alongside monitorsched, +// workflowsched and the sweepers: one role, one lock. N replicas each running +// this loop would mean N copies of the ~50MB database resident, N rescans of +// the same fleet on every database refresh, and N digests reaching the +// customer for one set of findings. +package vulnsched + +import ( + "context" + "errors" + "log" + "os" + "time" + + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/db" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/models" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/services" + "gitea.hostxtra.co.uk/mrhid6/vantage/server/internal/vulndb" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +const ( + tickInterval = 60 * time.Second + // trivy-db is rebuilt every six hours; pulling more often buys nothing. + dbMaxAge = 6 * time.Hour +) + +// Deps are injected from main.go rather than imported, following +// workflowsched. It keeps this package's reach explicit and reviewable. +type Deps struct { + LogEvent func(instanceID, eventType, actor, serverID, keyID, details string) + SendDigest func(instanceID string, newly []models.VulnFinding) +} + +type scheduler struct { + deps Deps + dir string + store *vulndb.Store + version int + pulled time.Time +} + +func Start(ctx context.Context, deps Deps) { + if vulndb.Disabled() { + log.Println("vulnsched: disabled by VANTAGE_VULNDB_DISABLED") + return + } + + dir, err := os.MkdirTemp("", "vantage-vulndb-") + if err != nil { + log.Printf("vulnsched: temp dir: %v", err) + return + } + + s := &scheduler{deps: deps, dir: dir} + + go func() { + defer os.RemoveAll(dir) + defer s.closeStore() + + ticker := time.NewTicker(tickInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.tick(ctx) + } + } + }() +} + +func (s *scheduler) tick(ctx context.Context) { + if err := s.ensureDB(ctx); err != nil { + // Keep the last good database and carry on scanning against it. A + // network blip must never clear findings or read as "all fixed". + log.Printf("vulnsched: database unavailable: %v", err) + s.recordDBError(ctx, err) + if s.store == nil { + return + } + } + s.scanPending(ctx) +} + +// ensureDB pulls a fresh database when the local copy is stale, and marks the +// whole fleet for rescanning when the version changes — which is what makes a +// newly published CVE flag existing servers within a minute rather than at the +// next agent report. +func (s *scheduler) ensureDB(ctx context.Context) error { + if s.store != nil && time.Since(s.pulled) < dbMaxAge { + return nil + } + + version, err := vulndb.Pull(ctx, s.dir) + if err != nil { + return err + } + + s.closeStore() + store, err := vulndb.Open(s.dir) + if err != nil { + return err + } + s.store = store + s.pulled = time.Now() + + changed := version != s.version + s.version = version + + _, _ = db.Col("vulndb_meta").UpdateOne(ctx, bson.M{}, + bson.M{"$set": bson.M{"db_version": version, "pulled_at": s.pulled}, "$unset": bson.M{"last_error": ""}}, + options.UpdateOne().SetUpsert(true), + ) + + if changed { + res, err := db.Col("server_packages").UpdateMany(ctx, + bson.M{"status": bson.M{"$ne": models.ScanStatusUnsupported}}, + bson.M{"$set": bson.M{"scan_pending": true}}, + ) + if err != nil { + log.Printf("vulnsched: mark fleet pending: %v", err) + } else { + log.Printf("vulnsched: database version %d, %d servers marked for rescan", version, res.ModifiedCount) + } + } + return nil +} + +func (s *scheduler) recordDBError(ctx context.Context, err error) { + _, _ = db.Col("vulndb_meta").UpdateOne(ctx, bson.M{}, + bson.M{"$set": bson.M{"last_error": err.Error()}}, + options.UpdateOne().SetUpsert(true), + ) +} + +func (s *scheduler) scanPending(ctx context.Context) { + cur, err := db.Col("server_packages").Find(ctx, bson.M{"scan_pending": true}) + if err != nil { + log.Printf("vulnsched: find pending: %v", err) + return + } + defer cur.Close(ctx) + + var pending []models.ServerPackages + if err := cur.All(ctx, &pending); err != nil { + log.Printf("vulnsched: decode pending: %v", err) + return + } + + // Newly opened findings are collected across the whole tick and sent as one + // digest per instance. A database refresh can open several hundred findings + // at once; one message per finding would rate-limit the webhook or get the + // channel muted, and either way the alerts stop being read. + newly := map[string][]models.VulnFinding{} + + for _, sp := range pending { + if ctx.Err() != nil { + // Leadership lost. scan_pending is still set, so the next leader + // picks these up — which is why it lives on the document. + return + } + opened := s.scanOne(ctx, sp) + newly[sp.InstanceID] = append(newly[sp.InstanceID], opened...) + } + + for instanceID, findings := range newly { + if len(findings) > 0 && s.deps.SendDigest != nil { + s.deps.SendDigest(instanceID, findings) + } + } + + _, _ = db.Col("vulndb_meta").UpdateOne(ctx, bson.M{}, + bson.M{"$set": bson.M{"last_full_scan_at": time.Now()}}, + options.UpdateOne().SetUpsert(true), + ) +} + +func (s *scheduler) scanOne(ctx context.Context, sp models.ServerPackages) []models.VulnFinding { + now := time.Now() + + results, err := vulndb.Match(s.store, sp.OS, sp.Packages) + if err != nil { + // We hold no feed for this distribution, so we cannot answer whether it + // is vulnerable. Say "unsupported" — reporting zero findings here would + // be indistinguishable from reporting a clean host, and one of those is + // a lie. + status := models.ScanStatusUnsupported + if !errors.Is(err, vulndb.ErrUnsupportedFamily) { + log.Printf("vulnsched: scan %s: %v", sp.ServerID, err) + status = sp.Status + } + s.clearPending(ctx, sp.ID, status, now) + return nil + } + + existing, err := services.ListFindings(ctx, sp.InstanceID, sp.ServerID) + if err != nil { + log.Printf("vulnsched: list findings %s: %v", sp.ServerID, err) + return nil + } + + diff := services.DiffFindings(existing, results, now) + if err := services.ApplyFindingDiff(ctx, sp.InstanceID, sp.ServerID, diff, now); err != nil { + log.Printf("vulnsched: apply diff %s: %v", sp.ServerID, err) + return nil + } + + s.clearPending(ctx, sp.ID, models.ScanStatusOK, now) + + for i := range diff.NewlyOpened { + diff.NewlyOpened[i].ServerID = sp.ServerID + } + return diff.NewlyOpened +} + +func (s *scheduler) clearPending(ctx context.Context, id bson.ObjectID, status string, now time.Time) { + _, _ = db.Col("server_packages").UpdateOne(ctx, + bson.M{"_id": id}, + bson.M{"$set": bson.M{ + "scan_pending": false, + "status": status, + "scanned_at": now, + "db_version": s.version, + }}, + ) +} + +func (s *scheduler) closeStore() { + if s.store != nil { + _ = s.store.Close() + s.store = nil + } +} diff --git a/shared/mail/templates/vuln_digest.html.tmpl b/shared/mail/templates/vuln_digest.html.tmpl new file mode 100644 index 0000000..9d2679e --- /dev/null +++ b/shared/mail/templates/vuln_digest.html.tmpl @@ -0,0 +1,13 @@ +{{define "title"}}New vulnerabilities detected{{end}} +{{define "pill"}}{{template "chip" (dict "label" (upper .TopSeverity) "tone" "down")}}{{end}} +{{define "body"}} +{{template "lead" .Summary}} +{{template "rows" (list + (dict "k" "Instance" "v" .InstanceName) + (dict "k" "New findings" "v" .Count))}} +{{range .Rows}} +{{if .FixedIn}}{{template "well" (printf "%s (%s) — %s on %s, fixed in %s" .CVEID .Severity .PackageName .ServerName .FixedIn)}}{{else}}{{template "well" (printf "%s (%s) — %s on %s, no fix published" .CVEID .Severity .PackageName .ServerName)}}{{end}} +{{end}} +{{if .More}}{{template "p" (printf "…and %d more." .More)}}{{end}} +{{template "note" (printf "Scanned against a vulnerability database pulled %s ago." .DBAge)}} +{{end}} diff --git a/shared/mail/templates/vuln_digest.txt.tmpl b/shared/mail/templates/vuln_digest.txt.tmpl new file mode 100644 index 0000000..086c803 --- /dev/null +++ b/shared/mail/templates/vuln_digest.txt.tmpl @@ -0,0 +1,12 @@ +{{define "subject"}}{{.Count}} new {{if eq .Count 1}}vulnerability{{else}}vulnerabilities{{end}} on {{.InstanceName}}{{end}} +{{define "title"}}New vulnerabilities detected{{end}} +{{define "pill"}}{{.TopSeverity}}{{end}} +{{define "body"}} +{{template "lead" .Summary}} + +{{range .Rows}}- {{.CVEID}} ({{.Severity}}) — {{.PackageName}} on {{.ServerName}}{{if .FixedIn}}, fixed in {{.FixedIn}}{{else}}, no fix published{{end}} +{{end}} +{{if .More}}...and {{.More}} more.{{end}} + +Scanned against vulnerability database pulled {{.DBAge}} ago. +{{end}} \ No newline at end of file diff --git a/shared/mail/vuln.go b/shared/mail/vuln.go new file mode 100644 index 0000000..a82215d --- /dev/null +++ b/shared/mail/vuln.go @@ -0,0 +1,41 @@ +package mail + +// VulnDigestRow is one newly opened finding as the digest shows it. +// +// It lives here rather than in server/ so the templates and the caller agree on +// the fields without server's model package leaking into shared. +type VulnDigestRow struct { + CVEID string + Severity string + PackageName string + ServerName string + // FixedIn empty means no vendor fix has been published, which the template + // says explicitly rather than leaving blank — it is a real state, not + // missing data. + FixedIn string +} + +// VulnDigest is one batch of newly opened findings. +// +// One message per rule per scan, never one per finding: a database refresh can +// open several hundred at once, and one message each would rate-limit the +// webhook or get the channel muted. +type VulnDigest struct { + InstanceName string + // Count is every newly opened finding in the batch, which may exceed + // len(Rows) — Rows is capped and More carries the remainder. + Count int + TopSeverity string + Summary string + Rows []VulnDigestRow + More int + // DBAge is pre-formatted by the caller. A digest scanned against a + // three-week-old database must say so rather than quietly imply freshness. + DBAge string +} + +// SendVulnDigest delivers one digest to an SMTP notification channel's +// recipients, which may be a comma-separated list. +func (s Sender) SendVulnDigest(to string, d VulnDigest) error { + return s.sendTemplate(to, "", "vuln_digest", d) +} diff --git a/shared/models/settings.go b/shared/models/settings.go index 9ea06b2..4b2dc75 100644 --- a/shared/models/settings.go +++ b/shared/models/settings.go @@ -34,6 +34,11 @@ type Settings struct { // absent as disabled — turning off password login for the entire fleet at // upgrade. Nil means enabled. LocalLoginEnabled *bool `bson:"local_login_enabled,omitempty" json:"local_login_enabled,omitempty"` + + // VulnFindingRetentionDays is a pointer for the same reason + // WorkflowLogRetentionDays is: absent must mean the default, not zero. + // Nil is 90 days, 0 is forever. Only "fixed" findings are ever swept. + VulnFindingRetentionDays *int `bson:"vuln_finding_retention_days,omitempty" json:"vuln_finding_retention_days,omitempty"` } // LocalLoginEnabled reads the setting with its absent-means-on default. Every