diff --git a/server/internal/services/findings.go b/server/internal/services/findings.go index be0213d..078e0af 100644 --- a/server/internal/services/findings.go +++ b/server/internal/services/findings.go @@ -303,15 +303,33 @@ func ListFindings(ctx context.Context, instanceID, serverID string) ([]models.Vu func ApplyFindingDiff(ctx context.Context, instanceID, serverID string, d FindingDiff, now time.Time) error { col := db.Col("vuln_findings") + // One BulkWrite per batch, not one UpdateOne per finding. A freshly scanned + // Ubuntu host opens tens of thousands of findings, and at one round trip each + // that is minutes of sequential latency during which the tick holds the + // leader and every other server waits its turn. Unordered, because the + // upserts are independent and one duplicate-key race must not abandon the + // rest of the batch. + const bulkBatch = 1000 + ops := make([]mongo.WriteModel, 0, bulkBatch) + + flush := func() error { + if len(ops) == 0 { + return nil + } + _, err := col.BulkWrite(ctx, ops, options.BulkWrite().SetOrdered(false)) + ops = ops[:0] + return err + } + for _, f := range d.Upserts { - _, err := col.UpdateOne(ctx, - bson.M{ + ops = append(ops, mongo.NewUpdateOneModel(). + SetFilter(bson.M{ "instance_id": instanceID, "server_id": serverID, "cve_id": f.CVEID, "package_name": f.PackageName, - }, - bson.M{ + }). + SetUpdate(bson.M{ "$set": bson.M{ "installed_version": f.Installed, "fixed_in": f.FixedIn, @@ -329,13 +347,18 @@ func ApplyFindingDiff(ctx context.Context, instanceID, serverID string, d Findin "first_seen": f.FirstSeen, }, "$unset": bson.M{"fixed_at": "", "accepted": ""}, - }, - options.UpdateOne().SetUpsert(true), - ) - if err != nil { - return err + }). + SetUpsert(true)) + + if len(ops) >= bulkBatch { + if err := flush(); err != nil { + return err + } } } + if err := flush(); err != nil { + return err + } if len(d.FixedIDs) > 0 { if _, err := col.UpdateMany(ctx, diff --git a/server/internal/vulndb/db.go b/server/internal/vulndb/db.go index b3c3fd4..44b1684 100644 --- a/server/internal/vulndb/db.go +++ b/server/internal/vulndb/db.go @@ -15,6 +15,11 @@ type Advisory struct { // state, not an absence of data, and callers must treat it as vulnerable. FixedVersion string Severity string + // Status is filled by trivy-db ONLY when FixedVersion is empty — when there + // is a fix, "fixed" is the obvious state and the field is left zero. It is + // what separates "the vendor confirms this package is affected and has not + // fixed it" from "nobody has looked yet". + Status string } // VulnInfo is the CVE's own metadata, shared across every server it affects. @@ -58,6 +63,7 @@ func (s *Store) Advisories(bucket, srcName string) ([]Advisory, error) { // Vulnerability.Severity which is a string. They are genuinely // different types in trivy-db, not an inconsistency here. Severity: severityFromLevel(a.Severity), + Status: a.Status.String(), }) } return out, nil diff --git a/server/internal/vulndb/match.go b/server/internal/vulndb/match.go index 125dcb6..76278f8 100644 --- a/server/internal/vulndb/match.go +++ b/server/internal/vulndb/match.go @@ -37,7 +37,15 @@ func Match(src AdvisorySource, os models.OSRelease, pkgs []models.InstalledPacka } log.Printf("vulndb: matching %d packages against bucket %q", len(pkgs), bucket) - var advisoryCount, skipped int + var advisoryCount, skipped, unactionable, noFix int + + // Advisories are keyed on the SOURCE package, and several hundred binary + // packages on a host resolve to the same few hundred sources — linux-modules, + // linux-image and linux-headers all ask about "linux", whose advisory list is + // thousands long. Without this the same bolt read is repeated once per binary + // package, which is most of what made a single Ubuntu host take minutes. + cache := make(map[string][]Advisory, len(pkgs)) + var out []Result for _, p := range pkgs { // Debian and Ubuntu advisories are keyed on the source package: one @@ -47,9 +55,14 @@ func Match(src AdvisorySource, os models.OSRelease, pkgs []models.InstalledPacka srcName = p.Name } - advs, err := src.Advisories(bucket, srcName) - if err != nil { - return nil, fmt.Errorf("advisories for %s: %w", srcName, err) + advs, cached := cache[srcName] + if !cached { + var err error + advs, err = src.Advisories(bucket, srcName) + if err != nil { + return nil, fmt.Errorf("advisories for %s: %w", srcName, err) + } + cache[srcName] = advs } advisoryCount += len(advs) if len(advs) > 0 { @@ -57,9 +70,15 @@ func Match(src AdvisorySource, os models.OSRelease, pkgs []models.InstalledPacka } for _, a := range advs { - // No published fix. Vulnerable, and the finding most in need of - // acceptance, since there is nothing to patch. + // No published fix. Whether that is a finding depends entirely on the + // status the vendor attached to it — see actionable(). if a.FixedVersion == "" { + if !actionable(a.Status) { + unactionable++ + Debugf("%s (src %s): %s skipped, status %q", p.Name, srcName, a.CVEID, a.Status) + continue + } + noFix++ out = append(out, Result{ CVEID: a.CVEID, PackageName: p.Name, Installed: p.Version, Severity: a.Severity, @@ -86,7 +105,35 @@ func Match(src AdvisorySource, os models.OSRelease, pkgs []models.InstalledPacka } } } - log.Printf("vulndb: bucket %q done: %d packages, %d advisories considered, %d results, %d unparseable comparisons", - bucket, len(pkgs), advisoryCount, len(out), skipped) + log.Printf("vulndb: bucket %q done: %d packages, %d sources, %d advisories considered, "+ + "%d results (%d with no vendor fix), %d skipped as not-yet-triaged, %d unparseable comparisons", + bucket, len(pkgs), len(cache), advisoryCount, len(out), noFix, unactionable, skipped) return out, nil } + +// actionable decides whether an advisory with no fixed version is a finding. +// +// trivy-db fills Status only when FixedVersion is empty, and Ubuntu publishes a +// status for every CVE against every source package it ships — the vast +// majority being "under_investigation" (the tracker's needs-triage), meaning +// nobody has yet established that the package is affected at all. Reporting +// those produced ~24,000 findings for a single 797-package host, which is not a +// security report, it is a wall. A "not_affected" is the vendor stating the +// opposite of a finding, so it is never one. +// +// What survives is what the vendor has confirmed: affected, will_not_fix, +// fix_deferred, end_of_life. Those are exactly the findings the CLAUDE.md rule +// is about — an empty fixed_in that means "no fix exists", the one most in need +// of acceptance rather than patching. +// Only the two statuses that positively say "this is not a finding" are +// dropped. "unknown" is kept: a feed that sets no status at all must not become +// a silent false negative, and it is not what generates the noise — Ubuntu +// states under_investigation explicitly. +func actionable(status string) bool { + switch status { + case "not_affected", "under_investigation": + return false + default: + return true + } +}