diff --git a/shared/backup/dump.go b/shared/backup/dump.go new file mode 100644 index 0000000..ab17ece --- /dev/null +++ b/shared/backup/dump.go @@ -0,0 +1,170 @@ +package backup + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "time" + + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" +) + +// DumpOptions configures one backup. +type DumpOptions struct { + Client *mongo.Client + Database string + Exclude []string + + // KeyHex is KEY_ENCRYPTION_KEY. It is fingerprinted and discarded; it is + // never written to the archive. + KeyHex string + + // AllowNoKey permits a backup of a deployment that stores no encrypted + // material. The manifest then records a null fingerprint, which restore + // reports rather than treating as a match. + AllowNoKey bool + + VantageVersion string + Out io.Writer +} + +// Dump writes a complete archive of one database to opt.Out. +// +// Collections are enumerated live rather than read from a list. A backup tool +// has no equivalent of AssertNoScopedCollectionMissed to catch a hardcoded list +// drifting, and the first symptom of that drift would be a restore silently +// missing a collection added since the list was written. +func Dump(ctx context.Context, opt DumpOptions) (Manifest, error) { + fingerprint, err := dumpFingerprint(opt) + if err != nil { + return Manifest{}, err + } + + db := opt.Client.Database(opt.Database) + names, err := db.ListCollectionNames(ctx, bson.M{}) + if err != nil { + return Manifest{}, fmt.Errorf("list collections: %w", err) + } + sort.Strings(names) + + excluded := map[string]bool{} + for _, e := range opt.Exclude { + excluded[e] = true + } + + serverVersion, err := mongoServerVersion(ctx, opt.Client) + if err != nil { + return Manifest{}, err + } + hostname, err := os.Hostname() + if err != nil { + hostname = "unknown" + } + + w := NewWriter(opt.Out) + entries := make([]CollectionEntry, 0, len(names)) + for _, name := range names { + if excluded[name] { + continue + } + entry, err := dumpCollection(ctx, w, db, name) + if err != nil { + return Manifest{}, err + } + entries = append(entries, entry) + } + + m := Manifest{ + FormatVersion: FormatVersion, + CreatedAt: time.Now().UTC(), + VantageVersion: opt.VantageVersion, + Hostname: hostname, + MongoDB: opt.Database, + MongoServerVersion: serverVersion, + KeyFingerprint: fingerprint, + Collections: entries, + Excluded: append([]string{}, opt.Exclude...), + } + if err := w.Close(m); err != nil { + return Manifest{}, err + } + return m, nil +} + +// dumpFingerprint applies the key policy before any output is produced. An +// archive of ciphertext whose key was never recorded is worse than no archive, +// because it looks like a backup. +func dumpFingerprint(opt DumpOptions) (*string, error) { + if opt.KeyHex == "" { + if opt.AllowNoKey { + return nil, nil + } + return nil, fmt.Errorf("%w: pass --allow-no-key only if this deployment stores no encrypted data", ErrNoKey) + } + fp, err := FingerprintHex(opt.KeyHex) + if err != nil { + return nil, err + } + return &fp, nil +} + +func dumpCollection(ctx context.Context, w *Writer, db *mongo.Database, name string) (CollectionEntry, error) { + cur, err := db.Collection(name).Find(ctx, bson.M{}) + if err != nil { + return CollectionEntry{}, fmt.Errorf("find %s: %w", name, err) + } + defer cur.Close(ctx) + + var docs [][]byte + for cur.Next(ctx) { + // cur.Current is only valid until the next Next, and it is written to + // the archive verbatim rather than through a map, so every BSON type + // survives exactly as the server stored it. + docs = append(docs, append([]byte(nil), cur.Current...)) + } + if err := cur.Err(); err != nil { + return CollectionEntry{}, fmt.Errorf("iterate %s: %w", name, err) + } + + entry, err := w.WriteCollection(name, docs) + if err != nil { + return CollectionEntry{}, err + } + if err := dumpIndexes(ctx, w, db, name); err != nil { + return CollectionEntry{}, err + } + return entry, nil +} + +func dumpIndexes(ctx context.Context, w *Writer, db *mongo.Database, name string) error { + cur, err := db.Collection(name).Indexes().List(ctx) + if err != nil { + return fmt.Errorf("list indexes on %s: %w", name, err) + } + defer cur.Close(ctx) + + var specs []bson.M + if err := cur.All(ctx, &specs); err != nil { + return fmt.Errorf("read indexes on %s: %w", name, err) + } + raw, err := json.Marshal(specs) + if err != nil { + return fmt.Errorf("encode indexes on %s: %w", name, err) + } + return w.WriteIndexes(name, raw) +} + +func mongoServerVersion(ctx context.Context, client *mongo.Client) (string, error) { + var res struct { + Version string `bson:"version"` + } + err := client.Database("admin").RunCommand(ctx, bson.D{{Key: "buildInfo", Value: 1}}).Decode(&res) + if err != nil { + return "", fmt.Errorf("buildInfo: %w", err) + } + return res.Version, nil +} diff --git a/shared/backup/dump_test.go b/shared/backup/dump_test.go new file mode 100644 index 0000000..f13a845 --- /dev/null +++ b/shared/backup/dump_test.go @@ -0,0 +1,206 @@ +package backup + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" + + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" +) + +func seed(t *testing.T, client *mongo.Client, dbName string) { + t.Helper() + ctx := context.Background() + db := client.Database(dbName) + if _, err := db.Collection("servers").InsertMany(ctx, []any{ + bson.M{"_id": bson.NewObjectID(), "name": "alpha", "instance_id": "i1"}, + bson.M{"_id": bson.NewObjectID(), "name": "beta", "instance_id": "i1"}, + }); err != nil { + t.Fatalf("insert servers: %v", err) + } + if _, err := db.Collection("audit_logs").InsertOne(ctx, bson.M{"action": "login"}); err != nil { + t.Fatalf("insert audit_logs: %v", err) + } +} + +func dumpToFile(t *testing.T, opt DumpOptions) (string, Manifest) { + t.Helper() + path := filepath.Join(t.TempDir(), "out.tar.gz") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + opt.Out = f + m, err := Dump(context.Background(), opt) + if cerr := f.Close(); cerr != nil { + t.Fatalf("close: %v", cerr) + } + if err != nil { + t.Fatalf("Dump: %v", err) + } + return path, m +} + +func TestDumpEnumeratesEveryCollection(t *testing.T) { + client, dbName := testDB(t) + seed(t, client, dbName) + + _, m := dumpToFile(t, DumpOptions{ + Client: client, Database: dbName, KeyHex: validKeyHex, VantageVersion: "test", + }) + + if _, ok := m.Collection("servers"); !ok { + t.Fatal("servers missing from the manifest") + } + if _, ok := m.Collection("audit_logs"); !ok { + t.Fatal("audit_logs missing; enumeration must not filter by a hardcoded list") + } + servers, _ := m.Collection("servers") + if servers.Documents != 2 { + t.Fatalf("servers documents %d, want 2", servers.Documents) + } + if m.MongoDB != dbName { + t.Fatalf("manifest database %q, want %q", m.MongoDB, dbName) + } + if m.MongoServerVersion == "" { + t.Fatal("manifest records no MongoDB server version") + } + if m.Hostname == "" { + t.Fatal("manifest records no hostname") + } +} + +func TestDumpRecordsKeyFingerprint(t *testing.T) { + client, dbName := testDB(t) + seed(t, client, dbName) + + _, m := dumpToFile(t, DumpOptions{Client: client, Database: dbName, KeyHex: validKeyHex}) + + want, err := FingerprintHex(validKeyHex) + if err != nil { + t.Fatalf("FingerprintHex: %v", err) + } + if m.KeyFingerprint == nil || *m.KeyFingerprint != want { + t.Fatalf("fingerprint %v, want %s", m.KeyFingerprint, want) + } +} + +func TestDumpRefusesWithoutAKey(t *testing.T) { + client, dbName := testDB(t) + seed(t, client, dbName) + + var buf bytes.Buffer + _, err := Dump(context.Background(), DumpOptions{ + Client: client, Database: dbName, Out: &buf, + }) + if !errors.Is(err, ErrNoKey) { + t.Fatalf("got %v, want ErrNoKey", err) + } + if buf.Len() != 0 { + t.Fatal("refusal must happen before anything is written") + } +} + +func TestDumpAllowNoKeyStampsNull(t *testing.T) { + client, dbName := testDB(t) + seed(t, client, dbName) + + _, m := dumpToFile(t, DumpOptions{Client: client, Database: dbName, AllowNoKey: true}) + if m.KeyFingerprint != nil { + t.Fatalf("want a null fingerprint, got %v", *m.KeyFingerprint) + } +} + +func TestDumpRejectsMalformedKey(t *testing.T) { + client, dbName := testDB(t) + var buf bytes.Buffer + _, err := Dump(context.Background(), DumpOptions{ + Client: client, Database: dbName, KeyHex: "nonsense", Out: &buf, + }) + if !errors.Is(err, ErrBadKey) { + t.Fatalf("got %v, want ErrBadKey", err) + } +} + +func TestDumpExcludeIsRecordedAndOmitted(t *testing.T) { + client, dbName := testDB(t) + seed(t, client, dbName) + + path, m := dumpToFile(t, DumpOptions{ + Client: client, Database: dbName, KeyHex: validKeyHex, + Exclude: []string{"audit_logs"}, + }) + + if _, ok := m.Collection("audit_logs"); ok { + t.Fatal("excluded collection is in the manifest's collection list") + } + if len(m.Excluded) != 1 || m.Excluded[0] != "audit_logs" { + t.Fatalf("excluded recorded as %v", m.Excluded) + } + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + if _, err := r.OpenCollection("audit_logs"); err == nil { + t.Fatal("excluded collection is present in the archive") + } +} + +func TestDumpPreservesAwkwardBSONTypes(t *testing.T) { + client, dbName := testDB(t) + ctx := context.Background() + + dec, err := bson.ParseDecimal128("1234.5678") + if err != nil { + t.Fatalf("ParseDecimal128: %v", err) + } + doc := bson.M{ + "_id": bson.NewObjectID(), + "decimal": dec, + "when": bson.NewDateTimeFromTime(mustTime(t)), + "binary": bson.Binary{Subtype: 0x00, Data: []byte{0x01, 0x02, 0x03}}, + "nothing": nil, + "nested": bson.A{bson.M{"deep": bson.A{1, 2, 3}}}, + } + if _, err := client.Database(dbName).Collection("odd").InsertOne(ctx, doc); err != nil { + t.Fatalf("insert: %v", err) + } + + path, _ := dumpToFile(t, DumpOptions{Client: client, Database: dbName, KeyHex: validKeyHex}) + + original, err := client.Database(dbName).Collection("odd").FindOne(ctx, bson.M{}).Raw() + if err != nil { + t.Fatalf("read back: %v", err) + } + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + rc, err := r.OpenCollection("odd") + if err != nil { + t.Fatalf("OpenCollection: %v", err) + } + defer rc.Close() + archived, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(archived, []byte(original)) { + t.Fatal("archived BSON differs from what the driver returned") + } +} + +func mustTime(t *testing.T) time.Time { + t.Helper() + return time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) +} diff --git a/shared/backup/mongo_test.go b/shared/backup/mongo_test.go new file mode 100644 index 0000000..b9b2081 --- /dev/null +++ b/shared/backup/mongo_test.go @@ -0,0 +1,44 @@ +package backup + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +// testDB connects to the MongoDB named by MONGO_TEST_URI and returns a client +// plus a database name unique to this test, dropped when the test ends. +// +// Skips rather than fails when the variable is unset: these tests need a real +// server, and a developer without one should still be able to run the rest of +// the suite. +func testDB(t *testing.T) (*mongo.Client, string) { + t.Helper() + uri := os.Getenv("MONGO_TEST_URI") + if uri == "" { + t.Skip("MONGO_TEST_URI is not set; skipping tests that need MongoDB") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + client, err := mongo.Connect(options.Client().ApplyURI(uri)) + if err != nil { + t.Fatalf("connect: %v", err) + } + if err := client.Ping(ctx, nil); err != nil { + t.Fatalf("ping: %v", err) + } + name := fmt.Sprintf("vantage_test_%d", time.Now().UnixNano()) + t.Cleanup(func() { + c, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = client.Database(name).Drop(c) + _ = client.Disconnect(c) + }) + return client, name +}