fix: replay index specs verbatim instead of reconstructing them

backup/restore.go

@@ -83,6 +83,9 @@ func Restore(ctx context.Context, opt RestoreOptions) (RestoreResult, error) {
 	if err := checkTarget(ctx, opt); err != nil {
 		return RestoreResult{}, err
 	}
+	if err := warnLeftovers(ctx, opt, m); err != nil {
+		return RestoreResult{}, err
+	}
 	if len(m.Excluded) > 0 {
 		opt.warn("this archive excluded %s; those collections will be empty after the restore",
 			strings.Join(m.Excluded, ", "))
@@ -154,6 +157,43 @@ func checkTarget(ctx context.Context, opt RestoreOptions) error {
 	return fmt.Errorf("%w: %s holds %s", ErrTargetNotEmpty, opt.Database, strings.Join(found, ", "))
 }
 
+// warnLeftovers names the collections already in the target that the archive
+// does not carry.
+//
+// They are named rather than dropped. A --force restore of an archive taken
+// with --exclude workflow_log_lines leaves the old logs joined to restored
+// runs, which the operator must know; but dropping a collection the archive
+// never mentioned would delete data nobody asked to delete, and there is no
+// way back from that.
+func warnLeftovers(ctx context.Context, opt RestoreOptions, m Manifest) error {
+	if !opt.Force {
+		// Without Force the target was already proven empty.
+		return nil
+	}
+	names, err := opt.Client.Database(opt.Database).ListCollectionNames(ctx, bson.M{})
+	if err != nil {
+		return fmt.Errorf("inspect target: %w", err)
+	}
+	inArchive := map[string]bool{}
+	for _, c := range m.Collections {
+		inArchive[c.Name] = true
+	}
+	var leftover []string
+	for _, n := range names {
+		if !inArchive[n] {
+			leftover = append(leftover, n)
+		}
+	}
+	if len(leftover) == 0 {
+		return nil
+	}
+	sort.Strings(leftover)
+	opt.warn("this archive does not carry %s, which already exist in %s and are left "+
+		"untouched: their contents will sit alongside the restored data",
+		strings.Join(leftover, ", "), opt.Database)
+	return nil
+}
+
 // warnVersionGap reports a major version difference between the server that
 // produced the archive and the one receiving it. It warns rather than refuses:
 // restoring across a major version is a normal part of an upgrade, and a tool
@@ -259,6 +299,14 @@ func splitBSON(raw []byte) (bson.Raw, []byte, error) {
 
 // replayIndexes recreates the archived indexes.
 //
+// The specs are handed to the createIndexes command exactly as the source
+// server reported them, rather than reconstructed into a mongo.IndexModel from
+// a hand-picked set of options. Reconstruction dropped every option nobody had
+// thought to pick — partialFilterExpression above all, which this codebase
+// relies on for partial unique indexes, and which replayed as a full unique
+// index fails on any real database. It also lost compound key order, which is
+// significant.
+//
 // A unique index that will not build means the restored data violates it, and
 // the unique indexes here — (instance_id, email), instance slug, settings
 // instance, the ESO token hash — are tenant-isolation properties rather than
@@ -272,18 +320,26 @@ func replayIndexes(ctx context.Context, opt RestoreOptions, coll *mongo.Collecti
 	if len(raw) == 0 {
 		return 0, nil
 	}
-	var specs []map[string]any
-	if err := json.Unmarshal(raw, &specs); err != nil {
+	var encoded []json.RawMessage
+	if err := json.Unmarshal(raw, &encoded); err != nil {
 		return 0, fmt.Errorf("parse index specs for %s: %w", name, err)
 	}
 
+	db := coll.Database()
 	created := 0
-	for _, spec := range specs {
-		model, indexName, unique, ok := indexModelFrom(spec)
+	for _, ej := range encoded {
+		spec, indexName, unique, ok, err := indexSpecFrom(ej)
+		if err != nil {
+			return created, fmt.Errorf("parse index specs for %s: %w", name, err)
+		}
 		if !ok {
 			continue
 		}
-		if _, err := coll.Indexes().CreateOne(ctx, model); err != nil {
+		cmd := bson.D{
+			{Key: "createIndexes", Value: name},
+			{Key: "indexes", Value: bson.A{spec}},
+		}
+		if err := db.RunCommand(ctx, cmd).Err(); err != nil {
 			if unique {
 				return created, fmt.Errorf("%w: %s on %s: %v", ErrIndexBuild, indexName, name, err)
 			}
@@ -295,45 +351,43 @@ func replayIndexes(ctx context.Context, opt RestoreOptions, coll *mongo.Collecti
 	return created, nil
 }
 
-// indexModelFrom converts one archived index specification into a model.
+// droppedIndexSpecFields are the fields the server reports on an existing index
+// but rejects when creating one. Everything else is passed through untouched.
+var droppedIndexSpecFields = map[string]bool{"v": true, "ns": true}
+
+// indexSpecFrom decodes one archived extended-JSON index specification into an
+// ordered bson.D suitable for createIndexes.
+//
 // The _id_ index is skipped: MongoDB creates it itself and refuses an explicit
 // attempt to create it.
-func indexModelFrom(spec map[string]any) (mongo.IndexModel, string, bool, bool) {
-	name, _ := spec["name"].(string)
-	if name == "_id_" {
-		return mongo.IndexModel{}, name, false, false
-	}
-	keys, ok := spec["key"].(map[string]any)
-	if !ok || len(keys) == 0 {
-		return mongo.IndexModel{}, name, false, false
+func indexSpecFrom(ej []byte) (bson.D, string, bool, bool, error) {
+	var d bson.D
+	if err := bson.UnmarshalExtJSON(ej, false, &d); err != nil {
+		return nil, "", false, false, err
 	}
 
-	// JSON objects do not preserve order but compound index key order is
-	// significant, so the field order recorded by the server is recovered from
-	// the spec's own ordering where available and sorted otherwise. bson.M
-	// round-trips through json as a map; the archive therefore stores the key
-	// document and this reconstructs a deterministic bson.D from it.
-	fields := make([]string, 0, len(keys))
-	for k := range keys {
-		fields = append(fields, k)
-	}
-	sort.Strings(fields)
-	d := make(bson.D, 0, len(fields))
-	for _, f := range fields {
-		d = append(d, bson.E{Key: f, Value: keys[f]})
-	}
-
-	opts := options.Index().SetName(name)
+	out := make(bson.D, 0, len(d))
+	var name string
 	unique := false
-	if u, ok := spec["unique"].(bool); ok && u {
-		unique = true
-		opts = opts.SetUnique(true)
+	hasKey := false
+	for _, e := range d {
+		switch e.Key {
+		case "name":
+			name, _ = e.Value.(string)
+		case "unique":
+			if u, ok := e.Value.(bool); ok {
+				unique = u
+			}
+		case "key":
+			hasKey = true
+		}
+		if droppedIndexSpecFields[e.Key] {
+			continue
+		}
+		out = append(out, e)
 	}
-	if s, ok := spec["sparse"].(bool); ok && s {
-		opts = opts.SetSparse(true)
+	if name == "_id_" || !hasKey {
+		return nil, name, false, false, nil
 	}
-	if e, ok := spec["expireAfterSeconds"].(float64); ok {
-		opts = opts.SetExpireAfterSeconds(int32(e))
-	}
-	return mongo.IndexModel{Keys: d, Options: opts}, name, unique, true
+	return out, name, unique, true, nil
 }