[PBM-1155] use new thisBackupName on each retries

pbm/backup/physical.go

@@ -45,12 +45,15 @@ type BackupCursorData struct {
 	Meta *Meta
 	Data []pbm.File
 }
+
 type BackupCursor struct {
 	id    UUID
 	n     *pbm.Node
 	l     *plog.Event
 	opts  bson.D
 	close chan struct{}
+
+	CustomThisID string
 }
 
 func NewBackupCursor(n *pbm.Node, l *plog.Event, opts bson.D) *BackupCursor {
@@ -64,12 +67,33 @@ func NewBackupCursor(n *pbm.Node, l *plog.Event, opts bson.D) *BackupCursor {
 var triesLimitExceededErr = errors.New("tries limit exceeded")
 
 func (bc *BackupCursor) create(ctx context.Context, retry int) (*mongo.Cursor, error) {
+	opts := bc.opts
 	for i := 0; i < retry; i++ {
+		if i != 0 {
+			// on retry, make new thisBackupName
+			// otherwise, WT error: "Incremental identifier already exists"
+			opts = make(bson.D, len(bc.opts))
+			for j, a := range bc.opts {
+				val := a.Value
+				if a.Key == "thisBackupName" {
+					bc.CustomThisID = fmt.Sprintf("%s.%d", a.Value, i)
+					val = bc.CustomThisID
+				}
+
+				opts[j] = bson.E{a.Key, val}
+			}
+		}
+
 		cur, err := bc.n.Session().Database("admin").Aggregate(ctx, mongo.Pipeline{
-			{{"$backupCursor", bc.opts}},
+			{{"$backupCursor", opts}},
 		})
 		if err != nil {
-			if se, ok := err.(mongo.ServerError); ok && se.HasErrorCode(50915) {
+			se, ok := err.(mongo.ServerError)
+			if !ok {
+				return nil, err
+			}
+
+			if se.HasErrorCode(50915) {
 				// {code: 50915,name: BackupCursorOpenConflictWithCheckpoint, categories: [RetriableError]}
 				// https://github.com/percona/percona-server-mongodb/blob/psmdb-6.0.6-5/src/mongo/base/error_codes.yml#L526
 				bc.l.Debug("a checkpoint took place, retrying")
@@ -170,6 +194,7 @@ func (b *Backup) doPhysical(ctx context.Context, bcp *pbm.BackupCmd, opid pbm.OP
 	currOpts := bson.D{}
 	if b.typ == pbm.IncrementalBackup {
 		currOpts = bson.D{
+			// thisBackupName can be customized on retry
 			{"thisBackupName", pbm.BackupCursorName(bcp.Name)},
 			{"incrementalBackup", true},
 		}
@@ -189,7 +214,21 @@ func (b *Backup) doPhysical(ctx context.Context, bcp *pbm.BackupCmd, opid pbm.OP
 					return errors.Wrap(err, "set source backup in meta")
 				}
 			}
-			currOpts = append(currOpts, bson.E{"srcBackupName", pbm.BackupCursorName(src.Name)})
+
+			// realSrcID is actual thisBackupName of the replset
+			var realSrcID string
+			for _, rs := range src.Replsets {
+				if rs.Name == rsMeta.Name {
+					realSrcID = rs.CustomThisID
+					break
+				}
+			}
+			if realSrcID == "" {
+				// no custom thisBackupName was used. fallback to default
+				realSrcID = pbm.BackupCursorName(src.Name)
+			}
+
+			currOpts = append(currOpts, bson.E{"srcBackupName", realSrcID})
 		} else {
 			// We don't need any previous incremental backup history if
 			// this is a base backup. So we can flush it to free up resources.
@@ -241,6 +280,10 @@ func (b *Backup) doPhysical(ctx context.Context, bcp *pbm.BackupCmd, opid pbm.OP
 	rsMeta.Status = pbm.StatusRunning
 	rsMeta.FirstWriteTS = bcur.Meta.OplogEnd.TS
 	rsMeta.LastWriteTS = lwts
+	if cursor.CustomThisID != "" {
+		// custom thisBackupName was used
+		rsMeta.CustomThisID = cursor.CustomThisID
+	}
 	err = b.cn.AddRSMeta(bcp.Name, *rsMeta)
 	if err != nil {
 		return errors.Wrap(err, "add shard's metadata")
@@ -466,7 +509,8 @@ func (id *UUID) IsZero() bool {
 // unchanged files (Len == 0) but add them to the meta as we need know
 // what files shouldn't be restored (those which isn't in the target backup).
 func uploadFiles(ctx context.Context, files []pbm.File, subdir, trimPrefix string, incr bool,
-	stg storage.Storage, comprT compress.CompressionType, comprL *int, l *plog.Event) (data []pbm.File, err error) {
+	stg storage.Storage, comprT compress.CompressionType, comprL *int, l *plog.Event,
+) (data []pbm.File, err error) {
 	if len(files) == 0 {
 		return data, err
 	}

pbm/pbm.go

@@ -549,6 +549,10 @@ type BackupReplset struct {
 	Error            string              `bson:"error,omitempty" json:"error,omitempty"`
 	Conditions       []Condition         `bson:"conditions" json:"conditions"`
 	MongodOpts       *MongodOpts         `bson:"mongod_opts,omitempty" json:"mongod_opts,omitempty"`
+
+	// CustomThisID is customized thisBackupName value for $backupCursor (in WT: "this_id").
+	// If it is not set (empty), the default value was used.
+	CustomThisID string `bson:"this_id,omitempty" json:"this_id,omitempty"`
 }
 
 type File struct {