feat(#45): Add 'rotateEnabled' option for backup rotation settings

Changes:
- Replaced individual setter methods for storage with a new Configure method that uses a Params struct.
- Added a new 'rotateEnabled' field in the Params struct for storage and implemented backup rotation control.
- Updated use of 'SkipBackupRotate' option which is now deprecated, and replaced with 'storages_options[].enable_rotate'.
- Added error message if used deprecated 'skip_backup_rotate' option.
- Reflect the changes in configuration of all storage types.
- Updated the generate_config module to use the new 'enable_rotate' setting.

Refs: #45

ctx/conf.go

@@ -82,7 +82,7 @@ type jobConf struct {
 	Sources          []sourceConf    `conf:"sources"`
 	StoragesOptions  []storageConf   `conf:"storages_options"`
 	DumpCmd          string          `conf:"dump_cmd"`
-	SkipBackupRotate bool            `conf:"skip_backup_rotate" conf_extraopts:"default=false"` // used by external
+	SkipBackupRotate bool            `conf:"skip_backup_rotate" conf_extraopts:"default=false"` // deprecated, used by external
 	Limits           *limitsConf     `conf:"limits"`
 }
 
@@ -119,9 +119,10 @@ type sourceConnectConf struct {
 }
 
 type storageConf struct {
-	StorageName string        `conf:"storage_name" conf_extraopts:"required"`
-	BackupPath  string        `conf:"backup_path" conf_extraopts:"required"`
-	Retention   retentionConf `conf:"retention" conf_extraopts:"required"`
+	StorageName  string        `conf:"storage_name" conf_extraopts:"required"`
+	BackupPath   string        `conf:"backup_path" conf_extraopts:"required"`
+	EnableRotate bool          `conf:"enable_rotate" conf_extraopts:"default=true"`
+	Retention    retentionConf `conf:"retention" conf_extraopts:"required"`
 }
 
 type retentionConf struct {

ctx/context.go

@@ -285,3 +285,7 @@ func getRateLimit(limit *string) (rl int64, err error) {
 
 	return
 }
+
+func checkDeprecated(o ConfOpts) {
+
+}

ctx/jobs_init.go

@@ -100,14 +100,17 @@ func jobsInit(o jobsOpts) ([]interfaces.Job, error) {
 			}
 
 			st := s.Clone()
-			st.SetBackupPath(opt.BackupPath)
-			st.SetRetention(storage.Retention(opt.Retention))
-
+			stParams := storage.Params{
+				BackupPath:    opt.BackupPath,
+				RotateEnabled: opt.EnableRotate,
+				Retention:     storage.Retention(opt.Retention),
+			}
 			if opt.StorageName == "local" {
-				st.SetRateLimit(diskRate)
+				stParams.RateLimit = diskRate
 			} else if withStorageRate {
-				st.SetRateLimit(nrl)
+				stParams.RateLimit = nrl
 			}
+			st.Configure(stParams)
 
 			if storage.IsNeedToBackup(opt.Retention.Days, opt.Retention.Weeks, opt.Retention.Months) {
 				needToMakeBackup = true
@@ -401,6 +404,9 @@ func jobsInit(o jobsOpts) ([]interfaces.Job, error) {
 			})
 
 		case misc.External:
+			if j.SkipBackupRotate {
+				errs = multierror.Append(errs, fmt.Errorf("Used deprecated option `skip_backup_rotate` for job \"%s\". Use `storages_options[].enable_rotate` instead. ", j.Name))
+			}
 			job, err = external.Init(external.JobParams{
 				Name:             j.Name,
 				DumpCmd:          j.DumpCmd,

interfaces/storage.go

@@ -16,12 +16,10 @@ import (
 
 type Storage interface {
 	IsLocal() int
-	SetBackupPath(path string)
-	SetRateLimit(rl int64)
-	SetRetention(r storage.Retention)
+	Configure(storage.Params)
 	DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupPath, ofs, bakType string) error
 	DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job Job, full bool) error
-	GetFileReader(path string) (io.Reader, error)
+	GetFileReader(string) (io.Reader, error)
 	Close() error
 	Clone() Storage
 	GetName() string

modules/backup/external/external.go

@@ -21,7 +21,7 @@ type job struct {
 	envs             map[string]string
 	needToMakeBackup bool
 	safetyBackup     bool
-	skipBackupRotate bool
+	skipBackupRotate bool // deprecated
 	storages         interfaces.Storages
 	dumpedObjects    map[string]interfaces.DumpObject
 	appMetrics       *metrics.Data
@@ -34,7 +34,7 @@ type JobParams struct {
 	Envs             map[string]string
 	NeedToMakeBackup bool
 	SafetyBackup     bool
-	SkipBackupRotate bool
+	SkipBackupRotate bool // deprecated
 	Storages         interfaces.Storages
 	Metrics          *metrics.Data
 }

modules/cmd_handler/generate_config/generate_config.go

@@ -56,9 +56,10 @@ type srcConnectYaml struct {
 }
 
 type storageOptsYaml struct {
-	StorageName string           `yaml:"storage_name"`
-	BackupPath  string           `yaml:"backup_path"`
-	Retention   cfgRetentionYaml `yaml:"retention"`
+	StorageName  string           `yaml:"storage_name"`
+	BackupPath   string           `yaml:"backup_path"`
+	EnableRotate bool             `yaml:"enable_rotate"`
+	Retention    cfgRetentionYaml `yaml:"retention"`
 }
 
 type cfgRetentionYaml struct {
@@ -384,16 +385,18 @@ func genStorageOpts(storages map[string]string, incBackup bool) (sts []storageOp
 	}
 
 	sts = append(sts, storageOptsYaml{
-		StorageName: "local",
-		BackupPath:  "/var/nxs-backup/dump",
-		Retention:   defaultRetention,
+		StorageName:  "local",
+		BackupPath:   "/var/nxs-backup/dump",
+		EnableRotate: true,
+		Retention:    defaultRetention,
 	})
 
 	for nm := range storages {
 		sts = append(sts, storageOptsYaml{
-			StorageName: nm,
-			BackupPath:  "/nxs-backup/dump",
-			Retention:   defaultRetention,
+			StorageName:  nm,
+			BackupPath:   "/nxs-backup/dump",
+			EnableRotate: true,
+			Retention:    defaultRetention,
 		})
 	}
 

modules/storage/common.go

@@ -22,6 +22,13 @@ const (
 
 var RetentionPeriodsList = []retentionPeriod{Monthly, Weekly, Daily}
 
+type Params struct {
+	RateLimit     int64
+	BackupPath    string
+	RotateEnabled bool
+	Retention
+}
+
 type Retention struct {
 	Days     int
 	Weeks    int

modules/storage/ftp/ftp.go

@@ -25,11 +25,12 @@ import (
 )
 
 type FTP struct {
-	conn       *ftp.ServerConn
-	backupPath string
-	name       string
-	rateLimit  int64
-	opts       Opts
+	conn          *ftp.ServerConn
+	name          string
+	backupPath    string
+	rateLimit     int64
+	rotateEnabled bool
+	opts          Opts
 	Retention
 }
 
@@ -58,6 +59,13 @@ func Init(name string, opts Opts, rl int64) (s *FTP, err error) {
 	return
 }
 
+func (f *FTP) Configure(p Params) {
+	f.backupPath = p.BackupPath
+	f.rateLimit = p.RateLimit
+	f.rotateEnabled = p.RotateEnabled
+	f.Retention = p.Retention
+}
+
 func (f *FTP) updateConn() error {
 
 	if f.conn != nil {
@@ -88,18 +96,6 @@ func (f *FTP) updateConn() error {
 
 func (f *FTP) IsLocal() int { return 0 }
 
-func (f *FTP) SetBackupPath(path string) {
-	f.backupPath = path
-}
-
-func (f *FTP) SetRateLimit(rl int64) {
-	f.rateLimit = rl
-}
-
-func (f *FTP) SetRetention(r Retention) {
-	f.Retention = r
-}
-
 func (f *FTP) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs string, bakType string) error {
 	var bakRemPaths, mtdRemPaths []string
 
@@ -157,6 +153,10 @@ func (f *FTP) copy(logCh chan logger.LogRecord, job, dst, src string) error {
 }
 
 func (f *FTP) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error {
+	if !f.rotateEnabled {
+		logCh <- logger.Log(job.GetName(), f.name).Debugf("Backup rotate skipped by config.")
+		return nil
+	}
 
 	if err := f.updateConn(); err != nil {
 		return err

modules/storage/local/local.go

@@ -23,8 +23,9 @@ import (
 )
 
 type Local struct {
-	backupPath string
-	rateLimit  int64
+	backupPath    string
+	rateLimit     int64
+	rotateEnabled bool
 	Retention
 }
 
@@ -34,19 +35,14 @@ func Init(rl int64) *Local {
 	}
 }
 
-func (l *Local) IsLocal() int { return 1 }
-
-func (l *Local) SetBackupPath(path string) {
-	l.backupPath = path
-}
-
-func (l *Local) SetRateLimit(rl int64) {
-	l.rateLimit = rl
+func (l *Local) Configure(p Params) {
+	l.backupPath = p.BackupPath
+	l.rateLimit = p.RateLimit
+	l.rotateEnabled = p.RotateEnabled
+	l.Retention = p.Retention
 }
 
-func (l *Local) SetRetention(r Retention) {
-	l.Retention = r
-}
+func (l *Local) IsLocal() int { return 1 }
 
 func (l *Local) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) (err error) {
 	var (
@@ -156,6 +152,10 @@ func (l *Local) deliveryBackupMetadata(logCh chan logger.LogRecord, jobName, tmp
 }
 
 func (l *Local) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error {
+	if !l.rotateEnabled {
+		logCh <- logger.Log(job.GetName(), l.GetName()).Debugf("Backup rotate skipped by config.")
+		return nil
+	}
 
 	if job.GetType() == misc.IncFiles {
 		return l.deleteIncBackup(logCh, job.GetName(), ofsPart, full)

modules/storage/nfs/nfs.go

@@ -26,10 +26,11 @@ import (
 )
 
 type NFS struct {
-	target     *nfs.Target
-	backupPath string
-	name       string
-	rateLimit  int64
+	target        *nfs.Target
+	name          string
+	backupPath    string
+	rateLimit     int64
+	rotateEnabled bool
 	Retention
 }
 
@@ -73,19 +74,14 @@ func Init(name string, params Opts, rl int64) (*NFS, error) {
 	}, nil
 }
 
-func (n *NFS) IsLocal() int { return 0 }
-
-func (n *NFS) SetBackupPath(path string) {
-	n.backupPath = path
-}
-
-func (n *NFS) SetRateLimit(rl int64) {
-	n.rateLimit = rl
+func (n *NFS) Configure(p Params) {
+	n.backupPath = p.BackupPath
+	n.rateLimit = p.RateLimit
+	n.rotateEnabled = p.RotateEnabled
+	n.Retention = p.Retention
 }
 
-func (n *NFS) SetRetention(r Retention) {
-	n.Retention = r
-}
+func (n *NFS) IsLocal() int { return 0 }
 
 func (n *NFS) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) error {
 	var bakRemPaths, mtdRemPaths []string
@@ -157,6 +153,10 @@ func (n *NFS) copy(logCh chan logger.LogRecord, jobName, dst, src string) error
 }
 
 func (n *NFS) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error {
+	if !n.rotateEnabled {
+		logCh <- logger.Log(job.GetName(), n.name).Debugf("Backup rotate skipped by config.")
+		return nil
+	}
 
 	if job.GetType() == misc.IncFiles {
 		return n.deleteIncBackup(logCh, job.GetName(), ofsPart, full)

modules/storage/s3/s3.go

@@ -25,11 +25,12 @@ import (
 
 type S3 struct {
 	client        *minio.Client
+	name          string
 	bucketName    string
 	backupPath    string
-	name          string
-	batchDeletion bool
 	rateLimit     int64
+	rotateEnabled bool
+	batchDeletion bool
 	Retention
 }
 
@@ -70,19 +71,14 @@ func Init(name string, opts Opts, rl int64) (*S3, error) {
 	}, nil
 }
 
-func (s *S3) IsLocal() int { return 0 }
-
-func (s *S3) SetBackupPath(path string) {
-	s.backupPath = strings.TrimPrefix(path, "/")
+func (s *S3) Configure(p Params) {
+	s.backupPath = p.BackupPath
+	s.rateLimit = p.RateLimit
+	s.rotateEnabled = p.RotateEnabled
+	s.Retention = p.Retention
 }
 
-func (s *S3) SetRateLimit(rl int64) {
-	s.rateLimit = rl
-}
-
-func (s *S3) SetRetention(r Retention) {
-	s.Retention = r
-}
+func (s *S3) IsLocal() int { return 0 }
 
 func (s *S3) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) error {
 	var bakRemPaths, mtdRemPaths []string
@@ -140,6 +136,10 @@ func (s *S3) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile,
 }
 
 func (s *S3) DeleteOldBackups(logCh chan logger.LogRecord, ofs string, job interfaces.Job, full bool) error {
+	if !s.rotateEnabled {
+		logCh <- logger.Log(job.GetName(), s.name).Debugf("Backup rotate skipped by config.")
+		return nil
+	}
 
 	curDate := time.Now().Round(24 * time.Hour)