diff --git a/ctx/conf.go b/ctx/conf.go index 2ee7652..4e3cc6d 100644 --- a/ctx/conf.go +++ b/ctx/conf.go @@ -83,7 +83,7 @@ type jobConf struct { Sources []sourceConf `conf:"sources"` StoragesOptions []storageConf `conf:"storages_options"` DumpCmd string `conf:"dump_cmd"` - SkipBackupRotate bool `conf:"skip_backup_rotate" conf_extraopts:"default=false"` // used by external + SkipBackupRotate bool `conf:"skip_backup_rotate" conf_extraopts:"default=false"` // deprecated, used by external Limits *limitsConf `conf:"limits"` } @@ -120,9 +120,10 @@ type sourceConnectConf struct { } type storageConf struct { - StorageName string `conf:"storage_name" conf_extraopts:"required"` - BackupPath string `conf:"backup_path" conf_extraopts:"required"` - Retention retentionConf `conf:"retention" conf_extraopts:"required"` + StorageName string `conf:"storage_name" conf_extraopts:"required"` + BackupPath string `conf:"backup_path" conf_extraopts:"required"` + EnableRotate bool `conf:"enable_rotate" conf_extraopts:"default=true"` + Retention retentionConf `conf:"retention" conf_extraopts:"required"` } type retentionConf struct { diff --git a/ctx/context.go b/ctx/context.go index 6afda0f..28bcf3e 100755 --- a/ctx/context.go +++ b/ctx/context.go @@ -281,3 +281,7 @@ func getRateLimit(limit *string) (rl int64, err error) { return } + +func checkDeprecated(o ConfOpts) { + +} diff --git a/ctx/jobs_init.go b/ctx/jobs_init.go index ea49e05..aaed5b6 100644 --- a/ctx/jobs_init.go +++ b/ctx/jobs_init.go @@ -100,14 +100,17 @@ func jobsInit(o jobsOpts) ([]interfaces.Job, error) { } st := s.Clone() - st.SetBackupPath(opt.BackupPath) - st.SetRetention(storage.Retention(opt.Retention)) - + stParams := storage.Params{ + BackupPath: opt.BackupPath, + RotateEnabled: opt.EnableRotate, + Retention: storage.Retention(opt.Retention), + } if opt.StorageName == "local" { - st.SetRateLimit(diskRate) + stParams.RateLimit = diskRate } else if withStorageRate { - st.SetRateLimit(nrl) + stParams.RateLimit = nrl } + st.Configure(stParams) if storage.IsNeedToBackup(opt.Retention.Days, opt.Retention.Weeks, opt.Retention.Months) { needToMakeBackup = true @@ -401,6 +404,9 @@ func jobsInit(o jobsOpts) ([]interfaces.Job, error) { }) case misc.External: + if j.SkipBackupRotate { + errs = multierror.Append(errs, fmt.Errorf("Used deprecated option `skip_backup_rotate` for job \"%s\". Use `storages_options[].enable_rotate` instead. ", j.Name)) + } job, err = external.Init(external.JobParams{ Name: j.Name, DumpCmd: j.DumpCmd, diff --git a/interfaces/storage.go b/interfaces/storage.go index e5f0f33..3bb0d1d 100755 --- a/interfaces/storage.go +++ b/interfaces/storage.go @@ -16,12 +16,10 @@ import ( type Storage interface { IsLocal() int - SetBackupPath(path string) - SetRateLimit(rl int64) - SetRetention(r storage.Retention) + Configure(storage.Params) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupPath, ofs, bakType string) error DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job Job, full bool) error - GetFileReader(path string) (io.Reader, error) + GetFileReader(string) (io.Reader, error) Close() error Clone() Storage GetName() string diff --git a/modules/backup/external/external.go b/modules/backup/external/external.go index b9da8e4..b8bac12 100644 --- a/modules/backup/external/external.go +++ b/modules/backup/external/external.go @@ -21,7 +21,7 @@ type job struct { envs map[string]string needToMakeBackup bool safetyBackup bool - skipBackupRotate bool + skipBackupRotate bool // deprecated storages interfaces.Storages dumpedObjects map[string]interfaces.DumpObject appMetrics *metrics.Data @@ -34,7 +34,7 @@ type JobParams struct { Envs map[string]string NeedToMakeBackup bool SafetyBackup bool - SkipBackupRotate bool + SkipBackupRotate bool // deprecated Storages interfaces.Storages Metrics *metrics.Data } diff --git a/modules/cmd_handler/generate_config/generate_config.go b/modules/cmd_handler/generate_config/generate_config.go index 27e2368..70e81dd 100755 --- a/modules/cmd_handler/generate_config/generate_config.go +++ b/modules/cmd_handler/generate_config/generate_config.go @@ -56,9 +56,10 @@ type srcConnectYaml struct { } type storageOptsYaml struct { - StorageName string `yaml:"storage_name"` - BackupPath string `yaml:"backup_path"` - Retention cfgRetentionYaml `yaml:"retention"` + StorageName string `yaml:"storage_name"` + BackupPath string `yaml:"backup_path"` + EnableRotate bool `yaml:"enable_rotate"` + Retention cfgRetentionYaml `yaml:"retention"` } type cfgRetentionYaml struct { @@ -384,16 +385,18 @@ func genStorageOpts(storages map[string]string, incBackup bool) (sts []storageOp } sts = append(sts, storageOptsYaml{ - StorageName: "local", - BackupPath: "/var/nxs-backup/dump", - Retention: defaultRetention, + StorageName: "local", + BackupPath: "/var/nxs-backup/dump", + EnableRotate: true, + Retention: defaultRetention, }) for nm := range storages { sts = append(sts, storageOptsYaml{ - StorageName: nm, - BackupPath: "/nxs-backup/dump", - Retention: defaultRetention, + StorageName: nm, + BackupPath: "/nxs-backup/dump", + EnableRotate: true, + Retention: defaultRetention, }) } diff --git a/modules/storage/common.go b/modules/storage/common.go index 542d061..73656fe 100755 --- a/modules/storage/common.go +++ b/modules/storage/common.go @@ -22,6 +22,13 @@ const ( var RetentionPeriodsList = []retentionPeriod{Monthly, Weekly, Daily} +type Params struct { + RateLimit int64 + BackupPath string + RotateEnabled bool + Retention +} + type Retention struct { Days int Weeks int diff --git a/modules/storage/ftp/ftp.go b/modules/storage/ftp/ftp.go index aa64108..219f994 100755 --- a/modules/storage/ftp/ftp.go +++ b/modules/storage/ftp/ftp.go @@ -25,11 +25,12 @@ import ( ) type FTP struct { - conn *ftp.ServerConn - backupPath string - name string - rateLimit int64 - opts Opts + conn *ftp.ServerConn + name string + backupPath string + rateLimit int64 + rotateEnabled bool + opts Opts Retention } @@ -58,6 +59,13 @@ func Init(name string, opts Opts, rl int64) (s *FTP, err error) { return } +func (f *FTP) Configure(p Params) { + f.backupPath = p.BackupPath + f.rateLimit = p.RateLimit + f.rotateEnabled = p.RotateEnabled + f.Retention = p.Retention +} + func (f *FTP) updateConn() error { if f.conn != nil { @@ -88,18 +96,6 @@ func (f *FTP) updateConn() error { func (f *FTP) IsLocal() int { return 0 } -func (f *FTP) SetBackupPath(path string) { - f.backupPath = path -} - -func (f *FTP) SetRateLimit(rl int64) { - f.rateLimit = rl -} - -func (f *FTP) SetRetention(r Retention) { - f.Retention = r -} - func (f *FTP) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs string, bakType string) error { var bakRemPaths, mtdRemPaths []string @@ -157,6 +153,10 @@ func (f *FTP) copy(logCh chan logger.LogRecord, job, dst, src string) error { } func (f *FTP) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error { + if !f.rotateEnabled { + logCh <- logger.Log(job.GetName(), f.name).Debugf("Backup rotate skipped by config.") + return nil + } if err := f.updateConn(); err != nil { return err diff --git a/modules/storage/local/local.go b/modules/storage/local/local.go index 6941384..38bdeaf 100755 --- a/modules/storage/local/local.go +++ b/modules/storage/local/local.go @@ -23,8 +23,9 @@ import ( ) type Local struct { - backupPath string - rateLimit int64 + backupPath string + rateLimit int64 + rotateEnabled bool Retention } @@ -34,19 +35,14 @@ func Init(rl int64) *Local { } } -func (l *Local) IsLocal() int { return 1 } - -func (l *Local) SetBackupPath(path string) { - l.backupPath = path -} - -func (l *Local) SetRateLimit(rl int64) { - l.rateLimit = rl +func (l *Local) Configure(p Params) { + l.backupPath = p.BackupPath + l.rateLimit = p.RateLimit + l.rotateEnabled = p.RotateEnabled + l.Retention = p.Retention } -func (l *Local) SetRetention(r Retention) { - l.Retention = r -} +func (l *Local) IsLocal() int { return 1 } func (l *Local) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) (err error) { var ( @@ -156,6 +152,10 @@ func (l *Local) deliveryBackupMetadata(logCh chan logger.LogRecord, jobName, tmp } func (l *Local) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error { + if !l.rotateEnabled { + logCh <- logger.Log(job.GetName(), l.GetName()).Debugf("Backup rotate skipped by config.") + return nil + } if job.GetType() == misc.IncFiles { return l.deleteIncBackup(logCh, job.GetName(), ofsPart, full) diff --git a/modules/storage/nfs/nfs.go b/modules/storage/nfs/nfs.go index d78fb5d..efa4cfe 100755 --- a/modules/storage/nfs/nfs.go +++ b/modules/storage/nfs/nfs.go @@ -26,10 +26,11 @@ import ( ) type NFS struct { - target *nfs.Target - backupPath string - name string - rateLimit int64 + target *nfs.Target + name string + backupPath string + rateLimit int64 + rotateEnabled bool Retention } @@ -73,19 +74,14 @@ func Init(name string, params Opts, rl int64) (*NFS, error) { }, nil } -func (n *NFS) IsLocal() int { return 0 } - -func (n *NFS) SetBackupPath(path string) { - n.backupPath = path -} - -func (n *NFS) SetRateLimit(rl int64) { - n.rateLimit = rl +func (n *NFS) Configure(p Params) { + n.backupPath = p.BackupPath + n.rateLimit = p.RateLimit + n.rotateEnabled = p.RotateEnabled + n.Retention = p.Retention } -func (n *NFS) SetRetention(r Retention) { - n.Retention = r -} +func (n *NFS) IsLocal() int { return 0 } func (n *NFS) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) error { var bakRemPaths, mtdRemPaths []string @@ -157,6 +153,10 @@ func (n *NFS) copy(logCh chan logger.LogRecord, jobName, dst, src string) error } func (n *NFS) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error { + if !n.rotateEnabled { + logCh <- logger.Log(job.GetName(), n.name).Debugf("Backup rotate skipped by config.") + return nil + } if job.GetType() == misc.IncFiles { return n.deleteIncBackup(logCh, job.GetName(), ofsPart, full) diff --git a/modules/storage/s3/s3.go b/modules/storage/s3/s3.go index 9651cc8..a15e2da 100755 --- a/modules/storage/s3/s3.go +++ b/modules/storage/s3/s3.go @@ -25,11 +25,12 @@ import ( type S3 struct { client *minio.Client + name string bucketName string backupPath string - name string - batchDeletion bool rateLimit int64 + rotateEnabled bool + batchDeletion bool Retention } @@ -70,19 +71,14 @@ func Init(name string, opts Opts, rl int64) (*S3, error) { }, nil } -func (s *S3) IsLocal() int { return 0 } - -func (s *S3) SetBackupPath(path string) { - s.backupPath = strings.TrimPrefix(path, "/") +func (s *S3) Configure(p Params) { + s.backupPath = p.BackupPath + s.rateLimit = p.RateLimit + s.rotateEnabled = p.RotateEnabled + s.Retention = p.Retention } -func (s *S3) SetRateLimit(rl int64) { - s.rateLimit = rl -} - -func (s *S3) SetRetention(r Retention) { - s.Retention = r -} +func (s *S3) IsLocal() int { return 0 } func (s *S3) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) error { var bakRemPaths, mtdRemPaths []string @@ -140,6 +136,10 @@ func (s *S3) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, } func (s *S3) DeleteOldBackups(logCh chan logger.LogRecord, ofs string, job interfaces.Job, full bool) error { + if !s.rotateEnabled { + logCh <- logger.Log(job.GetName(), s.name).Debugf("Backup rotate skipped by config.") + return nil + } curDate := time.Now().Round(24 * time.Hour) diff --git a/modules/storage/sftp/sftp.go b/modules/storage/sftp/sftp.go index e448839..a17b5a5 100755 --- a/modules/storage/sftp/sftp.go +++ b/modules/storage/sftp/sftp.go @@ -27,10 +27,11 @@ import ( ) type SFTP struct { - client *sftp.Client - backupPath string - name string - rateLimit int64 + client *sftp.Client + name string + backupPath string + rateLimit int64 + rotateEnabled bool Retention } @@ -89,19 +90,14 @@ func Init(name string, opts Opts, rl int64) (*SFTP, error) { } -func (s *SFTP) IsLocal() int { return 0 } - -func (s *SFTP) SetBackupPath(path string) { - s.backupPath = path -} - -func (s *SFTP) SetRateLimit(rl int64) { - s.rateLimit = rl +func (s *SFTP) Configure(p Params) { + s.backupPath = p.BackupPath + s.rateLimit = p.RateLimit + s.rotateEnabled = p.RotateEnabled + s.Retention = p.Retention } -func (s *SFTP) SetRetention(r Retention) { - s.Retention = r -} +func (s *SFTP) IsLocal() int { return 0 } func (s *SFTP) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) (err error) { var ( @@ -204,6 +200,10 @@ func (s *SFTP) deliveryBackupMetadata(logCh chan logger.LogRecord, jobName, tmpB } func (s *SFTP) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error { + if !s.rotateEnabled { + logCh <- logger.Log(job.GetName(), s.name).Debugf("Backup rotate skipped by config.") + return nil + } if job.GetType() == misc.IncFiles { return s.deleteIncBackup(logCh, job.GetName(), ofsPart, full) diff --git a/modules/storage/smb/smb.go b/modules/storage/smb/smb.go index f694354..8276b1c 100755 --- a/modules/storage/smb/smb.go +++ b/modules/storage/smb/smb.go @@ -26,11 +26,12 @@ import ( ) type SMB struct { - session *smb2.Session - share *smb2.Share - backupPath string - name string - rateLimit int64 + session *smb2.Session + share *smb2.Share + name string + backupPath string + rateLimit int64 + rotateEnabled bool Retention } @@ -82,19 +83,14 @@ func Init(sName string, params Opts, rl int64) (s *SMB, err error) { return } -func (s *SMB) IsLocal() int { return 0 } - -func (s *SMB) SetBackupPath(path string) { - s.backupPath = strings.TrimPrefix(path, "/") -} - -func (s *SMB) SetRateLimit(rl int64) { - s.rateLimit = rl +func (s *SMB) Configure(p Params) { + s.backupPath = p.BackupPath + s.rateLimit = p.RateLimit + s.rotateEnabled = p.RotateEnabled + s.Retention = p.Retention } -func (s *SMB) SetRetention(r Retention) { - s.Retention = r -} +func (s *SMB) IsLocal() int { return 0 } func (s *SMB) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) (err error) { @@ -174,6 +170,10 @@ func (s *SMB) copy(logCh chan logger.LogRecord, jobName, srcPath, dstPath string } func (s *SMB) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error { + if !s.rotateEnabled { + logCh <- logger.Log(job.GetName(), s.name).Debugf("Backup rotate skipped by config.") + return nil + } if job.GetType() == misc.IncFiles { return s.deleteIncBackup(logCh, job.GetName(), ofsPart, full) diff --git a/modules/storage/webdav/wedav.go b/modules/storage/webdav/wedav.go index 3251070..e538137 100755 --- a/modules/storage/webdav/wedav.go +++ b/modules/storage/webdav/wedav.go @@ -25,10 +25,11 @@ import ( ) type WebDav struct { - client *webdav.Client - backupPath string - name string - rateLimit int64 + client *webdav.Client + name string + backupPath string + rateLimit int64 + rotateEnabled bool Retention } @@ -60,19 +61,14 @@ func Init(name string, params Opts, rl int64) (*WebDav, error) { }, nil } -func (wd *WebDav) IsLocal() int { return 0 } - -func (wd *WebDav) SetBackupPath(path string) { - wd.backupPath = path -} - -func (wd *WebDav) SetRateLimit(rl int64) { - wd.rateLimit = rl +func (wd *WebDav) Configure(p Params) { + wd.backupPath = p.BackupPath + wd.rateLimit = p.RateLimit + wd.rotateEnabled = p.RotateEnabled + wd.Retention = p.Retention } -func (wd *WebDav) SetRetention(r Retention) { - wd.Retention = r -} +func (wd *WebDav) IsLocal() int { return 0 } func (wd *WebDav) DeliveryBackup(logCh chan logger.LogRecord, jobName, tmpBackupFile, ofs, bakType string) (err error) { @@ -147,6 +143,10 @@ func (wd *WebDav) copy(logCh chan logger.LogRecord, jobName, srcPath, dstPath st } func (wd *WebDav) DeleteOldBackups(logCh chan logger.LogRecord, ofsPart string, job interfaces.Job, full bool) error { + if !wd.rotateEnabled { + logCh <- logger.Log(job.GetName(), wd.name).Debugf("Backup rotate skipped by config.") + return nil + } if job.GetType() == misc.IncFiles { return wd.deleteIncBackup(logCh, job.GetName(), ofsPart, full)