chore: Add missing operator metrics (#5751)

Co-authored-by: csuzhangxc <csuzhangxc@gmail.com>
pingcap · Sep 25, 2024 · 6320ae0 · 6320ae0
1 parent 29f6519
commit 6320ae0
Show file tree

Hide file tree

Showing 17 changed files with 168 additions and 14 deletions.
diff --git a/cmd/br-federation-manager/main.go b/cmd/br-federation-manager/main.go
@@ -133,6 +133,11 @@ func main() {
 
 		initMetrics := func(c Controller) {
 			metrics.ActiveWorkers.WithLabelValues(c.Name()).Set(0)
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Add(0)
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Add(0)
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Add(0)
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Add(0)
+			metrics.WorkerCount.WithLabelValues(c.Name()).Set(float64(cliCfg.Workers))
 		}
 
 		// Initialize all controllers

diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go
@@ -169,6 +169,11 @@ func main() {
 
 		initMetrics := func(c Controller) {
 			metrics.ActiveWorkers.WithLabelValues(c.Name()).Set(0)
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Add(0)
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Add(0)
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Add(0)
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Add(0)
+			metrics.WorkerCount.WithLabelValues(c.Name()).Set(float64(cliCfg.Workers))
 		}
 
 		// Initialize all controllers

diff --git a/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go b/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go
@@ -94,11 +94,21 @@ func (c *Controller) processNextWorkItem() bool {
 	return true
 }
 
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing TidbClusterAutoScaler %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/backup/backup_controller.go b/pkg/controller/backup/backup_controller.go
@@ -124,11 +124,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given backup.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing Backup %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/backupschedule/backup_schedule_controller.go b/pkg/controller/backupschedule/backup_schedule_controller.go
@@ -117,11 +117,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given backupSchedule.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing BackupSchedule %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/dmcluster/dm_cluster_controller.go b/pkg/controller/dmcluster/dm_cluster_controller.go
@@ -138,11 +138,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given dmcluster.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing DMCluster %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/fedvolumebackup/fed_volume_backup_controller.go b/pkg/controller/fedvolumebackup/fed_volume_backup_controller.go
@@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given VolumeBackup.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing VolumeBackup %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go
@@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given VolumeBackupSchedule.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing VolumeBackupSchedule %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/fedvolumerestore/fed_volume_restore_controller.go b/pkg/controller/fedvolumerestore/fed_volume_restore_controller.go
@@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given VolumeRestore.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing VolumeRestore %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/restore/restore_controller.go b/pkg/controller/restore/restore_controller.go
@@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given restore.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing Restore %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/tidbcluster/pod_control.go b/pkg/controller/tidbcluster/pod_control.go
@@ -170,7 +170,7 @@ func (c *PodController) processNextWorkItem() bool {
 	return true
 }
 
-func (c *PodController) sync(key string) (reconcile.Result, error) {
+func (c *PodController) sync(key string) (result reconcile.Result, err error) {
 	ns, name, err := cache.SplitMetaNamespaceKey(key)
 	if err != nil {
 		return reconcile.Result{}, err
@@ -213,6 +213,16 @@ func (c *PodController) sync(key string) (reconcile.Result, error) {
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing TidbCluster pod %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go
@@ -151,11 +151,21 @@ func (c *Controller) processNextWorkItem() bool {
 }
 
 // sync syncs the given tidbcluster.
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing TidbCluster %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/tidbdashboard/tidb_dashboard_controller.go b/pkg/controller/tidbdashboard/tidb_dashboard_controller.go
@@ -123,11 +123,21 @@ func (c *Controller) processNextWorkItem() bool {
 	return true
 }
 
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing TidbDashboard %s (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/tidbinitializer/tidb_initializer_controller.go b/pkg/controller/tidbinitializer/tidb_initializer_controller.go
@@ -112,11 +112,21 @@ func (c *Controller) processNextWorkItem() bool {
 	return true
 }
 
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing TiDBInitializer %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/tidbmonitor/tidb_monitor_controller.go b/pkg/controller/tidbmonitor/tidb_monitor_controller.go
@@ -107,11 +107,21 @@ func (c *Controller) processNextWorkItem() bool {
 	return true
 }
 
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing TidbMonitor %q (%v)", key, duration)
 	}()
 

diff --git a/pkg/controller/tidbngmonitoring/tidb_ng_monitoring_controller.go b/pkg/controller/tidbngmonitoring/tidb_ng_monitoring_controller.go
@@ -122,11 +122,21 @@ func (c *Controller) processNextWorkItem() bool {
 	return true
 }
 
-func (c *Controller) sync(key string) error {
+func (c *Controller) sync(key string) (err error) {
 	startTime := time.Now()
 	defer func() {
 		duration := time.Since(startTime)
 		metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds())
+
+		if err == nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc()
+		} else if perrors.Find(err, controller.IsRequeueError) != nil {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc()
+		} else {
+			metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc()
+			metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc()
+		}
+
 		klog.V(4).Infof("Finished syncing TidbNGMonitoring %s (%v)", key, duration)
 	}()
 

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -22,6 +22,10 @@ const (
 	LabelNamespace = "namespace"
 	LabelName      = "name"
 	LabelComponent = "component"
+
+	LabelError   = "error"
+	LabelRequeue = "requeue"
+	LabelSuccess = "success"
 )
 
 var (