Added alerting rule for absence of MDS server

ceph-mixins-bug-id: /issues/62 Signed-off-by: Shubhendu <shtripat@redhat.com>
ceph · May 24, 2019 · 92eac24 · 92eac24
1 parent 864c39b
commit 92eac24
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 0 deletions.
diff --git a/alerts/absent_alerts.libsonnet b/alerts/absent_alerts.libsonnet
@@ -38,6 +38,27 @@
           },
         ],
       },
+      {
+        name: 'ceph-mds-status',
+        rules: [
+          {
+            alert: 'CephMdsMissingReplicas',
+            expr: |||
+              sum(ceph_mds_metadata{%(cephExporterSelector)s} == 1) < 2
+            ||| % $._config,
+            'for': $._config.mdsMissingReplicasAlertTime,
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              message: 'Insufficient replicas for storage metadata service.',
+              description: 'Minimum required replicas for storage metadata service not available. Might affect the working of storage cluster.',
+              storage_type: $._config.storageType,
+              severity_level: 'warning',
+            },
+          },
+        ],
+      },
     ],
   },
 }
diff --git a/config.libsonnet b/config.libsonnet
@@ -21,6 +21,7 @@
     PGRepairAlertTime: '1h',
     mgrMissingReplicasAlertTime: '5m',
     mgrIsAbsentAlertTime: '5m',
+    mdsMissingReplicasAlertTime: '5m',
 
     // Constants
     storageType: 'ceph',

diff --git a/extras/manifests/prometheus-ceph-rules.yaml b/extras/manifests/prometheus-ceph-rules.yaml
@@ -40,6 +40,20 @@ spec:
       for: 5m
       labels:
         severity: warning
+  - name: ceph-mds-status
+    rules:
+    - alert: CephMdsMissingReplicas
+      annotations:
+        description: Minimum required replicas for storage metadata service not available.
+          Might affect the working of storage cluster.
+        message: Insufficient replicas for storage metadata service.
+        severity_level: warning
+        storage_type: ceph
+      expr: |
+        sum(ceph_mds_metadata{job="rook-ceph-mgr"} == 1) < 2
+      for: 5m
+      labels:
+        severity: warning
   - name: quorum-alert.rules
     rules:
     - alert: CephMonQuorumAtRisk

diff --git a/test_alerts.yaml b/test_alerts.yaml
@@ -468,3 +468,41 @@ tests:
              severity_level: 'error'
       - alertname: CephOSDDiskUnavailable
         eval_time: 6m
+  - interval: 1m
+    input_series:
+      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-a", job="rook-ceph-mgr"}'
+        values: '1+0x15'
+      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-b", job="rook-ceph-mgr"}'
+        values: '1+0x05 0+0x10'
+    alert_rule_test:
+      - alertname: CephMdsMissingReplicas
+        eval_time: 4m
+      - alertname: CephMdsMissingReplicas
+        eval_time: 11m
+        exp_alerts:
+         - exp_labels:
+             severity: warning
+           exp_annotations:
+             message: 'Insufficient replicas for storage metadata service.'
+             description: 'Minimum required replicas for storage metadata service not available. Might affect the working of storage cluster.'
+             storage_type: 'ceph'
+             severity_level: 'warning'
+  - interval: 1m
+    input_series:
+      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-a", job="rook-ceph-mgr"}'
+        values: '0+0x10 1+0x05'
+      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-b", job="rook-ceph-mgr"}'
+        values: '1+0x15'
+    alert_rule_test:
+      - alertname: CephMdsMissingReplicas
+        eval_time: 6m
+        exp_alerts:
+         - exp_labels:
+             severity: warning
+           exp_annotations:
+             message: 'Insufficient replicas for storage metadata service.'
+             description: 'Minimum required replicas for storage metadata service not available. Might affect the working of storage cluster.'
+             storage_type: 'ceph'
+             severity_level: 'warning'
+      - alertname: CephMdsMissingReplicas
+        eval_time: 11m