Add settings to split up organizations across exporters

grafana · Oct 4, 2024 · a389c74 · a389c74
1 parent ac7dc97
commit a389c74
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/engine/apps/metrics_exporter/helpers.py b/engine/apps/metrics_exporter/helpers.py
@@ -2,6 +2,7 @@
 import random
 import typing
 
+from django.conf import settings
 from django.core.cache import cache
 from django.utils import timezone
 
@@ -50,7 +51,10 @@ def get_organization_ids():
     if not organizations_ids:
         organizations_ids = get_organization_ids_from_db()
         cache.set(organizations_ids, METRICS_ORGANIZATIONS_IDS, METRICS_ORGANIZATIONS_IDS_CACHE_TIMEOUT)
-    return organizations_ids
+
+    group_id = settings.METRICS_EXPORTER_ORGANIZATION_GROUP_ID
+    group_count = settings.METRICS_EXPORTER_TOTAL_ORGANIZATION_GROUPS
+    return [i for i in organizations_ids if i % group_count == group_id]
 
 
 def is_allowed_to_start_metrics_calculation(organization_id, force=False) -> bool:

diff --git a/engine/settings/base.py b/engine/settings/base.py
@@ -120,6 +120,11 @@
 # List of metrics to collect. Collect all available application metrics by default
 METRICS_TO_COLLECT = getenv_list("METRICS_TO_COLLECT", METRICS_ALL)
 
+# Total number of exporters collecting the same set of metrics
+METRICS_EXPORTER_TOTAL_ORGANIZATION_GROUPS = getenv_integer("METRICS_EXPORTER_TOTAL_ORGANIZATION_GROUPS", 1)
+# ID of this exporter, used to filter which orgs to collect for
+METRICS_EXPORTER_ORGANIZATION_GROUP_ID = getenv_integer("METRICS_EXPORTER_ORGANIZATION_GROUP_ID", 0)
+
 
 # Database
 class DatabaseTypes: