feat: Multi queue consumption

Ref: https://frappe.io/blog/engineering/reducing-memory-footprint-of-frappe-framework
frappe · Aug 7, 2023 · f7a6ac5 · f7a6ac5
1 parent 87efb75
commit f7a6ac5
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 54 deletions.
diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py
@@ -58,6 +58,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
 			"bench_cmd": which("bench"),
 			"skip_redis": skip_redis,
 			"workers": config.get("workers", {}),
+			"multi_queue_consumption": True,
 		}
 	)
 

diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf
@@ -12,7 +12,6 @@ stderr_logfile={{ bench_dir }}/logs/web.error.log
 user={{ user }}
 directory={{ sites_dir }}
 
-{% if use_rq %}
 [program:{{ bench_name }}-frappe-schedule]
 command={{ bench_cmd }} schedule
 priority=3
@@ -23,6 +22,7 @@ stderr_logfile={{ bench_dir }}/logs/schedule.error.log
 user={{ user }}
 directory={{ bench_dir }}
 
+{% if not multi_queue_consumption %}
 [program:{{ bench_name }}-frappe-default-worker]
 command={{ bench_cmd }} worker --queue default
 priority=4
@@ -36,9 +36,10 @@ directory={{ bench_dir }}
 killasgroup=true
 numprocs={{ background_workers }}
 process_name=%(program_name)s-%(process_num)d
+{% endif %}
 
 [program:{{ bench_name }}-frappe-short-worker]
-command={{ bench_cmd }} worker --queue short
+command={{ bench_cmd }} worker --queue short{{',default' if multi_queue_consumption else ''}}
 priority=4
 autostart=true
 autorestart=true
@@ -52,7 +53,7 @@ numprocs={{ background_workers }}
 process_name=%(program_name)s-%(process_num)d
 
 [program:{{ bench_name }}-frappe-long-worker]
-command={{ bench_cmd }} worker --queue long
+command={{ bench_cmd }} worker --queue long{{',default,short' if multi_queue_consumption else ''}}
 priority=4
 autostart=true
 autorestart=true
@@ -81,54 +82,6 @@ numprocs={{ worker_details["background_workers"] or background_workers }}
 process_name=%(program_name)s-%(process_num)d
 {% endfor %}
 
-{% else %}
-[program:{{ bench_name }}-frappe-workerbeat]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule
-priority=3
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/workerbeat.log
-stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log
-user={{ user }}
-directory={{ sites_dir }}
-
-[program:{{ bench_name }}-frappe-worker]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n jobs@%%h -Ofair --soft-time-limit 360 --time-limit 390 --loglevel INFO
-priority=4
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/worker.log
-stderr_logfile={{ bench_dir }}/logs/worker.error.log
-user={{ user }}
-stopwaitsecs=400
-directory={{ sites_dir }}
-killasgroup=true
-
-[program:{{ bench_name }}-frappe-longjob-worker]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n longjobs@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
-priority=2
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/worker.log
-stderr_logfile={{ bench_dir }}/logs/worker.error.log
-user={{ user }}
-stopwaitsecs=1540
-directory={{ sites_dir }}
-killasgroup=true
-
-[program:{{ bench_name }}-frappe-async-worker]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n async@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
-priority=2
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/worker.log
-stderr_logfile={{ bench_dir }}/logs/worker.error.log
-user={{ user }}
-stopwaitsecs=1540
-directory={{ sites_dir }}
-killasgroup=true
-
-{% endif %}
 
 {% if not skip_redis %}
 [program:{{ bench_name }}-redis-cache]
@@ -167,15 +120,16 @@ directory={{ bench_dir }}
 [group:{{ bench_name }}-web]
 programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%}
 
-{% if use_rq %}
+
+{% if multi_queue_consumption %}
 
 [group:{{ bench_name }}-workers]
-programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
+programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
 
 {% else %}
 
 [group:{{ bench_name }}-workers]
-programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
+programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
 
 {% endif %}