Moved background jobs to separate queues, with a separate worker thre…

…ad per nginx worker process for each queue
ledgetech · May 27, 2016 · e054d69 · e054d69
1 parent 09ed938
commit e054d69
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 3 deletions.
diff --git a/lib/ledge/jobs/purge.lua b/lib/ledge/jobs/purge.lua
@@ -86,7 +86,7 @@ function _M.expire_pattern(cursor, job)
                     _M.put_background_job(
                         job.redis_params,
                         job.redis_qless_database,
-                        "ledge",
+                        "ledge_revalidate",
                         "ledge.jobs.revalidate", {
                             uri = uri,
                             entity_keys = entity_keys,

diff --git a/lib/ledge/ledge.lua b/lib/ledge/ledge.lua
@@ -400,6 +400,20 @@ function _M.run_workers(self, options)
         reserver = "ordered",
         queues = { "ledge" },
     })
+
+    worker:start({
+        interval = options.interval or 1,
+        concurrency = options.concurrency or 1,
+        reserver = "ordered",
+        queues = { "ledge_purge" },
+    })
+
+    worker:start({
+        interval = options.interval or 1,
+        concurrency = options.concurrency or 1,
+        reserver = "ordered",
+        queues = { "ledge_revalidate" },
+    })
 end
 
 
@@ -2476,7 +2490,7 @@ function _M.revalidate_in_background(self, update_revalidation_data)
 
     -- Schedule the background job (immediately). jid is a function of the
     -- URI for automatic de-duping.
-    self:put_background_job("ledge", "ledge.jobs.revalidate", {
+    self:put_background_job("ledge_revalidate", "ledge.jobs.revalidate", {
         uri = ngx_var.request_uri,
         entity_keys = entity_keys,
     }, {
@@ -2696,7 +2710,7 @@ function _M.purge(self)
 
     -- Do we have asterisks?
     if ngx_re_find(key_chain.root, "\\*", "soj") then
-        self:put_background_job("ledge", "ledge.jobs.purge", {
+        self:put_background_job("ledge_purge", "ledge.jobs.purge", {
             key_chain = key_chain,
             keyspace_scan_count = self:config_get("keyspace_scan_count"),
             revalidate = revalidate,