Skip to content

Commit

Permalink
ESI include recursion detection now bubbles up to parent requests, to…
Browse files Browse the repository at this point in the history 
… avoid exponential requests when fragments have multiple recursive URIs
  • Loading branch information
@pintsized
pintsized committed Jan 25, 2016
1 parent 0c0fd1a commit 00d1143
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 78 deletions.
146 changes: 91 additions & 55 deletions lib/ledge/esi.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ local tostring, ipairs, pairs, type, tonumber, next, unpack, pcall =
tostring, ipairs, pairs, type, tonumber, next, unpack, pcall

local str_sub = string.sub
local str_find = string.find
local ngx_re_gsub = ngx.re.gsub
local ngx_re_sub = ngx.re.sub
local ngx_re_match = ngx.re.match
Expand Down Expand Up @@ -265,10 +266,12 @@ end


local function esi_fetch_include(include_tag, buffer_size, pre_include_callback, recursion_limit)
-- We track incude recursion, and bail past the limit
-- We track include recursion, and bail past the limit, yielding a special "esi:abort_includes"
-- instruction which the outer process filter checks for.
local recursion_count = tonumber(ngx_req_get_headers()["X-ESI-Recursion-Level"]) or 0
if recursion_count > recursion_limit then
if recursion_count >= recursion_limit then
ngx_log(ngx_ERR, "ESI recursion limit (", recursion_limit, ") exceeded")
co_yield("<esi:abort_includes />")
return nil
end

Expand Down Expand Up @@ -483,65 +486,98 @@ end


function _M.get_process_filter(reader, pre_include_callback, recursion_limit)
return co_wrap(function(buffer_size)
repeat
local chunk, err, has_esi = reader(buffer_size)
local escaped = 0
if chunk then
if has_esi then
-- Remove <!--esi-->
chunk, escaped = ngx_re_gsub(chunk, "(<!--esi(.*?)-->)", "$2", "soj")

-- Remove comments.
chunk = ngx_re_gsub(chunk, "<esi:comment (?:.*?)/>", "", "soj")

-- Remove 'remove' blocks
chunk = ngx_re_gsub(chunk, "(<esi:remove>.*?</esi:remove>)", "", "soj")

-- Evaluate and replace all esi vars
chunk = esi_replace_vars(chunk, buffer_size)

-- Evaluate choose / when / otherwise conditions...
chunk = ngx_re_gsub(chunk, esi_choose_pattern, _esi_gsub_choose, "soj")

-- Find and loop over esi:include tags
local re_ctx = { pos = 1 }
local yield_from = 1
repeat
local from, to, err = ngx_re_find(
chunk,
[[<esi:include\s*src="[^"]+"\s*/>]],
"oj",
re_ctx
)
local recursion_count = tonumber(ngx_req_get_headers()["X-ESI-Recursion-Level"]) or 0

if from then
-- Yield up to the start of the include tag
co_yield(str_sub(chunk, yield_from, from - 1))
ngx_flush()
yield_from = to + 1

-- Fetches and yields the streamed response
esi_fetch_include(
str_sub(chunk, from, to),
buffer_size,
pre_include_callback,
recursion_limit
-- We use an outer coroutine to filter the processed output in case we have to
-- abort recursive includes.
return co_wrap(function(buffer_size)
local esi_abort_flag = false

-- This is the actual process filter coroutine
local inner_reader = co_wrap(function(buffer_size)
repeat
local chunk, err, has_esi = reader(buffer_size)
local escaped = 0
if chunk then
if has_esi then
-- Remove <!--esi-->
chunk, escaped = ngx_re_gsub(chunk, "(<!--esi(.*?)-->)", "$2", "soj")

-- Remove comments.
chunk = ngx_re_gsub(chunk, "<esi:comment (?:.*?)/>", "", "soj")

-- Remove 'remove' blocks
chunk = ngx_re_gsub(chunk, "(<esi:remove>.*?</esi:remove>)", "", "soj")

-- Evaluate and replace all esi vars
chunk = esi_replace_vars(chunk, buffer_size)

-- Evaluate choose / when / otherwise conditions...
chunk = ngx_re_gsub(chunk, esi_choose_pattern, _esi_gsub_choose, "soj")

-- Find and loop over esi:include tags
local re_ctx = { pos = 1 }
local yield_from = 1
repeat
local from, to, err = ngx_re_find(
chunk,
[[<esi:include\s*src="[^"]+"\s*/>]],
"oj",
re_ctx
)
else
if yield_from == 1 then
-- No includes found, yield everything
co_yield(chunk)

if from then
-- Yield up to the start of the include tag
co_yield(str_sub(chunk, yield_from, from - 1))
ngx_flush()
yield_from = to + 1

-- This will be true if an include has previously yielded
-- the "esi:abort_includes instruction.
if esi_abort_flag == false then
-- Fetches and yields the streamed response
esi_fetch_include(
str_sub(chunk, from, to),
buffer_size,
pre_include_callback,
recursion_limit
)
end
else
-- No *more* includes, yield what's left
co_yield(str_sub(chunk, re_ctx.pos, -1))
if yield_from == 1 then
-- No includes found, yield everything
co_yield(chunk)
else
-- No *more* includes, yield what's left
co_yield(str_sub(chunk, re_ctx.pos, -1))
end
end
end

until not from
else
co_yield(chunk)
until not from
else
co_yield(chunk)
end
end
until not chunk
end)

-- Outer filter, which checks for an esi:abort_includes instruction, so that
-- we can handle accidental recursion.
repeat
local chunk, err = inner_reader(buffer_size)
if chunk then
-- If we see an abort instruction, we set a flag to stop further esi:includes.
if str_find(chunk, "<esi:abort_includes", 1, true) then
esi_abort_flag = true
end

-- We don't wish to see abort instructions in the final output, so the the top most
-- request (recursion_count 0) is responsible for removing them.
if recursion_count == 0 then
chunk = ngx_re_gsub(chunk, "<esi:abort_includes />", "", "soj")
end

co_yield(chunk)
end
until not chunk
end)
Expand Down
46 changes: 23 additions & 23 deletions t/10-esi.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1658,33 +1658,33 @@ location /fragment_24_prx {
location /fragment_24 {
default_type text/html;
content_by_lua '
ngx.say("c: ", ngx.req.get_headers()["X-ESI-Recursion-Level"] or "0")
ngx.print("<esi:include src=\\"/esi_24_prx\\" />")
ngx.print("<esi:include src=\\"/esi_24_prx\\" />")
ngx.say("CHILD")
';
}
location /esi_24 {
default_type text/html;
content_by_lua '
ngx.say("p: ", ngx.req.get_headers()["X-ESI-Recursion-Level"] or "0")
ngx.print("<esi:include src=\\"/fragment_24_prx\\" />")
ngx.say("PARENT")
';
}
--- request
GET /esi_24_prx
--- raw_response_headers_unlike: Surrogate-Control: content="ESI/1.0\"\r\n
--- response_body
CHILD
PARENT
CHILD
PARENT
CHILD
PARENT
CHILD
PARENT
CHILD
PARENT
CHILD
PARENT
p: 0
c: 1
p: 2
c: 3
p: 4
c: 5
p: 6
c: 7
p: 8
c: 9
p: 10
--- error_log
ESI recursion limit (10) exceeded
Expand All @@ -1709,28 +1709,28 @@ location /fragment_24_prx {
location /fragment_24 {
default_type text/html;
content_by_lua '
ngx.say("c: ", ngx.req.get_headers()["X-ESI-Recursion-Level"] or "0")
ngx.print("<esi:include src=\\"/esi_24_prx\\" />")
ngx.print("<esi:include src=\\"/esi_24_prx\\" />")
ngx.say("CHILD")
';
}
location /esi_24 {
default_type text/html;
content_by_lua '
ngx.say("p: ", ngx.req.get_headers()["X-ESI-Recursion-Level"] or "0")
ngx.print("<esi:include src=\\"/fragment_24_prx\\" />")
ngx.say("PARENT")
';
}
--- request
GET /esi_24_prx
--- raw_response_headers_unlike: Surrogate-Control: content="ESI/1.0\"\r\n
--- response_body
PARENT
CHILD
PARENT
CHILD
PARENT
CHILD
PARENT
p: 0
c: 1
p: 2
c: 3
p: 4
c: 5
--- error_log
ESI recursion limit (5) exceeded
Expand Down

0 comments on commit 00d1143

Please sign in to comment.