Add 'alwaysMatchEndPattern' option to end patterns

spec/fixtures/always-match-end-pattern.cson

@@ -0,0 +1,38 @@
+name: "alwaysMatchEndPattern"
+scopeName: "source.always-match-end-pattern"
+patterns: [
+  {
+    begin: 'outer-inner'
+    end: '```'
+    name: 'outer'
+    alwaysMatchEndPattern: true
+    patterns: [
+      {
+        begin: '/\\*'
+        end: '\\*/'
+        name: 'inner'
+      }
+    ]
+  }
+  {
+    begin: 'outer-middle-inner'
+    end: '```'
+    name: 'outer'
+    alwaysMatchEndPattern: true
+    patterns: [
+      {
+        begin: '/\\*'
+        end: '```'
+        name: 'middle'
+        alwaysMatchEndPattern: true
+        patterns: [
+          {
+            begin: '<!--'
+            end: '-->'
+            name: 'inner'
+          }
+        ]
+      }
+    ]
+  }
+]

spec/grammar-spec.coffee

@@ -295,6 +295,35 @@ describe "Grammar tokenization", ->
           expect(lines[4][2]).toEqual value: "}", scopes: ['source.apply-end-pattern-last', 'normal-env', 'scope']
           expect(lines[4][3]).toEqual value: "excentricSyntax }", scopes: ['source.apply-end-pattern-last', 'normal-env']
 
+      describe "when the alwaysMatchEndPattern flag is set in a pattern", ->
+        beforeEach ->
+          grammar = loadGrammarSync('always-match-end-pattern.cson')
+
+        it "attempts to match that end pattern first even when its included patterns have not finished matching", ->
+          lines = grammar.tokenizeLines """
+            outer-inner
+            /*
+            stuff
+            ```
+          """
+
+          expect(lines[1][0]).toEqual value: '/*', scopes: ['source.always-match-end-pattern', 'outer', 'inner']
+          expect(lines[3][0]).toEqual value: '```', scopes: ['source.always-match-end-pattern', 'outer']
+
+        it "attempts to match the outermost pattern", ->
+          lines = grammar.tokenizeLines """
+            outer-middle-inner
+            /*
+            stuff
+            <!--
+            stuff
+            ```
+          """
+
+          expect(lines[1][0]).toEqual value: '/*', scopes: ['source.always-match-end-pattern', 'outer', 'middle']
+          expect(lines[3][0]).toEqual value: '<!--', scopes: ['source.always-match-end-pattern', 'outer', 'middle', 'inner']
+          expect(lines[5][0]).toEqual value: '```', scopes: ['source.always-match-end-pattern', 'outer']
+
       describe "when the end pattern contains a back reference", ->
         it "constructs the end rule based on its back-references to captures in the begin rule", ->
           grammar = registry.grammarForScopeName('source.ruby')

src/pattern.coffee

@@ -8,7 +8,7 @@ module.exports =
 class Pattern
   constructor: (@grammar, @registry, options={}) ->
     {name, contentName, match, begin, end, patterns} = options
-    {captures, beginCaptures, endCaptures, applyEndPatternLast} = options
+    {captures, beginCaptures, endCaptures, applyEndPatternLast, @alwaysMatchEndPattern} = options
     {@include, @popRule, @hasBackReferences} = options
 
     @pushRule = null
@@ -25,8 +25,8 @@ class Pattern
     else if begin
       @regexSource = begin
       @captures = beginCaptures ? captures
-      endPattern = @grammar.createPattern({match: end, captures: endCaptures ? captures, popRule: true})
-      @pushRule = @grammar.createRule({@scopeName, @contentScopeName, patterns, endPattern, applyEndPatternLast})
+      endPattern = @grammar.createPattern({match: end, captures: endCaptures ? captures, popRule: true, @alwaysMatchEndPattern})
+      @pushRule = @grammar.createRule({@scopeName, @contentScopeName, patterns, endPattern, applyEndPatternLast, @alwaysMatchEndPattern})
 
     if @captures?
       for group, capture of @captures
@@ -92,7 +92,7 @@ class Pattern
       else
         "\\#{index}"
 
-    @grammar.createPattern({hasBackReferences: false, match: resolvedMatch, @captures, @popRule})
+    @grammar.createPattern({hasBackReferences: false, match: resolvedMatch, @captures, @popRule, @alwaysMatchEndPattern})
 
   ruleForInclude: (baseGrammar, name) ->
     hashIndex = name.indexOf('#')
@@ -132,38 +132,61 @@ class Pattern
       else
         match
 
-  handleMatch: (stack, line, captureIndices, rule, endPatternMatch) ->
+  handleMatch: (stack, line, captureIndices, override) ->
     tags = []
 
     zeroWidthMatch = captureIndices[0].start is captureIndices[0].end
 
-    if @popRule
+    if @popRule and override
       # Pushing and popping a rule based on zero width matches at the same index
       # leads to an infinite loop. We bail on parsing if we detect that case here.
       if zeroWidthMatch and _.last(stack).zeroWidthMatch and _.last(stack).rule.anchorPosition is captureIndices[0].end
         return false
 
-      {contentScopeName} = _.last(stack)
-      tags.push(@grammar.endIdForScope(contentScopeName)) if contentScopeName
-    else if @scopeName
-      scopeName = @resolveScopeName(@scopeName, line, captureIndices)
-      tags.push(@grammar.startIdForScope(scopeName))
+      if @captures
+        tags.push(@tagsForCaptureIndices(line, _.clone(captureIndices), captureIndices, stack)...)
+      else
+        {start, end} = captureIndices[0]
+        tags.push(end - start) unless end is start
 
-    if @captures
-      tags.push(@tagsForCaptureIndices(line, _.clone(captureIndices), captureIndices, stack)...)
-    else
-      {start, end} = captureIndices[0]
-      tags.push(end - start) unless end is start
-
-    if @pushRule
-      ruleToPush = @pushRule.getRuleToPush(line, captureIndices)
-      ruleToPush.anchorPosition = captureIndices[0].end
-      {contentScopeName} = ruleToPush
-      stack.push({rule: ruleToPush, scopeName, contentScopeName, zeroWidthMatch})
-      tags.push(@grammar.startIdForScope(contentScopeName)) if contentScopeName
+      overrideTags = []
+      while stack.length
+        {contentScopeName, scopeName, rule} = stack.pop() if @popRule
+        overrideTags.push(@grammar.endIdForScope(contentScopeName)) if contentScopeName
+
+        break if rule.alwaysMatchEndPattern and rule.endPattern is this
+
+        overrideTags.push(@grammar.endIdForScope(scopeName)) if scopeName
+
+      tags.unshift(overrideTags...)
     else
-      {scopeName} = stack.pop() if @popRule
-      tags.push(@grammar.endIdForScope(scopeName)) if scopeName
+      if @popRule
+        # Pushing and popping a rule based on zero width matches at the same index
+        # leads to an infinite loop. We bail on parsing if we detect that case here.
+        if zeroWidthMatch and _.last(stack).zeroWidthMatch and _.last(stack).rule.anchorPosition is captureIndices[0].end
+          return false
+
+        {contentScopeName} = _.last(stack)
+        tags.push(@grammar.endIdForScope(contentScopeName)) if contentScopeName
+      else if @scopeName
+        scopeName = @resolveScopeName(@scopeName, line, captureIndices)
+        tags.push(@grammar.startIdForScope(scopeName))
+
+      if @captures
+        tags.push(@tagsForCaptureIndices(line, _.clone(captureIndices), captureIndices, stack)...)
+      else
+        {start, end} = captureIndices[0]
+        tags.push(end - start) unless end is start
+
+      if @pushRule
+        ruleToPush = @pushRule.getRuleToPush(line, captureIndices)
+        ruleToPush.anchorPosition = captureIndices[0].end
+        {contentScopeName} = ruleToPush
+        stack.push({rule: ruleToPush, scopeName, contentScopeName, zeroWidthMatch})
+        tags.push(@grammar.startIdForScope(contentScopeName)) if contentScopeName
+      else
+        {scopeName} = stack.pop() if @popRule
+        tags.push(@grammar.endIdForScope(scopeName)) if scopeName
 
     tags
 

src/rule.coffee

@@ -4,7 +4,7 @@ Scanner = require './scanner'
 
 module.exports =
 class Rule
-  constructor: (@grammar, @registry, {@scopeName, @contentScopeName, patterns, @endPattern, @applyEndPatternLast}={}) ->
+  constructor: (@grammar, @registry, {@scopeName, @contentScopeName, patterns, @endPattern, @applyEndPatternLast, @alwaysMatchEndPattern}={}) ->
     @patterns = []
     for pattern in patterns ? []
       @patterns.push(@grammar.createPattern(pattern)) unless pattern.disabled
@@ -38,6 +38,14 @@ class Rule
     @scannersByBaseGrammarName[baseGrammar.name] = scanner
     scanner
 
+  getEndPatternScanner: (ruleStack) ->
+    # TODO: Return a cached scanner here if applicable
+    patterns = @getIncludedPatterns(ruleStack.shift().rule.grammar)
+    for stack in ruleStack
+      patterns.push(stack.rule.endPattern) if stack.rule.endPattern?.alwaysMatchEndPattern
+
+    new Scanner(patterns)
+
   scanInjections: (ruleStack, line, position, firstLine) ->
     baseGrammar = ruleStack[0].rule.grammar
     if injections = baseGrammar.injections
@@ -57,7 +65,11 @@ class Rule
     baseGrammar = ruleStack[0].rule.grammar
     results = []
 
-    scanner = @getScanner(baseGrammar)
+    if @endPattern
+      scanner = @getEndPatternScanner(ruleStack.slice())
+    else
+      scanner = @getScanner(baseGrammar)
+
     if result = scanner.findNextMatch(lineWithNewline, firstLine, position, @anchorPosition)
       results.push(result)
 
@@ -103,13 +115,13 @@ class Rule
 
     {index, captureIndices, scanner} = result
     [firstCapture] = captureIndices
-    endPatternMatch = @endPattern is scanner.patterns[index]
-    if nextTags = scanner.handleMatch(result, ruleStack, line, this, endPatternMatch)
+    override = @endPattern isnt scanner.patterns[index] and scanner.patterns[index].alwaysMatchEndPattern
+    if nextTags = scanner.handleMatch(result, ruleStack, line, override)
       {nextTags, tagsStart: firstCapture.start, tagsEnd: firstCapture.end}
 
   getRuleToPush: (line, beginPatternCaptureIndices) ->
     if @endPattern.hasBackReferences
-      rule = @grammar.createRule({@scopeName, @contentScopeName})
+      rule = @grammar.createRule({@scopeName, @contentScopeName, @alwaysMatchEndPattern})
       rule.endPattern = @endPattern.resolveBackReferences(line, beginPatternCaptureIndices)
       rule.patterns = [rule.endPattern, @patterns...]
       rule

src/scanner.coffee

@@ -59,10 +59,9 @@ class Scanner
   # match - An object returned from a previous call to `findNextMatch`.
   # stack - An array of {Rule} objects.
   # line - The string being scanned.
-  # rule - The rule that matched.
-  # endPatternMatch - true if the rule's end pattern matched.
+  # override - true if an endPattern with `alwaysMatchEndPattern` matched.
   #
   # Returns an array of tokens representing the match.
-  handleMatch: (match, stack, line, rule, endPatternMatch) ->
+  handleMatch: (match, stack, line, override) ->
     pattern = @patterns[match.index]
-    pattern.handleMatch(stack, line, match.captureIndices, rule, endPatternMatch)
+    pattern.handleMatch(stack, line, match.captureIndices, override)