Refactor em strong to consolidate code and fix issue #792

docs/change_log/release-3.2.md

@@ -0,0 +1,34 @@
+title: Release Notes for v3.2
+
+# Python-Markdown 3.2 Release Notes
+
+Python-Markdown version 3.2 supports Python versions 2.7, 3.5, 3.6, 3.7,
+PyPy and PyPy3.
+
+## Backwards-incompatible changes
+
+## `em` and `strong` inline processor changes
+
+In order to fix issue #792, `em`/`strong` inline processors were refactored. This
+translated into removing many of the existing inline processors that handled this
+logic:
+
+* `em_strong`
+* `strong`
+* `emphasis`
+* `strong2`
+* `emphasis`
+
+These processors were replaced with two new ones:
+
+* `em_strong`
+* `em_strong2`
+
+The [`legacy_em`](../extensions/legacy_em.md) extension was also modified with new,
+refactored logic and simply overrides the `em_strong2` inline processor.
+
+## Bug fixes
+
+The following bug fixes are included in the 3.2 release:
+
+* Refactor bold and italic logic in order to solve complex nesting issues (#792).

markdown/extensions/legacy_em.py

@@ -13,18 +13,39 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 from . import Extension
-from ..inlinepatterns import SimpleTagInlineProcessor
+from ..inlinepatterns import UnderscoreProcessor, EmStrongItem, EM_STRONG2_RE, STRONG_EM2_RE
+import re
 
-EMPHASIS_RE = r'(\*|_)(.+?)\1'
-STRONG_RE = r'(\*{2}|_{2})(.+?)\1'
+# _emphasis_
+EMPHASIS_RE = r'(_)([^_]+)\1'
+
+# __strong__
+STRONG_RE = r'(_{2})(.+?)\1'
+
+# __strong_em___
+STRONG_EM_RE = r'(_)\1(?!\1)(.+?)\1(?!\1)(.+?)\1{3}'
+
+
+class LegacyUnderscoreProcessor(UnderscoreProcessor):
+    """Emphasis processor for handling strong and em matches inside underscores."""
+
+    PATTERNS = [
+        EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'),
+        EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'),
+        EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'),
+        EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'),
+        EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em')
+    ]
 
 
 class LegacyEmExtension(Extension):
     """ Add legacy_em extension to Markdown class."""
 
     def extendMarkdown(self, md):
         """ Modify inline patterns. """
-        md.inlinePatterns.register(SimpleTagInlineProcessor(STRONG_RE, 'strong'), 'strong', 40)
-        md.inlinePatterns.register(SimpleTagInlineProcessor(EMPHASIS_RE, 'em'), 'emphasis', 30)
-        md.inlinePatterns.deregister('strong2')
-        md.inlinePatterns.deregister('emphasis2')
+        md.inlinePatterns.register(LegacyUnderscoreProcessor(r'_'), 'em_strong2', 50)
+
+
+def makeExtension(**kwargs):  # pragma: no cover
+    """ Return an instance of the LegacyEmExtension """
+    return LegacyEmExtension(**kwargs)

markdown/inlinepatterns.py

@@ -64,6 +64,7 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 from . import util
+from collections import namedtuple
 import re
 try:  # pragma: no cover
     from html import entities
@@ -91,12 +92,8 @@ def build_inlinepatterns(md, **kwargs):
     inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90)
     inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80)
     inlinePatterns.register(SimpleTextInlineProcessor(NOT_STRONG_RE), 'not_strong', 70)
-    inlinePatterns.register(DoubleTagInlineProcessor(EM_STRONG_RE, 'strong,em'), 'em_strong', 60)
-    inlinePatterns.register(DoubleTagInlineProcessor(STRONG_EM_RE, 'em,strong'), 'strong_em', 50)
-    inlinePatterns.register(SimpleTagInlineProcessor(STRONG_RE, 'strong'), 'strong', 40)
-    inlinePatterns.register(SimpleTagInlineProcessor(EMPHASIS_RE, 'em'), 'emphasis', 30)
-    inlinePatterns.register(SimpleTagInlineProcessor(SMART_STRONG_RE, 'strong'), 'strong2', 20)
-    inlinePatterns.register(SimpleTagInlineProcessor(SMART_EMPHASIS_RE, 'em'), 'emphasis2', 10)
+    inlinePatterns.register(AsteriskProcessor(r'\*'), 'em_strong', 60)
+    inlinePatterns.register(UnderscoreProcessor(r'_'), 'em_strong2', 50)
     return inlinePatterns
 
 
@@ -125,11 +122,23 @@ def build_inlinepatterns(md, **kwargs):
 # _smart_emphasis_
 SMART_EMPHASIS_RE = r'(?<!\w)(_)(?!_)(.+?)(?<!_)\1(?!\w)'
 
+# __strong _em__
+SMART_STRONG_EM_RE = r'(?<!\w)(\_)\1(?!\1)(.+?)(?<!\w)\1(?!\1)(.+?)\1{3}(?!\w)'
+
 # ***strongem*** or ***em*strong**
-EM_STRONG_RE = r'(\*|_)\1{2}(.+?)\1(.*?)\1{2}'
+EM_STRONG_RE = r'(\*)\1{2}(.+?)\1(.*?)\1{2}'
+
+# ___strongem___ or ___em_strong__
+EM_STRONG2_RE = r'(_)\1{2}(.+?)\1(.*?)\1{2}'
 
 # ***strong**em*
-STRONG_EM_RE = r'(\*|_)\1{2}(.+?)\1{2}(.*?)\1'
+STRONG_EM_RE = r'(\*)\1{2}(.+?)\1{2}(.*?)\1'
+
+# ___strong__em_
+STRONG_EM2_RE = r'(_)\1{2}(.+?)\1{2}(.*?)\1'
+
+# __strong_em___
+STRONG_EM3_RE = r'(\*)\1(?!\1)(.+?)\1(?!\1)(.+?)\1{3}'
 
 # [text](url) or [text](<url>) or [text](url "title")
 LINK_RE = NOIMG + r'\['
@@ -171,6 +180,10 @@ def dequote(string):
         return string
 
 
+class EmStrongItem(namedtuple('EmStrongItem', ['pattern', 'builder', 'tags'])):
+    """Emphasis/strong pattern item."""
+
+
 """
 The pattern classes
 -----------------------------------------------------------------------------
@@ -339,7 +352,7 @@ def __init__(self, pattern, tag):
         InlineProcessor.__init__(self, pattern)
         self.tag = tag
 
-    def handleMatch(self, m, data):
+    def handleMatch(self, m, data):  # pragma: no cover
         el = util.etree.Element(self.tag)
         el.text = m.group(2)
         return el, m.start(0), m.end(0)
@@ -395,7 +408,7 @@ class DoubleTagInlineProcessor(SimpleTagInlineProcessor):
     Useful for strong emphasis etc.
 
     """
-    def handleMatch(self, m, data):
+    def handleMatch(self, m, data):  # pragma: no cover
         tag1, tag2 = self.tag.split(",")
         el1 = util.etree.Element(tag1)
         el2 = util.etree.SubElement(el1, tag2)
@@ -431,6 +444,154 @@ def get_stash(m):
         return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text)
 
 
+class AsteriskProcessor(InlineProcessor):
+    """Emphasis processor for handling strong and em matches inside asterisks."""
+
+    PATTERNS = [
+        EmStrongItem(re.compile(EM_STRONG_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'),
+        EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'),
+        EmStrongItem(re.compile(STRONG_EM3_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'),
+        EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'),
+        EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em')
+    ]
+
+    def build_single(self, m, tag, idx):
+        """Return single tag."""
+        el1 = util.etree.Element(tag)
+        text = m.group(2)
+        self.parse_sub_patterns(text, el1, None, idx)
+        return el1
+
+    def build_double(self, m, tags, idx):
+        """Return double tag."""
+
+        tag1, tag2 = tags.split(",")
+        el1 = util.etree.Element(tag1)
+        el2 = util.etree.Element(tag2)
+        text = m.group(2)
+        self.parse_sub_patterns(text, el2, None, idx)
+        el1.append(el2)
+        if len(m.groups()) == 3:
+            text = m.group(3)
+            self.parse_sub_patterns(text, el1, el2, idx)
+        return el1
+
+    def build_double2(self, m, tags, idx):
+        """Return double tags (variant 2): `<strong>text <em>text</em></strong>`."""
+
+        tag1, tag2 = tags.split(",")
+        el1 = util.etree.Element(tag1)
+        el2 = util.etree.Element(tag2)
+        text = m.group(2)
+        self.parse_sub_patterns(text, el1, None, idx)
+        text = m.group(3)
+        el1.append(el2)
+        self.parse_sub_patterns(text, el2, None, idx)
+        return el1
+
+    def parse_sub_patterns(self, data, parent, last, idx):
+        """
+        Parses sub patterns.
+
+        `data` (`str`):
+            text to evaluate.
+
+        `parent` (`etree.Element`):
+            Parent to attach text and sub elements to.
+
+        `last` (`etree.Element`):
+            Last appended child to parent. Can also be None if parent has no children.
+
+        `idx` (`int`):
+            Current pattern index that was used to evaluate the parent.
+
+        """
+
+        offset = 0
+        pos = 0
+
+        length = len(data)
+        while pos < length:
+            # Find the start of potential emphasis or strong tokens
+            if self.compiled_re.match(data, pos):
+                matched = False
+                # See if the we can match an emphasis/strong pattern
+                for index, item in enumerate(self.PATTERNS):
+                    # Only evaluate patterns that are after what was used on the parent
+                    if index <= idx:
+                        continue
+                    m = item.pattern.match(data, pos)
+                    if m:
+                        # Append child nodes to parent
+                        # Text nodes should be appended to the last
+                        # child if present, and if not, it should
+                        # be added as the parent's text node.
+                        text = data[offset:m.start(0)]
+                        if text:
+                            if last is not None:
+                                last.tail = text
+                            else:
+                                parent.text = text
+                        el = self.build_element(m, item.builder, item.tags, index)
+                        parent.append(el)
+                        last = el
+                        # Move our position past the matched hunk
+                        offset = pos = m.end(0)
+                        matched = True
+                if not matched:
+                    # We matched nothing, move on to the next character
+                    pos += 1
+            else:
+                # Increment position as no potential emphasis start was found.
+                pos += 1
+
+        # Append any leftover text as a text node.
+        text = data[offset:]
+        if text:
+            if last is not None:
+                last.tail = text
+            else:
+                parent.text = text
+
+    def build_element(self, m, builder, tags, index):
+        """Element builder."""
+
+        if builder == 'double2':
+            return self.build_double2(m, tags, index)
+        elif builder == 'double':
+            return self.build_double(m, tags, index)
+        else:
+            return self.build_single(m, tags, index)
+
+    def handleMatch(self, m, data):
+        """Parse patterns."""
+
+        el = None
+        start = None
+        end = None
+
+        for index, item in enumerate(self.PATTERNS):
+            m1 = item.pattern.match(data, m.start(0))
+            if m1:
+                start = m1.start(0)
+                end = m1.end(0)
+                el = self.build_element(m1, item.builder, item.tags, index)
+                break
+        return el, start, end
+
+
+class UnderscoreProcessor(AsteriskProcessor):
+    """Emphasis processor for handling strong and em matches inside underscores."""
+
+    PATTERNS = [
+        EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'),
+        EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'),
+        EmStrongItem(re.compile(SMART_STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'),
+        EmStrongItem(re.compile(SMART_STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'),
+        EmStrongItem(re.compile(SMART_EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em')
+    ]
+
+
 class LinkInlineProcessor(InlineProcessor):
     """ Return a link element from the given match. """
     RE_LINK = re.compile(r'''\(\s*(?:(<[^<>]*>)\s*(?:('[^']*'|"[^"]*")\s*)?\))?''', re.DOTALL | re.UNICODE)

mkdocs.yml

@@ -40,6 +40,7 @@ nav:
   - Test Tools: test_tools.md
   - Contributing to Python-Markdown: contributing.md
   - Change Log: change_log/index.md
+  - Release Notes for v.3.2: change_log/release-3.2.md
   - Release Notes for v.3.1: change_log/release-3.1.md
   - Release Notes for v.3.0: change_log/release-3.0.md
   - Release Notes for v.2.6: change_log/release-2.6.md

tests/test_syntax/extensions/test_legacy_em.py

@@ -38,3 +38,17 @@ def test_legacy_strong(self):
             '<p><strong>connected</strong>words__</p>',
             extensions=['legacy_em']
         )
+
+    def test_complex_emphasis_underscore(self):
+        self.assertMarkdownRenders(
+            'This is text __bold _italic bold___ with more text',
+            '<p>This is text <strong>bold <em>italic bold</em></strong> with more text</p>',
+            extensions=['legacy_em']
+        )
+
+    def test_complex_emphasis_underscore_mid_word(self):
+        self.assertMarkdownRenders(
+            'This is text __bold_italic bold___ with more text',
+            '<p>This is text <strong>bold<em>italic bold</em></strong> with more text</p>',
+            extensions=['legacy_em']
+        )

tests/test_syntax/inline/test_emphasis.py

@@ -84,3 +84,34 @@ def test_standalone_understores_at_begin_end(self):
             '_ bar _',
             '<p>_ bar _</p>'
         )
+
+    def test_complex_emphasis_asterisk(self):
+        self.assertMarkdownRenders(
+            'This is text **bold *italic bold*** with more text',
+            '<p>This is text <strong>bold <em>italic bold</em></strong> with more text</p>'
+        )
+
+    def test_complex_emphasis_asterisk_mid_word(self):
+        self.assertMarkdownRenders(
+            'This is text **bold*italic bold*** with more text',
+            '<p>This is text <strong>bold<em>italic bold</em></strong> with more text</p>'
+        )
+
+    def test_complex_emphasis_smart_underscore(self):
+        self.assertMarkdownRenders(
+            'This is text __bold _italic bold___ with more text',
+            '<p>This is text <strong>bold <em>italic bold</em></strong> with more text</p>'
+        )
+
+    def test_complex_emphasis_smart_underscore_mid_word(self):
+        self.assertMarkdownRenders(
+            'This is text __bold_italic bold___ with more text',
+            '<p>This is text __bold_italic bold___ with more text</p>'
+        )
+
+    def test_nested_emphasis(self):
+
+        self.assertMarkdownRenders(
+            'This text is **bold *italic* *italic* bold**',
+            '<p>This text is <strong>bold <em>italic</em> <em>italic</em> bold</strong></p>'
+        )