preserve format=flowed with nl2br

[etc/neomutt.git] / .config / neomutt / buildmimetree.py

diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py
index c541ed12094e9ff1c22ab147e5eb9824ff9b6f73..f10158fbe01ef0dadf0a5bc61477e03d74b0d39f 100755 (executable)
--- a/.config/neomutt/buildmimetree.py
+++ b/.config/neomutt/buildmimetree.py
@@ -28,6 +28,9 @@
 #   - Pynliner, provides --css-file and thus inline styling of HTML output
 #   - Pygments, then syntax highlighting for fenced code is enabled
 #
 #   - Pynliner, provides --css-file and thus inline styling of HTML output
 #   - Pygments, then syntax highlighting for fenced code is enabled
 #

+# Running tests:
+#   pytest -x buildmimetree.py
+#

 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
 #


@@ -47,10 +50,15 @@ import bs4
 import xml.etree.ElementTree as etree
 import io
 import enum
 import xml.etree.ElementTree as etree
 import io
 import enum

+from contextlib import contextmanager

 from collections import namedtuple, OrderedDict
 from markdown.extensions import Extension
 from markdown.blockprocessors import BlockProcessor
 from collections import namedtuple, OrderedDict
 from markdown.extensions import Extension
 from markdown.blockprocessors import BlockProcessor

-from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
+from markdown.inlinepatterns import (
+    SimpleTextInlineProcessor,
+    ImageInlineProcessor,
+    IMAGE_LINK_RE,
+)

 from email.utils import make_msgid
 from urllib import request
 
 from email.utils import make_msgid
 from urllib import request
 


@@ -172,7 +180,6 @@ def parse_cli_args(*args, **kwargs):
 
 
 class File:
 
 
 class File:

-

     class Op(enum.Enum):
         R = enum.auto()
         W = enum.auto()
     class Op(enum.Enum):
         R = enum.auto()
         W = enum.auto()


@@ -191,10 +198,7 @@ class File:
         if content and not re.search(r"[r+]", mode):
             raise RuntimeError("Cannot specify content without read mode")
 
         if content and not re.search(r"[r+]", mode):
             raise RuntimeError("Cannot specify content without read mode")
 

-        self._cache = {
-            File.Op.R: [content] if content else [],
-            File.Op.W: []
-        }
+        self._cache = {File.Op.R: [content] if content else [], File.Op.W: []}

         self._lastop = None
         self._mode = mode
         self._kwargs = kwargs
         self._lastop = None
         self._mode = mode
         self._kwargs = kwargs


@@ -246,7 +250,6 @@ class File:
             return self._file.read()
 
     def write(self, s, *, cache=True):
             return self._file.read()
 
     def write(self, s, *, cache=True):

-

         if self._lastop == File.Op.R:
             try:
                 self._file.seek(0)
         if self._lastop == File.Op.R:
             try:
                 self._file.seek(0)


@@ -472,18 +475,33 @@ def apply_styling(html, css):
     )
 
 
     )
 
 

+# [ FORMAT=FLOWED HANDLING ] ##################################################
+
+
+class FormatFlowedNewlineExtension(Extension):
+    FFNL_RE = r"(?!\S)(\s)\n"
+
+    def extendMarkdown(self, md):
+        ffnl = SimpleTextInlineProcessor(self.FFNL_RE)
+        md.inlinePatterns.register(ffnl, "ffnl", 125)
+
+

 # [ QUOTE HANDLING ] ##########################################################
 
 
 class QuoteToAdmonitionExtension(Extension):
 # [ QUOTE HANDLING ] ##########################################################
 
 
 class QuoteToAdmonitionExtension(Extension):

-    class EmailQuoteBlockProcessor(BlockProcessor):
+    class BlockProcessor(BlockProcessor):

         RE = re.compile(r"(?:^|\n)>\s*(.*)")
 
         def __init__(self, parser):
             super().__init__(parser)
             self._title = None
         RE = re.compile(r"(?:^|\n)>\s*(.*)")
 
         def __init__(self, parser):
             super().__init__(parser)
             self._title = None

+            self._disable = False

 
         def test(self, parent, blocks):
 
         def test(self, parent, blocks):

+            if self._disable:
+                return False
+

             if markdown.util.nearing_recursion_limit():
                 return False
 
             if markdown.util.nearing_recursion_limit():
                 return False
 


@@ -519,9 +537,14 @@ class QuoteToAdmonitionExtension(Extension):
             self.parser.parseChunk(admonition, self._title)
 
             admonition[0].set("class", "admonition-title")
             self.parser.parseChunk(admonition, self._title)
 
             admonition[0].set("class", "admonition-title")

-            self.parser.parseChunk(
-                admonition, "\n".join(self.clean(line) for line in quotelines)
-            )
+            with self.disable():
+                self.parser.parseChunk(admonition, "\n".join(quotelines))
+
+        @contextmanager
+        def disable(self):
+            self._disable = True
+            yield True
+            self._disable = False

 
         @classmethod
         def clean(klass, line):
 
         @classmethod
         def clean(klass, line):


@@ -530,7 +553,7 @@ class QuoteToAdmonitionExtension(Extension):
 
     def extendMarkdown(self, md):
         md.registerExtension(self)
 
     def extendMarkdown(self, md):
         md.registerExtension(self)

-        email_quote_proc = self.EmailQuoteBlockProcessor(md.parser)
+        email_quote_proc = self.BlockProcessor(md.parser)

         md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25)
 
 
         md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25)
 
 


@@ -685,6 +708,7 @@ def convert_markdown_to_html(
     ] = _CODEHILITE_CLASS
 
     extensions = extensions or []
     ] = _CODEHILITE_CLASS
 
     extensions = extensions or []

+    extensions.append(FormatFlowedNewlineExtension())

     extensions.append(QuoteToAdmonitionExtension())
 
     draft = draft_f.read()
     extensions.append(QuoteToAdmonitionExtension())
 
     draft = draft_f.read()


@@ -1753,7 +1777,7 @@ try:
                 with (
                     File() as draft_f,
                     File(mode="w") as cmd_f,
                 with (
                     File() as draft_f,
                     File(mode="w") as cmd_f,

-                    File(content=css) as css_f
+                    File(content=css) as css_f,

                 ):
                     do_massage(
                         draft_f=draft_f,
                 ):
                     do_massage(
                         draft_f=draft_f,


@@ -1901,9 +1925,7 @@ try:
                 "This is the plain-text version",
             )
             htmlsig = "HTML Signature from {path} but as a string"
                 "This is the plain-text version",
             )
             htmlsig = "HTML Signature from {path} but as a string"

-            html = (
-                f'<div id="signature"><p>{htmlsig.format(path=fakepath2)}</p></div>'
-            )
+            html = f'<div id="signature"><p>{htmlsig.format(path=fakepath2)}</p></div>'

 
             sig_f = fakefilefactory(fakepath2, content=html)
 
 
             sig_f = fakefilefactory(fakepath2, content=html)
 


@@ -2037,6 +2059,29 @@ try:
             p = quote.p.extract()
             assert p.contents[1].name == "strong"
 
             p = quote.p.extract()
             assert p.contents[1].name == "strong"
 

+        @pytest.mark.converter
+        def test_converter_attribution_to_admonition_with_blockquote(
+            self, fakepath, fakefilefactory
+        ):
+            mailparts = (
+                "Regarding whatever",
+                "> blockquote line1",
+                "> blockquote line2",
+                "> ",
+                "> new para with **bold** text",
+            )
+            with fakefilefactory(
+                fakepath, content="\n".join(mailparts)
+            ) as draft_f:
+                convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
+
+            soup = bs4.BeautifulSoup(
+                fakefilefactory[fakepath.with_suffix(".html")].read(),
+                "html.parser",
+            )
+            quote = soup.select_one("div.admonition.quote")
+            assert quote.blockquote
+

         @pytest.mark.converter
         def test_converter_attribution_to_admonition_multiple(
             self, fakepath, fakefilefactory
         @pytest.mark.converter
         def test_converter_attribution_to_admonition_multiple(
             self, fakepath, fakefilefactory


@@ -2079,6 +2124,46 @@ try:
                 == mailparts[-2]
             )
 
                 == mailparts[-2]
             )
 

+        @pytest.mark.converter
+        def test_converter_format_flowed_with_nl2br(
+            self, fakepath, fakefilefactory
+        ):
+            mailparts = (
+                "This is format=flowed text ",
+                "with spaces at the end ",
+                "and there ought be no newlines.",
+                "",
+                "[link](https://example.org) ",
+                "and text.",
+                "",
+                "[link text ",
+                "broken up](https://example.org).",
+                "",
+                "This is on a new line with a hard break  ",
+                "due to the double space",
+            )
+            with fakefilefactory(
+                fakepath, content="\n".join(mailparts)
+            ) as draft_f:
+                convert_markdown_to_html(
+                    draft_f, extensions=["nl2br"], filefactory=fakefilefactory
+                )
+
+            soup = bs4.BeautifulSoup(
+                fakefilefactory[fakepath.with_suffix(".html")].read(),
+                "html.parser",
+            )
+            import ipdb
+
+            p = soup.p.extract().text
+            assert "".join(mailparts[0:3]) == p
+            p = ''.join(map(str, soup.p.extract().contents))
+            assert p == '<a href="https://example.org">link</a> and text.'
+            p = ''.join(map(str, soup.p.extract().contents))
+            assert (
+                p == '<a href="https://example.org">link text broken up</a>.'
+            )
+

         @pytest.mark.fileio
         def test_file_class_contextmanager(self, const1, monkeypatch):
             state = dict(o=False, c=False)
         @pytest.mark.fileio
         def test_file_class_contextmanager(self, const1, monkeypatch):
             state = dict(o=False, c=False)