don't make blockquotes smaller/grey

[etc/neomutt.git] / .config / neomutt / buildmimetree.py

diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py
index 2ad671a839d632a87eae622358efab1dccaa086d..f10158fbe01ef0dadf0a5bc61477e03d74b0d39f 100755 (executable)
--- a/.config/neomutt/buildmimetree.py
+++ b/.config/neomutt/buildmimetree.py
@@ -28,6 +28,9 @@
 #   - Pynliner, provides --css-file and thus inline styling of HTML output
 #   - Pygments, then syntax highlighting for fenced code is enabled
 #
 #   - Pynliner, provides --css-file and thus inline styling of HTML output
 #   - Pygments, then syntax highlighting for fenced code is enabled
 #

+# Running tests:
+#   pytest -x buildmimetree.py
+#

 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
 #


@@ -47,10 +50,15 @@ import bs4
 import xml.etree.ElementTree as etree
 import io
 import enum
 import xml.etree.ElementTree as etree
 import io
 import enum

+from contextlib import contextmanager

 from collections import namedtuple, OrderedDict
 from markdown.extensions import Extension
 from markdown.blockprocessors import BlockProcessor
 from collections import namedtuple, OrderedDict
 from markdown.extensions import Extension
 from markdown.blockprocessors import BlockProcessor

-from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
+from markdown.inlinepatterns import (
+    SimpleTextInlineProcessor,
+    ImageInlineProcessor,
+    IMAGE_LINK_RE,
+)

 from email.utils import make_msgid
 from urllib import request
 
 from email.utils import make_msgid
 from urllib import request
 


@@ -172,7 +180,6 @@ def parse_cli_args(*args, **kwargs):
 
 
 class File:
 
 
 class File:

-

     class Op(enum.Enum):
         R = enum.auto()
         W = enum.auto()
     class Op(enum.Enum):
         R = enum.auto()
         W = enum.auto()


@@ -191,10 +198,7 @@ class File:
         if content and not re.search(r"[r+]", mode):
             raise RuntimeError("Cannot specify content without read mode")
 
         if content and not re.search(r"[r+]", mode):
             raise RuntimeError("Cannot specify content without read mode")
 

-        self._cache = {
-            File.Op.R: [content] if content else [],
-            File.Op.W: []
-        }
+        self._cache = {File.Op.R: [content] if content else [], File.Op.W: []}

         self._lastop = None
         self._mode = mode
         self._kwargs = kwargs
         self._lastop = None
         self._mode = mode
         self._kwargs = kwargs


@@ -231,10 +235,6 @@ class File:
         if cache and self._cache[File.Op.R]:
             return self._get_cache(File.Op.R)
 
         if cache and self._cache[File.Op.R]:
             return self._get_cache(File.Op.R)
 

-        if not self._file:
-            with self as f:
-                return f.read(cache=cache)
-

         if self._lastop == File.Op.W:
             try:
                 self._file.seek(0)
         if self._lastop == File.Op.W:
             try:
                 self._file.seek(0)


@@ -250,11 +250,6 @@ class File:
             return self._file.read()
 
     def write(self, s, *, cache=True):
             return self._file.read()
 
     def write(self, s, *, cache=True):

-
-        if not self._file:
-            with self as f:
-                return f.write(s, cache=cache)
-

         if self._lastop == File.Op.R:
             try:
                 self._file.seek(0)
         if self._lastop == File.Op.R:
             try:
                 self._file.seek(0)


@@ -480,18 +475,33 @@ def apply_styling(html, css):
     )
 
 
     )
 
 

+# [ FORMAT=FLOWED HANDLING ] ##################################################
+
+
+class FormatFlowedNewlineExtension(Extension):
+    FFNL_RE = r"(?!\S)(\s)\n"
+
+    def extendMarkdown(self, md):
+        ffnl = SimpleTextInlineProcessor(self.FFNL_RE)
+        md.inlinePatterns.register(ffnl, "ffnl", 125)
+
+

 # [ QUOTE HANDLING ] ##########################################################
 
 
 class QuoteToAdmonitionExtension(Extension):
 # [ QUOTE HANDLING ] ##########################################################
 
 
 class QuoteToAdmonitionExtension(Extension):

-    class EmailQuoteBlockProcessor(BlockProcessor):
+    class BlockProcessor(BlockProcessor):

         RE = re.compile(r"(?:^|\n)>\s*(.*)")
 
         def __init__(self, parser):
             super().__init__(parser)
             self._title = None
         RE = re.compile(r"(?:^|\n)>\s*(.*)")
 
         def __init__(self, parser):
             super().__init__(parser)
             self._title = None

+            self._disable = False

 
         def test(self, parent, blocks):
 
         def test(self, parent, blocks):

+            if self._disable:
+                return False
+

             if markdown.util.nearing_recursion_limit():
                 return False
 
             if markdown.util.nearing_recursion_limit():
                 return False
 


@@ -527,9 +537,14 @@ class QuoteToAdmonitionExtension(Extension):
             self.parser.parseChunk(admonition, self._title)
 
             admonition[0].set("class", "admonition-title")
             self.parser.parseChunk(admonition, self._title)
 
             admonition[0].set("class", "admonition-title")

-            self.parser.parseChunk(
-                admonition, "\n".join(self.clean(line) for line in quotelines)
-            )
+            with self.disable():
+                self.parser.parseChunk(admonition, "\n".join(quotelines))
+
+        @contextmanager
+        def disable(self):
+            self._disable = True
+            yield True
+            self._disable = False

 
         @classmethod
         def clean(klass, line):
 
         @classmethod
         def clean(klass, line):


@@ -538,7 +553,7 @@ class QuoteToAdmonitionExtension(Extension):
 
     def extendMarkdown(self, md):
         md.registerExtension(self)
 
     def extendMarkdown(self, md):
         md.registerExtension(self)

-        email_quote_proc = self.EmailQuoteBlockProcessor(md.parser)
+        email_quote_proc = self.BlockProcessor(md.parser)

         md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25)
 
 
         md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25)
 
 


@@ -644,7 +659,9 @@ def extract_signature(text, *, filefactory=FileFactory()):
         path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1])
         textsig = "\n".join(lines)
 
         path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1])
         textsig = "\n".join(lines)
 

-        sig_input = filefactory(path.expanduser()).read()
+        with filefactory(path.expanduser()) as sig_f:
+            sig_input = sig_f.read()
+

         soup = bs4.BeautifulSoup(sig_input, "html.parser")
 
         style = str(soup.style.extract()) if soup.style else ""
         soup = bs4.BeautifulSoup(sig_input, "html.parser")
 
         style = str(soup.style.extract()) if soup.style else ""


@@ -691,6 +708,7 @@ def convert_markdown_to_html(
     ] = _CODEHILITE_CLASS
 
     extensions = extensions or []
     ] = _CODEHILITE_CLASS
 
     extensions = extensions or []

+    extensions.append(FormatFlowedNewlineExtension())

     extensions.append(QuoteToAdmonitionExtension())
 
     draft = draft_f.read()
     extensions.append(QuoteToAdmonitionExtension())
 
     draft = draft_f.read()


@@ -958,14 +976,6 @@ def do_massage(
     # variable used to identify the command file we're currently writing
     # to.
     cmds = MuttCommands(cmd_f, debug=debug_commands)
     # variable used to identify the command file we're currently writing
     # to.
     cmds = MuttCommands(cmd_f, debug=debug_commands)

-    cmds.cmd('set editor="$my_editor"')
-    cmds.cmd('set edit_headers="$my_edit_headers"')
-    cmds.cmd("unset my_editor")
-    cmds.cmd("unset my_edit_headers")
-
-    # let's flush those commands, as there'll be a lot of pushes from now
-    # on, which need to be run in reverse order
-    cmds.flush()

 
     extensions = extensions.split(",") if extensions else []
     tree = converter(
 
     extensions = extensions.split(",") if extensions else []
     tree = converter(


@@ -1111,6 +1121,10 @@ def do_massage(
     except AttributeError:
         filename = "pytest_internal_file"
     cmds.cmd(f"source 'rm -f {filename}|'")
     except AttributeError:
         filename = "pytest_internal_file"
     cmds.cmd(f"source 'rm -f {filename}|'")

+    cmds.cmd('set editor="$my_editor"')
+    cmds.cmd('set edit_headers="$my_edit_headers"')
+    cmds.cmd("unset my_editor")
+    cmds.cmd("unset my_edit_headers")

     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
     cmds.flush()
 
     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
     cmds.flush()
 


@@ -1405,14 +1419,14 @@ try:
                 )
                 lines = cmd_f.read().splitlines()
 
                 )
                 lines = cmd_f.read().splitlines()
 

-            assert '="$my_editor"' in lines.pop(0)
-            assert '="$my_edit_headers"' in lines.pop(0)
-            assert "unset my_editor" == lines.pop(0)
-            assert "unset my_edit_headers" == lines.pop(0)

             assert "send-message" in lines.pop(0)
             assert "update-encoding" in lines.pop(0)
             assert "first-entry" in lines.pop(0)
             assert "source 'rm -f " in lines.pop(0)
             assert "send-message" in lines.pop(0)
             assert "update-encoding" in lines.pop(0)
             assert "first-entry" in lines.pop(0)
             assert "source 'rm -f " in lines.pop(0)

+            assert '="$my_editor"' in lines.pop(0)
+            assert '="$my_edit_headers"' in lines.pop(0)
+            assert "unset my_editor" == lines.pop(0)
+            assert "unset my_edit_headers" == lines.pop(0)

             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 
         @pytest.mark.massage
             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 
         @pytest.mark.massage


@@ -1429,7 +1443,7 @@ try:
                     max_other_attachments=max_attachments,
                     converter=converter,
                 )
                     max_other_attachments=max_attachments,
                     converter=converter,
                 )

-                lines = cmd_f.read().splitlines()[4:-2]
+                lines = cmd_f.read().splitlines()[:-6]

 
             assert "first-entry" in lines.pop()
             assert "update-encoding" in lines.pop()
 
             assert "first-entry" in lines.pop()
             assert "update-encoding" in lines.pop()


@@ -1471,7 +1485,7 @@ try:
                     cmd_f=cmd_f,
                     converter=converter,
                 )
                     cmd_f=cmd_f,
                     converter=converter,
                 )

-                lines = cmd_f.read().splitlines()[4:-2]
+                lines = cmd_f.read().splitlines()[:-6]

 
             assert "first-entry" in lines.pop()
             assert "update-encoding" in lines.pop()
 
             assert "first-entry" in lines.pop()
             assert "update-encoding" in lines.pop()


@@ -1541,10 +1555,10 @@ try:
 
         @pytest.mark.converter
         def test_converter_tree_basic(self, fakepath, const1, fakefilefactory):
 
         @pytest.mark.converter
         def test_converter_tree_basic(self, fakepath, const1, fakefilefactory):

-            draft_f = fakefilefactory(fakepath, content=const1)
-            tree = convert_markdown_to_html(
-                draft_f, filefactory=fakefilefactory
-            )
+            with fakefilefactory(fakepath, content=const1) as draft_f:
+                tree = convert_markdown_to_html(
+                    draft_f, filefactory=fakefilefactory
+                )

 
             assert tree.subtype == "alternative"
             assert len(tree.children) == 2
 
             assert tree.subtype == "alternative"
             assert len(tree.children) == 2


@@ -1558,8 +1572,8 @@ try:
         def test_converter_writes(
             self, fakepath, fakefilefactory, const1, monkeypatch
         ):
         def test_converter_writes(
             self, fakepath, fakefilefactory, const1, monkeypatch
         ):

-            draft_f = fakefilefactory(fakepath, content=const1)
-            convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
+            with fakefilefactory(fakepath, content=const1) as draft_f:
+                convert_markdown_to_html(draft_f, filefactory=fakefilefactory)

 
             html = fakefilefactory.pop()
             assert fakepath.with_suffix(".html") == html[0]
 
             html = fakefilefactory.pop()
             assert fakepath.with_suffix(".html") == html[0]


@@ -1753,7 +1767,6 @@ try:
             @pytest.mark.styling
             def test_massage_styling_to_converter(self):
                 css = "p { color:red }"
             @pytest.mark.styling
             def test_massage_styling_to_converter(self):
                 css = "p { color:red }"

-                css_f = File(content=css)

                 css_applied = []
 
                 def converter(draft_f, css_f, **kwargs):
                 css_applied = []
 
                 def converter(draft_f, css_f, **kwargs):


@@ -1761,12 +1774,17 @@ try:
                     css_applied.append(css)
                     return Part("text", "plain", draft_f.path, orig=True)
 
                     css_applied.append(css)
                     return Part("text", "plain", draft_f.path, orig=True)
 

-                do_massage(
-                    draft_f=File(),
-                    cmd_f=File(),
-                    css_f=css_f,
-                    converter=converter,
-                )
+                with (
+                    File() as draft_f,
+                    File(mode="w") as cmd_f,
+                    File(content=css) as css_f,
+                ):
+                    do_massage(
+                        draft_f=draft_f,
+                        cmd_f=cmd_f,
+                        css_f=css_f,
+                        converter=converter,
+                    )

                 assert css_applied[0] == css
 
             @pytest.mark.converter
                 assert css_applied[0] == css
 
             @pytest.mark.converter


@@ -1835,11 +1853,10 @@ try:
             assert htmlsig == sigconst.format(path=fakepath)
 
         @pytest.mark.sig
             assert htmlsig == sigconst.format(path=fakepath)
 
         @pytest.mark.sig

-        def test_signature_extraction_file_not_found(self, const1):
-            path = pathlib.Path("/does/not/exist")
+        def test_signature_extraction_file_not_found(self, fakepath, const1):

             with pytest.raises(FileNotFoundError):
                 origtext, textsig, htmlsig = extract_signature(
             with pytest.raises(FileNotFoundError):
                 origtext, textsig, htmlsig = extract_signature(

-                    f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{path}\n{const1}"
+                    f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{fakepath}\n{const1}"

                 )
 
         @pytest.mark.imgproc
                 )
 
         @pytest.mark.imgproc


@@ -1908,9 +1925,7 @@ try:
                 "This is the plain-text version",
             )
             htmlsig = "HTML Signature from {path} but as a string"
                 "This is the plain-text version",
             )
             htmlsig = "HTML Signature from {path} but as a string"

-            html = (
-                f'<div id="signature"><p>{htmlsig.format(path=fakepath2)}</p></div>'
-            )
+            html = f'<div id="signature"><p>{htmlsig.format(path=fakepath2)}</p></div>'

 
             sig_f = fakefilefactory(fakepath2, content=html)
 
 
             sig_f = fakefilefactory(fakepath2, content=html)
 


@@ -2044,6 +2059,29 @@ try:
             p = quote.p.extract()
             assert p.contents[1].name == "strong"
 
             p = quote.p.extract()
             assert p.contents[1].name == "strong"
 

+        @pytest.mark.converter
+        def test_converter_attribution_to_admonition_with_blockquote(
+            self, fakepath, fakefilefactory
+        ):
+            mailparts = (
+                "Regarding whatever",
+                "> blockquote line1",
+                "> blockquote line2",
+                "> ",
+                "> new para with **bold** text",
+            )
+            with fakefilefactory(
+                fakepath, content="\n".join(mailparts)
+            ) as draft_f:
+                convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
+
+            soup = bs4.BeautifulSoup(
+                fakefilefactory[fakepath.with_suffix(".html")].read(),
+                "html.parser",
+            )
+            quote = soup.select_one("div.admonition.quote")
+            assert quote.blockquote
+

         @pytest.mark.converter
         def test_converter_attribution_to_admonition_multiple(
             self, fakepath, fakefilefactory
         @pytest.mark.converter
         def test_converter_attribution_to_admonition_multiple(
             self, fakepath, fakefilefactory


@@ -2086,6 +2124,46 @@ try:
                 == mailparts[-2]
             )
 
                 == mailparts[-2]
             )
 

+        @pytest.mark.converter
+        def test_converter_format_flowed_with_nl2br(
+            self, fakepath, fakefilefactory
+        ):
+            mailparts = (
+                "This is format=flowed text ",
+                "with spaces at the end ",
+                "and there ought be no newlines.",
+                "",
+                "[link](https://example.org) ",
+                "and text.",
+                "",
+                "[link text ",
+                "broken up](https://example.org).",
+                "",
+                "This is on a new line with a hard break  ",
+                "due to the double space",
+            )
+            with fakefilefactory(
+                fakepath, content="\n".join(mailparts)
+            ) as draft_f:
+                convert_markdown_to_html(
+                    draft_f, extensions=["nl2br"], filefactory=fakefilefactory
+                )
+
+            soup = bs4.BeautifulSoup(
+                fakefilefactory[fakepath.with_suffix(".html")].read(),
+                "html.parser",
+            )
+            import ipdb
+
+            p = soup.p.extract().text
+            assert "".join(mailparts[0:3]) == p
+            p = ''.join(map(str, soup.p.extract().contents))
+            assert p == '<a href="https://example.org">link</a> and text.'
+            p = ''.join(map(str, soup.p.extract().contents))
+            assert (
+                p == '<a href="https://example.org">link text broken up</a>.'
+            )
+

         @pytest.mark.fileio
         def test_file_class_contextmanager(self, const1, monkeypatch):
             state = dict(o=False, c=False)
         @pytest.mark.fileio
         def test_file_class_contextmanager(self, const1, monkeypatch):
             state = dict(o=False, c=False)


@@ -2113,6 +2191,11 @@ try:
                 f.write(const1, cache=False)
                 assert f.read(cache=False) == const1
 
                 f.write(const1, cache=False)
                 assert f.read(cache=False) == const1
 

+        @pytest.mark.fileio
+        def test_file_class_path_no_exists(self, fakepath):
+            with pytest.raises(FileNotFoundError):
+                File(fakepath, mode="r").open()
+

         @pytest.mark.fileio
         def test_file_class_cache(self, tmp_path, const1, const2):
             path = tmp_path / "file"
         @pytest.mark.fileio
         def test_file_class_cache(self, tmp_path, const1, const2):
             path = tmp_path / "file"