X-Git-Url: https://git.madduck.net/etc/neomutt.git/blobdiff_plain/8727c8d5c6ab58f4f112bac53c1252862a485e19..e75c1b39e98e2f4b4062620ec8ea1e8eede23c65:/.config/neomutt/buildmimetree.py diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py index 32fc17c..f10158f 100755 --- a/.config/neomutt/buildmimetree.py +++ b/.config/neomutt/buildmimetree.py @@ -28,6 +28,9 @@ # - Pynliner, provides --css-file and thus inline styling of HTML output # - Pygments, then syntax highlighting for fenced code is enabled # +# Running tests: +# pytest -x buildmimetree.py +# # Latest version: # https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py # @@ -46,10 +49,16 @@ import mimetypes import bs4 import xml.etree.ElementTree as etree import io +import enum +from contextlib import contextmanager from collections import namedtuple, OrderedDict from markdown.extensions import Extension from markdown.blockprocessors import BlockProcessor -from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE +from markdown.inlinepatterns import ( + SimpleTextInlineProcessor, + ImageInlineProcessor, + IMAGE_LINK_RE, +) from email.utils import make_msgid from urllib import request @@ -171,6 +180,10 @@ def parse_cli_args(*args, **kwargs): class File: + class Op(enum.Enum): + R = enum.auto() + W = enum.auto() + def __init__(self, path=None, mode="r", content=None, **kwargs): if path: if content: @@ -185,8 +198,8 @@ class File: if content and not re.search(r"[r+]", mode): raise RuntimeError("Cannot specify content without read mode") - self._rcache = [content] if content else [] - self._wcache = [] + self._cache = {File.Op.R: [content] if content else [], File.Op.W: []} + self._lastop = None self._mode = mode self._kwargs = kwargs self._file = None @@ -209,48 +222,48 @@ class File: def close(self): self._file.close() self._file = None - self._rcache = self._wcache - - def _get_rcache(self): - return (b"" if "b" in self._mode else "").join(self._rcache) + self._cache[File.Op.R] = self._cache[File.Op.W] + self._lastop = None - def _get_wcache(self): - return (b"" if "b" in self._mode else "").join(self._wcache) + def _get_cache(self, op): + return (b"" if "b" in self._mode else "").join(self._cache[op]) - def _add_to_rcache(self, s): - self._rcache.append(s) - - def _add_to_wcache(self, s): - self._wcache.append(s) + def _add_to_cache(self, op, s): + self._cache[op].append(s) def read(self, *, cache=True): - if cache and self._rcache: - return self._get_rcache() + if cache and self._cache[File.Op.R]: + return self._get_cache(File.Op.R) + + if self._lastop == File.Op.W: + try: + self._file.seek(0) + except io.UnsupportedOperation: + pass - if not self._file: - with self as f: - return f.read(cache=cache) + self._lastop = File.Op.R - self._file.seek(0) if cache: - self._add_to_rcache(self._file.read()) - return self._get_rcache() + self._add_to_cache(File.Op.R, self._file.read()) + return self._get_cache(File.Op.R) else: return self._file.read() def write(self, s, *, cache=True): - if not self._file: - with self as f: - return f.write(s, cache=cache) - - self._file.seek(0) - self._rcache = self._wcache + if self._lastop == File.Op.R: + try: + self._file.seek(0) + except io.UnsupportedOperation: + pass if cache: - self._add_to_wcache(s) + self._add_to_cache(File.Op.W, s) + + self._cache[File.Op.R] = self._cache[File.Op.W] written = self._file.write(s) self._file.flush() + self._lastop = File.Op.W return written path = property(lambda s: s._path) @@ -462,18 +475,33 @@ def apply_styling(html, css): ) +# [ FORMAT=FLOWED HANDLING ] ################################################## + + +class FormatFlowedNewlineExtension(Extension): + FFNL_RE = r"(?!\S)(\s)\n" + + def extendMarkdown(self, md): + ffnl = SimpleTextInlineProcessor(self.FFNL_RE) + md.inlinePatterns.register(ffnl, "ffnl", 125) + + # [ QUOTE HANDLING ] ########################################################## class QuoteToAdmonitionExtension(Extension): - class EmailQuoteBlockProcessor(BlockProcessor): + class BlockProcessor(BlockProcessor): RE = re.compile(r"(?:^|\n)>\s*(.*)") def __init__(self, parser): super().__init__(parser) self._title = None + self._disable = False def test(self, parent, blocks): + if self._disable: + return False + if markdown.util.nearing_recursion_limit(): return False @@ -509,9 +537,14 @@ class QuoteToAdmonitionExtension(Extension): self.parser.parseChunk(admonition, self._title) admonition[0].set("class", "admonition-title") - self.parser.parseChunk( - admonition, "\n".join(self.clean(line) for line in quotelines) - ) + with self.disable(): + self.parser.parseChunk(admonition, "\n".join(quotelines)) + + @contextmanager + def disable(self): + self._disable = True + yield True + self._disable = False @classmethod def clean(klass, line): @@ -520,7 +553,7 @@ class QuoteToAdmonitionExtension(Extension): def extendMarkdown(self, md): md.registerExtension(self) - email_quote_proc = self.EmailQuoteBlockProcessor(md.parser) + email_quote_proc = self.BlockProcessor(md.parser) md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25) @@ -626,7 +659,9 @@ def extract_signature(text, *, filefactory=FileFactory()): path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1]) textsig = "\n".join(lines) - sig_input = filefactory(path.expanduser()).read() + with filefactory(path.expanduser()) as sig_f: + sig_input = sig_f.read() + soup = bs4.BeautifulSoup(sig_input, "html.parser") style = str(soup.style.extract()) if soup.style else "" @@ -673,6 +708,7 @@ def convert_markdown_to_html( ] = _CODEHILITE_CLASS extensions = extensions or [] + extensions.append(FormatFlowedNewlineExtension()) extensions.append(QuoteToAdmonitionExtension()) draft = draft_f.read() @@ -940,14 +976,6 @@ def do_massage( # variable used to identify the command file we're currently writing # to. cmds = MuttCommands(cmd_f, debug=debug_commands) - cmds.cmd('set editor="$my_editor"') - cmds.cmd('set edit_headers="$my_edit_headers"') - cmds.cmd("unset my_editor") - cmds.cmd("unset my_edit_headers") - - # let's flush those commands, as there'll be a lot of pushes from now - # on, which need to be run in reverse order - cmds.flush() extensions = extensions.split(",") if extensions else [] tree = converter( @@ -1093,6 +1121,10 @@ def do_massage( except AttributeError: filename = "pytest_internal_file" cmds.cmd(f"source 'rm -f {filename}|'") + cmds.cmd('set editor="$my_editor"') + cmds.cmd('set edit_headers="$my_edit_headers"') + cmds.cmd("unset my_editor") + cmds.cmd("unset my_edit_headers") cmds.cmd("unset my_mdwn_postprocess_cmd_file") cmds.flush() @@ -1387,14 +1419,14 @@ try: ) lines = cmd_f.read().splitlines() - assert '="$my_editor"' in lines.pop(0) - assert '="$my_edit_headers"' in lines.pop(0) - assert "unset my_editor" == lines.pop(0) - assert "unset my_edit_headers" == lines.pop(0) assert "send-message" in lines.pop(0) assert "update-encoding" in lines.pop(0) assert "first-entry" in lines.pop(0) assert "source 'rm -f " in lines.pop(0) + assert '="$my_editor"' in lines.pop(0) + assert '="$my_edit_headers"' in lines.pop(0) + assert "unset my_editor" == lines.pop(0) + assert "unset my_edit_headers" == lines.pop(0) assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0) @pytest.mark.massage @@ -1411,7 +1443,7 @@ try: max_other_attachments=max_attachments, converter=converter, ) - lines = cmd_f.read().splitlines()[4:-2] + lines = cmd_f.read().splitlines()[:-6] assert "first-entry" in lines.pop() assert "update-encoding" in lines.pop() @@ -1453,7 +1485,7 @@ try: cmd_f=cmd_f, converter=converter, ) - lines = cmd_f.read().splitlines()[4:-2] + lines = cmd_f.read().splitlines()[:-6] assert "first-entry" in lines.pop() assert "update-encoding" in lines.pop() @@ -1523,10 +1555,10 @@ try: @pytest.mark.converter def test_converter_tree_basic(self, fakepath, const1, fakefilefactory): - draft_f = fakefilefactory(fakepath, content=const1) - tree = convert_markdown_to_html( - draft_f, filefactory=fakefilefactory - ) + with fakefilefactory(fakepath, content=const1) as draft_f: + tree = convert_markdown_to_html( + draft_f, filefactory=fakefilefactory + ) assert tree.subtype == "alternative" assert len(tree.children) == 2 @@ -1540,8 +1572,8 @@ try: def test_converter_writes( self, fakepath, fakefilefactory, const1, monkeypatch ): - draft_f = fakefilefactory(fakepath, content=const1) - convert_markdown_to_html(draft_f, filefactory=fakefilefactory) + with fakefilefactory(fakepath, content=const1) as draft_f: + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) html = fakefilefactory.pop() assert fakepath.with_suffix(".html") == html[0] @@ -1735,7 +1767,6 @@ try: @pytest.mark.styling def test_massage_styling_to_converter(self): css = "p { color:red }" - css_f = File(content=css) css_applied = [] def converter(draft_f, css_f, **kwargs): @@ -1743,12 +1774,17 @@ try: css_applied.append(css) return Part("text", "plain", draft_f.path, orig=True) - do_massage( - draft_f=File(), - cmd_f=File(), - css_f=css_f, - converter=converter, - ) + with ( + File() as draft_f, + File(mode="w") as cmd_f, + File(content=css) as css_f, + ): + do_massage( + draft_f=draft_f, + cmd_f=cmd_f, + css_f=css_f, + converter=converter, + ) assert css_applied[0] == css @pytest.mark.converter @@ -1817,11 +1853,10 @@ try: assert htmlsig == sigconst.format(path=fakepath) @pytest.mark.sig - def test_signature_extraction_file_not_found(self, const1): - path = pathlib.Path("/does/not/exist") + def test_signature_extraction_file_not_found(self, fakepath, const1): with pytest.raises(FileNotFoundError): origtext, textsig, htmlsig = extract_signature( - f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{path}\n{const1}" + f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{fakepath}\n{const1}" ) @pytest.mark.imgproc @@ -1890,9 +1925,7 @@ try: "This is the plain-text version", ) htmlsig = "HTML Signature from {path} but as a string" - html = ( - f'

{htmlsig.format(path=fakepath2)}

' - ) + html = f'

{htmlsig.format(path=fakepath2)}

' sig_f = fakefilefactory(fakepath2, content=html) @@ -2026,6 +2059,29 @@ try: p = quote.p.extract() assert p.contents[1].name == "strong" + @pytest.mark.converter + def test_converter_attribution_to_admonition_with_blockquote( + self, fakepath, fakefilefactory + ): + mailparts = ( + "Regarding whatever", + "> blockquote line1", + "> blockquote line2", + "> ", + "> new para with **bold** text", + ) + with fakefilefactory( + fakepath, content="\n".join(mailparts) + ) as draft_f: + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) + + soup = bs4.BeautifulSoup( + fakefilefactory[fakepath.with_suffix(".html")].read(), + "html.parser", + ) + quote = soup.select_one("div.admonition.quote") + assert quote.blockquote + @pytest.mark.converter def test_converter_attribution_to_admonition_multiple( self, fakepath, fakefilefactory @@ -2068,6 +2124,46 @@ try: == mailparts[-2] ) + @pytest.mark.converter + def test_converter_format_flowed_with_nl2br( + self, fakepath, fakefilefactory + ): + mailparts = ( + "This is format=flowed text ", + "with spaces at the end ", + "and there ought be no newlines.", + "", + "[link](https://example.org) ", + "and text.", + "", + "[link text ", + "broken up](https://example.org).", + "", + "This is on a new line with a hard break ", + "due to the double space", + ) + with fakefilefactory( + fakepath, content="\n".join(mailparts) + ) as draft_f: + convert_markdown_to_html( + draft_f, extensions=["nl2br"], filefactory=fakefilefactory + ) + + soup = bs4.BeautifulSoup( + fakefilefactory[fakepath.with_suffix(".html")].read(), + "html.parser", + ) + import ipdb + + p = soup.p.extract().text + assert "".join(mailparts[0:3]) == p + p = ''.join(map(str, soup.p.extract().contents)) + assert p == 'link and text.' + p = ''.join(map(str, soup.p.extract().contents)) + assert ( + p == 'link text broken up.' + ) + @pytest.mark.fileio def test_file_class_contextmanager(self, const1, monkeypatch): state = dict(o=False, c=False) @@ -2095,6 +2191,11 @@ try: f.write(const1, cache=False) assert f.read(cache=False) == const1 + @pytest.mark.fileio + def test_file_class_path_no_exists(self, fakepath): + with pytest.raises(FileNotFoundError): + File(fakepath, mode="r").open() + @pytest.mark.fileio def test_file_class_cache(self, tmp_path, const1, const2): path = tmp_path / "file"