From: martin f. krafft Date: Sun, 3 Sep 2023 19:05:58 +0000 (+0200) Subject: buildmimetree.py: rework of file io X-Git-Url: https://git.madduck.net/etc/neomutt.git/commitdiff_plain/8727c8d5c6ab58f4f112bac53c1252862a485e19?ds=inline buildmimetree.py: rework of file io --- diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py index fbc8040..32fc17c 100755 --- a/.config/neomutt/buildmimetree.py +++ b/.config/neomutt/buildmimetree.py @@ -36,7 +36,6 @@ # import sys -import os import os.path import pathlib import markdown @@ -45,8 +44,11 @@ import argparse import re import mimetypes import bs4 +import xml.etree.ElementTree as etree +import io from collections import namedtuple, OrderedDict from markdown.extensions import Extension +from markdown.blockprocessors import BlockProcessor from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE from email.utils import make_msgid from urllib import request @@ -133,6 +135,13 @@ def parse_cli_args(*args, **kwargs): help="Turn on debugging to stderr of the MIME tree walk", ) + parser.add_argument( + "--dump-html", + metavar="FILE", + type=pathlib.Path, + help="Write the generated HTML to the file", + ) + subp = parser.add_subparsers(help="Sub-command parsers", dest="mode") massage_p = subp.add_parser( "massage", help="Massaging phase (internal use)" @@ -158,6 +167,163 @@ def parse_cli_args(*args, **kwargs): return parser.parse_args(*args, **kwargs) +# [ FILE I/O HANDLING ] ####################################################### + + +class File: + def __init__(self, path=None, mode="r", content=None, **kwargs): + if path: + if content: + raise RuntimeError("Cannot specify path and content for File") + + self._path = ( + path if isinstance(path, pathlib.Path) else pathlib.Path(path) + ) + else: + self._path = None + + if content and not re.search(r"[r+]", mode): + raise RuntimeError("Cannot specify content without read mode") + + self._rcache = [content] if content else [] + self._wcache = [] + self._mode = mode + self._kwargs = kwargs + self._file = None + + def open(self): + if self._path: + self._file = open(self._path, self._mode, **self._kwargs) + elif "b" in self._mode: + self._file = io.BytesIO() + else: + self._file = io.StringIO() + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + self._file.close() + self._file = None + self._rcache = self._wcache + + def _get_rcache(self): + return (b"" if "b" in self._mode else "").join(self._rcache) + + def _get_wcache(self): + return (b"" if "b" in self._mode else "").join(self._wcache) + + def _add_to_rcache(self, s): + self._rcache.append(s) + + def _add_to_wcache(self, s): + self._wcache.append(s) + + def read(self, *, cache=True): + if cache and self._rcache: + return self._get_rcache() + + if not self._file: + with self as f: + return f.read(cache=cache) + + self._file.seek(0) + if cache: + self._add_to_rcache(self._file.read()) + return self._get_rcache() + else: + return self._file.read() + + def write(self, s, *, cache=True): + if not self._file: + with self as f: + return f.write(s, cache=cache) + + self._file.seek(0) + self._rcache = self._wcache + + if cache: + self._add_to_wcache(s) + + written = self._file.write(s) + self._file.flush() + return written + + path = property(lambda s: s._path) + + def __repr__(self): + return ( + f'" + ) + + +class FileFactory: + def __init__(self): + self._files = [] + + def __call__(self, path=None, mode="r", content=None, **kwargs): + f = File(path, mode, content, **kwargs) + self._files.append(f) + return f + + def __len__(self): + return self._files.__len__() + + def pop(self, idx=-1): + return self._files.pop(idx) + + def __getitem__(self, idx): + return self._files.__getitem__(idx) + + def __contains__(self, f): + return self._files.__contains__(f) + + +class FakeFileFactory(FileFactory): + def __init__(self): + super().__init__() + self._paths2files = OrderedDict() + + def __call__(self, path=None, mode="r", content=None, **kwargs): + if path in self._paths2files: + return self._paths2files[path] + + f = super().__call__(None, mode, content, **kwargs) + self._paths2files[path] = f + + mypath = path + + class FakeFile(File): + path = mypath + + # this is quality Python! We do this so that the fake file, which has + # no path, fake-pretends to have a path for testing purposes. + + f.__class__ = FakeFile + return f + + def __getitem__(self, path): + return self._paths2files.__getitem__(path) + + def get(self, path, default): + return self._paths2files.get(path, default) + + def pop(self, last=True): + return self._paths2files.popitem(last) + + def __repr__(self): + return ( + f"" + ) + + # [ IMAGE HANDLING ] ########################################################## @@ -171,6 +337,7 @@ class ImageRegistry: self._images = OrderedDict() def register(self, path, description=None): + # path = str(pathlib.Path(path).expanduser()) path = os.path.expanduser(path) if path.startswith("/"): path = f"file://{path}" @@ -295,6 +462,68 @@ def apply_styling(html, css): ) +# [ QUOTE HANDLING ] ########################################################## + + +class QuoteToAdmonitionExtension(Extension): + class EmailQuoteBlockProcessor(BlockProcessor): + RE = re.compile(r"(?:^|\n)>\s*(.*)") + + def __init__(self, parser): + super().__init__(parser) + self._title = None + + def test(self, parent, blocks): + if markdown.util.nearing_recursion_limit(): + return False + + lines = blocks.splitlines() + if len(lines) < 2: + if not self._title: + return False + + elif not self.RE.search(lines[0]): + return False + + return len(lines) > 0 + + elif not self.RE.search(lines[0]) and self.RE.search(lines[1]): + return True + + elif self._title and self.RE.search(lines[1]): + return True + + return False + + def run(self, parent, blocks): + quotelines = blocks.pop(0).splitlines() + + cont = bool(self._title) + if not self.RE.search(quotelines[0]): + self._title = quotelines.pop(0) + + admonition = etree.SubElement(parent, "div") + admonition.set( + "class", f"admonition quote{' continued' if cont else ''}" + ) + self.parser.parseChunk(admonition, self._title) + + admonition[0].set("class", "admonition-title") + self.parser.parseChunk( + admonition, "\n".join(self.clean(line) for line in quotelines) + ) + + @classmethod + def clean(klass, line): + m = klass.RE.match(line) + return m.group(1) if m else line + + def extendMarkdown(self, md): + md.registerExtension(self) + email_quote_proc = self.EmailQuoteBlockProcessor(md.parser) + md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25) + + # [ PARTS GENERATION ] ######################################################## @@ -324,18 +553,8 @@ class Multipart( return hash(str(self.subtype) + "".join(str(self.children))) -def filereader_fn(path, mode="r", **kwargs): - with open(path, mode, **kwargs) as in_f: - return in_f.read() - - -def filewriter_fn(path, content, mode="w", **kwargs): - with open(path, mode, **kwargs) as out_f: - out_f.write(content) - - def collect_inline_images( - image_registry, *, tempdir=None, filewriter_fn=filewriter_fn + image_registry, *, tempdir=None, filefactory=FileFactory() ): relparts = [] for path, info in image_registry.items(): @@ -349,7 +568,10 @@ def collect_inline_images( tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir) path = pathlib.Path(tempfilename[1]) - filewriter_fn(path, data.read(), "w+b") + with filefactory(path, "w+b") as out_f: + out_f.write(data.read()) + + # filewriter_fn(path, data.read(), "w+b") desc = ( f'Inline image: "{info.desc}"' @@ -394,7 +616,7 @@ def make_text_mail(text, sig=None): return EMAIL_SIG_SEP.join((text, sig)) if sig else text -def extract_signature(text, *, filereader_fn=filereader_fn): +def extract_signature(text, *, filefactory=FileFactory()): parts = text.split(EMAIL_SIG_SEP, 1) if len(parts) == 1: return text, None, None @@ -404,7 +626,7 @@ def extract_signature(text, *, filereader_fn=filereader_fn): path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1]) textsig = "\n".join(lines) - sig_input = filereader_fn(path.expanduser()) + sig_input = filefactory(path.expanduser()).read() soup = bs4.BeautifulSoup(sig_input, "html.parser") style = str(soup.style.extract()) if soup.style else "" @@ -434,24 +656,28 @@ def extract_signature(text, *, filereader_fn=filereader_fn): def convert_markdown_to_html( - origtext, - draftpath, + draft_f, *, related_to_html_only=False, - css=None, - filewriter_fn=filewriter_fn, - filereader_fn=filereader_fn, + css_f=None, + htmldump_f=None, + filefactory=FileFactory(), tempdir=None, extensions=None, extension_configs=None, ): # TODO extension_configs need to be handled differently extension_configs = extension_configs or {} - extension_configs.setdefault("pymdownx.highlight", {}) - extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS + extension_configs.setdefault("pymdownx.highlight", {})[ + "css_class" + ] = _CODEHILITE_CLASS + extensions = extensions or [] + extensions.append(QuoteToAdmonitionExtension()) + + draft = draft_f.read() origtext, textsig, htmlsig = extract_signature( - origtext, filereader_fn=filereader_fn + draft, filefactory=filefactory ) ( @@ -492,23 +718,31 @@ def convert_markdown_to_html( ) origtext = make_text_mail(origtext, textsig) - - filewriter_fn(draftpath, origtext, encoding="utf-8") + draft_f.write(origtext) textpart = Part( - "text", "plain", draftpath, "Plain-text version", orig=True + "text", "plain", draft_f.path, "Plain-text version", orig=True ) htmltext = make_html_doc(htmltext, htmlsig) - htmltext = apply_styling(htmltext, css) + htmltext = apply_styling(htmltext, css_f.read() if css_f else None) - htmlpath = draftpath.with_suffix(".html") - filewriter_fn( - htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace" - ) + if draft_f.path: + htmlpath = draft_f.path.with_suffix(".html") + else: + htmlpath = pathlib.Path( + tempfile.mkstemp(suffix=".html", dir=tempdir)[1] + ) + with filefactory( + htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace" + ) as out_f: + out_f.write(htmltext) htmlpart = Part("text", "html", htmlpath, "HTML version") + if htmldump_f: + htmldump_f.write(htmltext) + imgparts = collect_inline_images( - image_registry, tempdir=tempdir, filewriter_fn=filewriter_fn + image_registry, tempdir=tempdir, filefactory=filefactory ) if related_to_html_only: @@ -684,11 +918,11 @@ def do_setup( def do_massage( draft_f, - draftpath, cmd_f, *, extensions=None, css_f=None, + htmldump_f=None, converter=convert_markdown_to_html, related_to_html_only=True, only_build=False, @@ -717,9 +951,9 @@ def do_massage( extensions = extensions.split(",") if extensions else [] tree = converter( - draft_f.read(), - draftpath, - css=css_f.read() if css_f else None, + draft_f, + css_f=css_f, + htmldump_f=htmldump_f, related_to_html_only=related_to_html_only, tempdir=tempdir, extensions=extensions, @@ -875,15 +1109,18 @@ if __name__ == "__main__": ) elif args.mode == "massage": - with open(args.MAILDRAFT, "r") as draft_f, open( - args.cmdpath, "w" - ) as cmd_f, open(args.css_file, "r") as css_f: + with ( + File(args.MAILDRAFT, "r+") as draft_f, + File(args.cmdpath, "w") as cmd_f, + File(args.css_file, "r") as css_f, + File(args.dump_html, "w") as htmldump_f, + ): do_massage( draft_f, - args.MAILDRAFT, cmd_f, extensions=args.extensions, css_f=css_f, + htmldump_f=htmldump_f, related_to_html_only=args.related_to_html_only, max_other_attachments=args.max_number_other_attachments, only_build=args.only_build, @@ -897,16 +1134,23 @@ if __name__ == "__main__": try: import pytest - from io import StringIO class Tests: @pytest.fixture def const1(self): - return "CONSTANT STRING 1" + return "Curvature Vest Usher Dividing+T#iceps Senior" @pytest.fixture def const2(self): - return "CONSTANT STRING 2" + return "Habitant Celestial 2litzy Resurf/ce Headpiece Harmonics" + + @pytest.fixture + def fakepath(self): + return pathlib.Path("/does/not/exist") + + @pytest.fixture + def fakepath2(self): + return pathlib.Path("/does/not/exist/either") # NOTE: tests using the capsys fixture must specify sys.stdout to the # functions they call, else old stdout is used and not captured @@ -1021,6 +1265,53 @@ try: desc="Alternative", ) + @pytest.fixture + def mime_tree_nested(self): + return Multipart( + "relative", + children=[ + Multipart( + "alternative", + children=[ + Part( + "text", + "plain", + "part.txt", + desc="Plain", + orig=True, + ), + Multipart( + "alternative", + children=[ + Part( + "text", + "plain", + "part.txt", + desc="Nested plain", + ), + Part( + "text", + "html", + "part.html", + desc="Nested HTML", + ), + ], + desc="Nested alternative", + ), + ], + desc="Alternative", + ), + Part( + "text", + "png", + "logo.png", + cid="logo.png", + desc="Logo", + ), + ], + desc="Related", + ) + @pytest.mark.treewalk def test_MIMETreeDFWalker_depth_first_walk( self, mime_tree_related_to_alternative @@ -1084,26 +1375,18 @@ try: return StringIO(text or const1) @pytest.mark.massage - def test_do_massage_basic(self, const1, string_io, capsys): - def converter( - drafttext, - draftpath, - css, - related_to_html_only, - extensions, - tempdir, - ): - return Part("text", "plain", draftpath, orig=True) + def test_do_massage_basic(self): + def converter(draft_f, **kwargs): + return Part("text", "plain", draft_f.path, orig=True) - do_massage( - draft_f=string_io, - draftpath=const1, - cmd_f=sys.stdout, - converter=converter, - ) + with File() as draft_f, File() as cmd_f: + do_massage( + draft_f=draft_f, + cmd_f=cmd_f, + converter=converter, + ) + lines = cmd_f.read().splitlines() - captured = capsys.readouterr() - lines = captured.out.splitlines() assert '="$my_editor"' in lines.pop(0) assert '="$my_edit_headers"' in lines.pop(0) assert "unset my_editor" == lines.pop(0) @@ -1115,30 +1398,21 @@ try: assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0) @pytest.mark.massage - def test_do_massage_fulltree( - self, string_io, const1, mime_tree_related_to_alternative, capsys - ): - def converter( - drafttext, - draftpath, - css, - related_to_html_only, - extensions, - tempdir, - ): + def test_do_massage_fulltree(self, mime_tree_related_to_alternative): + def converter(draft_f, **kwargs): return mime_tree_related_to_alternative max_attachments = 5 - do_massage( - draft_f=string_io, - draftpath=const1, - cmd_f=sys.stdout, - max_other_attachments=max_attachments, - converter=converter, - ) - captured = capsys.readouterr() - lines = captured.out.splitlines()[4:-2] + with File() as draft_f, File() as cmd_f: + do_massage( + draft_f=draft_f, + cmd_f=cmd_f, + max_other_attachments=max_attachments, + converter=converter, + ) + lines = cmd_f.read().splitlines()[4:-2] + assert "first-entry" in lines.pop() assert "update-encoding" in lines.pop() assert "Plain" in lines.pop() @@ -1166,59 +1440,115 @@ try: assert "send-message" in lines.pop() assert len(lines) == 0 - @pytest.fixture - def fake_filewriter(self): - class FileWriter: - def __init__(self): - self._writes = [] + @pytest.mark.massage + def test_mime_tree_relative_within_alternative( + self, mime_tree_related_to_html + ): + def converter(draft_f, **kwargs): + return mime_tree_related_to_html - def __call__(self, path, content, mode="w", **kwargs): - self._writes.append((path, content)) + with File() as draft_f, File() as cmd_f: + do_massage( + draft_f=draft_f, + cmd_f=cmd_f, + converter=converter, + ) + lines = cmd_f.read().splitlines()[4:-2] + + assert "first-entry" in lines.pop() + assert "update-encoding" in lines.pop() + assert "Plain" in lines.pop() + assert "part.html" in lines.pop() + assert "toggle-unlink" in lines.pop() + assert "move-up" in lines.pop() + while True: + top = lines.pop() + if "move-up" not in top: + break + assert "move-down" in top + assert "HTML" in lines.pop() + assert "logo.png" in lines.pop() + assert "toggle-unlink" in lines.pop() + assert "content-id" in lines.pop() + assert "move-up" in lines.pop() + while True: + top = lines.pop() + if "move-up" not in top: + break + assert "move-down" in top + assert "move-down" in lines.pop() + assert "Logo" in lines.pop() + assert "jump>2" in lines.pop() + assert "jump>3" in lines.pop() + assert "group-related" in lines.pop() + assert "Related" in lines.pop() + assert "jump>1" in lines.pop() + assert "jump>2" in lines.pop() + assert "group-alternative" in lines.pop() + assert "Alternative" in lines.pop() + assert "send-message" in lines.pop() + assert len(lines) == 0 - def pop(self, index=-1): - return self._writes.pop(index) + @pytest.mark.massage + def test_mime_tree_nested_trees_does_not_break_positioning( + self, mime_tree_nested + ): + def converter(draft_f, **kwargs): + return mime_tree_nested - return FileWriter() + with File() as draft_f, File() as cmd_f: + do_massage( + draft_f=draft_f, + cmd_f=cmd_f, + converter=converter, + ) + lines = cmd_f.read().splitlines() - @pytest.fixture - def markdown_non_converter(self, const1, const2): - return lambda s, text: f"{const1}{text}{const2}" + while "logo.png" not in lines.pop(): + pass + lines.pop() + assert "content-id" in lines.pop() + assert "move-up" in lines.pop() + while True: + top = lines.pop() + if "move-up" not in top: + break + assert "move-down" in top + # Due to the nested trees, the number of descendents of the sibling + # actually needs to be considered, not just the nieces. So to move + # from position 1 to position 6, it only needs one + # because that jumps over the entire sibling tree. Thus what + # follows next must not be another + assert "Logo" in lines.pop() @pytest.mark.converter - def test_converter_tree_basic(self, const1, const2, fake_filewriter): - path = pathlib.Path(const2) + def test_converter_tree_basic(self, fakepath, const1, fakefilefactory): + draft_f = fakefilefactory(fakepath, content=const1) tree = convert_markdown_to_html( - const1, path, filewriter_fn=fake_filewriter + draft_f, filefactory=fakefilefactory ) assert tree.subtype == "alternative" assert len(tree.children) == 2 assert tree.children[0].subtype == "plain" - assert tree.children[0].path == path + assert tree.children[0].path == draft_f.path assert tree.children[0].orig assert tree.children[1].subtype == "html" - assert tree.children[1].path == path.with_suffix(".html") + assert tree.children[1].path == fakepath.with_suffix(".html") + @pytest.mark.converter def test_converter_writes( - self, - const1, - const2, - fake_filewriter, - monkeypatch, - markdown_non_converter, + self, fakepath, fakefilefactory, const1, monkeypatch ): - path = pathlib.Path(const2) - - with monkeypatch.context() as m: - m.setattr(markdown.Markdown, "convert", markdown_non_converter) - convert_markdown_to_html( - const1, path, filewriter_fn=fake_filewriter - ) + draft_f = fakefilefactory(fakepath, content=const1) + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) - assert (path, const1) == fake_filewriter.pop(0) - written = fake_filewriter.pop(0) - assert path.with_suffix(".html") == written[0] - assert const1 in written[1] + html = fakefilefactory.pop() + assert fakepath.with_suffix(".html") == html[0] + assert const1 in html[1].read() + text = fakefilefactory.pop() + assert fakepath == text[0] + assert const1 == text[1].read() @pytest.mark.imgproc def test_markdown_inline_image_processor(self): @@ -1316,76 +1646,77 @@ try: @pytest.mark.converter def test_converter_tree_inline_image_base64( - self, test_png, const1, fake_filewriter + self, test_png, fakefilefactory ): text = f"![inline base64 image]({test_png})" - path = pathlib.Path(const1) - tree = convert_markdown_to_html( - text, - path, - filewriter_fn=fake_filewriter, - related_to_html_only=False, - ) + with fakefilefactory(content=text) as draft_f: + tree = convert_markdown_to_html( + draft_f, + filefactory=fakefilefactory, + related_to_html_only=False, + ) assert tree.subtype == "relative" assert tree.children[0].subtype == "alternative" assert tree.children[1].subtype == "png" - written = fake_filewriter.pop() + written = fakefilefactory.pop() assert tree.children[1].path == written[0] - assert written[1] == request.urlopen(test_png).read() + assert b"PNG" in written[1].read() @pytest.mark.converter def test_converter_tree_inline_image_base64_related_to_html( - self, test_png, const1, fake_filewriter + self, test_png, fakefilefactory ): text = f"![inline base64 image]({test_png})" - path = pathlib.Path(const1) - tree = convert_markdown_to_html( - text, - path, - filewriter_fn=fake_filewriter, - related_to_html_only=True, - ) + with fakefilefactory(content=text) as draft_f: + tree = convert_markdown_to_html( + draft_f, + filefactory=fakefilefactory, + related_to_html_only=True, + ) assert tree.subtype == "alternative" assert tree.children[1].subtype == "relative" assert tree.children[1].children[1].subtype == "png" - written = fake_filewriter.pop() + written = fakefilefactory.pop() assert tree.children[1].children[1].path == written[0] - assert written[1] == request.urlopen(test_png).read() + assert b"PNG" in written[1].read() @pytest.mark.converter def test_converter_tree_inline_image_cid( - self, const1, fake_filewriter + self, const1, fakefilefactory ): text = f"![inline base64 image](cid:{const1})" - path = pathlib.Path(const1) - tree = convert_markdown_to_html( - text, - path, - filewriter_fn=fake_filewriter, - related_to_html_only=False, - ) + with fakefilefactory(content=text) as draft_f: + tree = convert_markdown_to_html( + draft_f, + filefactory=fakefilefactory, + related_to_html_only=False, + ) assert len(tree.children) == 2 assert tree.children[0].cid != const1 assert tree.children[0].type != "image" assert tree.children[1].cid != const1 assert tree.children[1].type != "image" + @pytest.fixture + def fakefilefactory(self): + return FakeFileFactory() + @pytest.mark.imgcoll def test_inline_image_collection( - self, test_png, const1, const2, fake_filewriter + self, test_png, const1, const2, fakefilefactory ): test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)} relparts = collect_inline_images( - test_images, filewriter_fn=fake_filewriter + test_images, filefactory=fakefilefactory ) - written = fake_filewriter.pop() - assert b"PNG" in written[1] + written = fakefilefactory.pop() + assert b"PNG" in written[1].read() assert relparts[0].subtype == "png" assert relparts[0].path == written[0] assert relparts[0].cid == const1 - assert relparts[0].desc.endswith(const2) + assert const2 in relparts[0].desc if _PYNLINER: @@ -1397,49 +1728,52 @@ try: assert 'p style="color' in out @pytest.mark.styling - def test_massage_styling_to_converter(self, string_io, const1): + def test_apply_no_stylesheet(self, const1): + out = apply_styling(const1, None) + + @pytest.mark.massage + @pytest.mark.styling + def test_massage_styling_to_converter(self): css = "p { color:red }" - css_f = StringIO(css) - out_f = StringIO() + css_f = File(content=css) css_applied = [] - def converter( - drafttext, - draftpath, - css, - related_to_html_only, - extensions, - tempdir, - ): + def converter(draft_f, css_f, **kwargs): + css = css_f.read() css_applied.append(css) - return Part("text", "plain", draftpath, orig=True) + return Part("text", "plain", draft_f.path, orig=True) do_massage( - draft_f=string_io, - draftpath=const1, - cmd_f=out_f, + draft_f=File(), + cmd_f=File(), css_f=css_f, converter=converter, ) assert css_applied[0] == css @pytest.mark.converter + @pytest.mark.styling def test_converter_apply_styles( - self, const1, fake_filewriter, monkeypatch + self, const1, monkeypatch, fakepath, fakefilefactory ): - path = pathlib.Path(const1) - text = "Hello, world!" css = "p { color:red }" - with monkeypatch.context() as m: + with ( + monkeypatch.context() as m, + fakefilefactory(fakepath, content=const1) as draft_f, + fakefilefactory(content=css) as css_f, + ): m.setattr( markdown.Markdown, "convert", lambda s, t: f"

{t}

", ) convert_markdown_to_html( - text, path, css=css, filewriter_fn=fake_filewriter + draft_f, css_f=css_f, filefactory=fakefilefactory ) - assert "color: red" in fake_filewriter.pop()[1] + assert re.search( + r"color:.*red", + fakefilefactory[fakepath.with_suffix(".html")].read(), + ) if _PYGMENTS_CSS: @@ -1452,146 +1786,6 @@ try: out = apply_styling(html, _PYGMENTS_CSS) assert f'{_CODEHILITE_CLASS}" style="' in out - @pytest.mark.massage - def test_mime_tree_relative_within_alternative( - self, string_io, const1, capsys, mime_tree_related_to_html - ): - def converter( - drafttext, - draftpath, - css, - related_to_html_only, - extensions, - tempdir, - ): - return mime_tree_related_to_html - - do_massage( - draft_f=string_io, - draftpath=const1, - cmd_f=sys.stdout, - converter=converter, - ) - - captured = capsys.readouterr() - lines = captured.out.splitlines()[4:-2] - assert "first-entry" in lines.pop() - assert "update-encoding" in lines.pop() - assert "Plain" in lines.pop() - assert "part.html" in lines.pop() - assert "toggle-unlink" in lines.pop() - assert "move-up" in lines.pop() - while True: - top = lines.pop() - if "move-up" not in top: - break - assert "move-down" in top - assert "HTML" in lines.pop() - assert "logo.png" in lines.pop() - assert "toggle-unlink" in lines.pop() - assert "content-id" in lines.pop() - assert "move-up" in lines.pop() - while True: - top = lines.pop() - if "move-up" not in top: - break - assert "move-down" in top - assert "move-down" in lines.pop() - assert "Logo" in lines.pop() - assert "jump>2" in lines.pop() - assert "jump>3" in lines.pop() - assert "group-related" in lines.pop() - assert "Related" in lines.pop() - assert "jump>1" in lines.pop() - assert "jump>2" in lines.pop() - assert "group-alternative" in lines.pop() - assert "Alternative" in lines.pop() - assert "send-message" in lines.pop() - assert len(lines) == 0 - - @pytest.mark.massage - def test_mime_tree_nested_trees_does_not_break_positioning( - self, string_io, const1, capsys - ): - def converter( - drafttext, - draftpath, - css, - related_to_html_only, - extensions, - tempdir, - ): - return Multipart( - "relative", - children=[ - Multipart( - "alternative", - children=[ - Part( - "text", - "plain", - "part.txt", - desc="Plain", - orig=True, - ), - Multipart( - "alternative", - children=[ - Part( - "text", - "plain", - "part.txt", - desc="Nested plain", - ), - Part( - "text", - "html", - "part.html", - desc="Nested HTML", - ), - ], - desc="Nested alternative", - ), - ], - desc="Alternative", - ), - Part( - "text", - "png", - "logo.png", - cid="logo.png", - desc="Logo", - ), - ], - desc="Related", - ) - - do_massage( - draft_f=string_io, - draftpath=const1, - cmd_f=sys.stdout, - converter=converter, - ) - - captured = capsys.readouterr() - lines = captured.out.splitlines() - while "logo.png" not in lines.pop(): - pass - lines.pop() - assert "content-id" in lines.pop() - assert "move-up" in lines.pop() - while True: - top = lines.pop() - if "move-up" not in top: - break - assert "move-down" in top - # Due to the nested trees, the number of descendents of the sibling - # actually needs to be considered, not just the nieces. So to move - # from position 1 to position 6, it only needs one - # because that jumps over the entire sibling tree. Thus what - # follows next must not be another - assert "Logo" in lines.pop() - @pytest.mark.sig def test_signature_extraction_no_signature(self, const1): assert (const1, None, None) == extract_signature(const1) @@ -1606,22 +1800,21 @@ try: assert htmlsig is None @pytest.mark.sig - def test_signature_extraction_html(self, const1, const2): - path = pathlib.Path("somepath") + def test_signature_extraction_html( + self, fakepath, fakefilefactory, const1, const2 + ): sigconst = "HTML signature from {path} but as a string" + sig = f'
{sigconst.format(path=fakepath)}
' - def filereader_fn(path): - return ( - f'
{sigconst.format(path=path)}
' - ) + sig_f = fakefilefactory(fakepath, content=sig) origtext, textsig, htmlsig = extract_signature( - f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {path}\n{const2}", - filereader_fn=filereader_fn, + f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {fakepath}\n{const2}", + filefactory=fakefilefactory, ) assert origtext == const1 assert textsig == const2 - assert htmlsig == sigconst.format(path=path) + assert htmlsig == sigconst.format(path=fakepath) @pytest.mark.sig def test_signature_extraction_file_not_found(self, const1): @@ -1651,29 +1844,27 @@ try: @pytest.mark.converter @pytest.mark.sig def test_converter_signature_handling( - self, const1, fake_filewriter, monkeypatch + self, fakepath, fakefilefactory, monkeypatch ): - path = pathlib.Path(const1) - mailparts = ( "This is the mail body\n", f"{EMAIL_SIG_SEP}", "This is a plain-text signature only", ) - def filereader_fn(path): - return "" - - with monkeypatch.context() as m: + with ( + fakefilefactory( + fakepath, content="".join(mailparts) + ) as draft_f, + monkeypatch.context() as m, + ): m.setattr(markdown.Markdown, "convert", lambda s, t: t) - convert_markdown_to_html( - "".join(mailparts), - path, - filewriter_fn=fake_filewriter, - filereader_fn=filereader_fn, - ) + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) - soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser") + soup = bs4.BeautifulSoup( + fakefilefactory[fakepath.with_suffix(".html")].read(), + "html.parser", + ) body = soup.body.contents assert mailparts[0] in body.pop(0) @@ -1690,114 +1881,284 @@ try: @pytest.mark.converter @pytest.mark.sig def test_converter_signature_handling_htmlsig( - self, const1, fake_filewriter, monkeypatch + self, fakepath, fakepath2, fakefilefactory, monkeypatch ): - path = pathlib.Path(const1) - mailparts = ( "This is the mail body", f"{EMAIL_SIG_SEP}", - f"{HTML_SIG_MARKER}{path}\n", + f"{HTML_SIG_MARKER}{fakepath2}\n", "This is the plain-text version", ) + htmlsig = "HTML Signature from {path} but as a string" + html = ( + f'

{htmlsig.format(path=fakepath2)}

' + ) - htmlsig = "HTML Signature from {path}" - - def filereader_fn(path): - return f'
{htmlsig.format(path=path)}
' + sig_f = fakefilefactory(fakepath2, content=html) def mdwn_fn(t): return t.upper() - with monkeypatch.context() as m: + with ( + fakefilefactory( + fakepath, content="".join(mailparts) + ) as draft_f, + monkeypatch.context() as m, + ): m.setattr( markdown.Markdown, "convert", lambda s, t: mdwn_fn(t) ) - convert_markdown_to_html( - "".join(mailparts), - path, - filewriter_fn=fake_filewriter, - filereader_fn=filereader_fn, - ) + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) - soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser") + soup = bs4.BeautifulSoup( + fakefilefactory[fakepath.with_suffix(".html")].read(), + "html.parser", + ) sig = soup.select_one("#signature") sig.span.extract() assert HTML_SIG_MARKER not in sig.text - assert htmlsig.format(path=path) == sig.text.strip() + assert htmlsig.format(path=fakepath2) == sig.text.strip() - plaintext = fake_filewriter.pop()[1] + plaintext = fakefilefactory[fakepath].read() assert plaintext.endswith(EMAIL_SIG_SEP + mailparts[-1]) @pytest.mark.converter @pytest.mark.sig def test_converter_signature_handling_htmlsig_with_image( - self, const1, fake_filewriter, monkeypatch, test_png + self, fakepath, fakepath2, fakefilefactory, monkeypatch, test_png ): - path = pathlib.Path(const1) - mailparts = ( "This is the mail body", f"{EMAIL_SIG_SEP}", - f"{HTML_SIG_MARKER}{path}\n", + f"{HTML_SIG_MARKER}{fakepath2}\n", "This is the plain-text version", ) - htmlsig = ( "HTML Signature from {path} with image\n" f'\n' ) + html = ( + f'
{htmlsig.format(path=fakepath2)}
' + ) - def filereader_fn(path): - return f'
{htmlsig.format(path=path)}
' + sig_f = fakefilefactory(fakepath2, content=html) def mdwn_fn(t): return t.upper() - with monkeypatch.context() as m: + with ( + fakefilefactory( + fakepath, content="".join(mailparts) + ) as draft_f, + monkeypatch.context() as m, + ): m.setattr( markdown.Markdown, "convert", lambda s, t: mdwn_fn(t) ) - convert_markdown_to_html( - "".join(mailparts), - path, - filewriter_fn=fake_filewriter, - filereader_fn=filereader_fn, - ) + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) - assert fake_filewriter.pop()[0].suffix == ".png" + assert fakefilefactory.pop()[0].suffix == ".png" - soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser") + soup = bs4.BeautifulSoup( + fakefilefactory[fakepath.with_suffix(".html")].read(), + "html.parser", + ) assert soup.img.attrs["src"].startswith("cid:") @pytest.mark.converter @pytest.mark.sig def test_converter_signature_handling_textsig_with_image( - self, const1, fake_filewriter, test_png + self, fakepath, fakefilefactory, test_png ): mailparts = ( "This is the mail body", f"{EMAIL_SIG_SEP}", "This is the plain-text version with image\n", f"![Inline]({test_png})", - - ) - tree = convert_markdown_to_html - "".join(mailparts), - pathlib.Path(const1), - filewriter_fn=fake_filewriter, ) + with ( + fakefilefactory( + fakepath, content="".join(mailparts) + ) as draft_f, + ): + tree = convert_markdown_to_html( + draft_f, filefactory=fakefilefactory + ) assert tree.subtype == "relative" assert tree.children[0].subtype == "alternative" assert tree.children[1].subtype == "png" - written = fake_filewriter.pop() + written = fakefilefactory.pop() assert tree.children[1].path == written[0] - assert written[1] == request.urlopen(test_png).read() + assert written[1].read() == request.urlopen(test_png).read() - def test_converter_attribution_to_admonition(self, fake_filewriter): + @pytest.mark.converter + def test_converter_attribution_to_admonition( + self, fakepath, fakefilefactory + ): + mailparts = ( + "Regarding whatever", + "> blockquote line1", + "> blockquote line2", + "> ", + "> new para with **bold** text", + ) + with fakefilefactory( + fakepath, content="\n".join(mailparts) + ) as draft_f: + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) + + soup = bs4.BeautifulSoup( + fakefilefactory[fakepath.with_suffix(".html")].read(), + "html.parser", + ) + quote = soup.select_one("div.admonition.quote") + assert quote + assert ( + soup.select_one("p.admonition-title").extract().text.strip() + == mailparts[0] + ) + p = quote.p.extract() + assert p.text.strip() == "\n".join(p[2:] for p in mailparts[1:3]) + + p = quote.p.extract() + assert p.contents[1].name == "strong" + + @pytest.mark.converter + def test_converter_attribution_to_admonition_multiple( + self, fakepath, fakefilefactory + ): + mailparts = ( + "Regarding whatever", + "> blockquote line1", + "> blockquote line2", + "", + "Normal text", + "", + "> continued emailquote", + "", + "Another email-quote", + "> something", + ) + with fakefilefactory( + fakepath, content="\n".join(mailparts) + ) as draft_f: + convert_markdown_to_html(draft_f, filefactory=fakefilefactory) + + soup = bs4.BeautifulSoup( + fakefilefactory[fakepath.with_suffix(".html")].read(), + "html.parser", + ) + quote = soup.select_one("div.admonition.quote.continued").extract() + assert quote + assert ( + quote.select_one("p.admonition-title").extract().text.strip() + == mailparts[0] + ) + + p = quote.p.extract() + assert p + + quote = soup.select_one("div.admonition.quote.continued").extract() + assert quote + assert ( + quote.select_one("p.admonition-title").extract().text.strip() + == mailparts[-2] + ) + + @pytest.mark.fileio + def test_file_class_contextmanager(self, const1, monkeypatch): + state = dict(o=False, c=False) + + def fn(t): + state[t] = True + + with monkeypatch.context() as m: + m.setattr(File, "open", lambda s: fn("o")) + m.setattr(File, "close", lambda s: fn("c")) + with File() as f: + assert state["o"] + assert not state["c"] + assert state["c"] + + @pytest.mark.fileio + def test_file_class_no_path(self, const1): + with File(mode="w+") as f: + f.write(const1, cache=False) + assert f.read(cache=False) == const1 + + @pytest.mark.fileio + def test_file_class_path(self, const1, tmp_path): + with File(tmp_path / "file", mode="w+") as f: + f.write(const1, cache=False) + assert f.read(cache=False) == const1 + + @pytest.mark.fileio + def test_file_class_cache(self, tmp_path, const1, const2): + path = tmp_path / "file" + file = File(path, mode="w+") + with file as f: + f.write(const1, cache=True) + with open(path, mode="w") as f: + f.write(const2) + with file as f: + assert f.read(cache=True) == const1 + + @pytest.mark.fileio + def test_file_class_cache_init(self, const1): + file = File(path=None, mode="r", content=const1) + with file as f: + assert f.read() == const1 + + @pytest.mark.fileio + def test_file_class_content_or_path(self, fakepath, const1): + with pytest.raises(RuntimeError): + file = File(path=fakepath, content=const1) + + @pytest.mark.fileio + def test_file_class_content_needs_read(self, const1): + with pytest.raises(RuntimeError): + file = File(mode="w", content=const1) + + @pytest.mark.fileio + def test_file_class_write_persists_close(self, const1): + f = File(mode="w+") + with f: + f.write(const1) + with f: + assert f.read() == const1 + + @pytest.mark.fileio + def test_file_class_write_resets_read_cache(self, const1, const2): + with File(mode="w+", content=const1) as f: + assert f.read() == const1 + f.write(const2) + assert f.read() == const2 + + @pytest.mark.fileio + def test_file_factory(self): + fact = FileFactory() + f = fact() + assert isinstance(f, File) + assert len(fact) == 1 + assert f in fact + assert f == fact[0] + + @pytest.mark.fileio + def test_fake_file_factory(self, fakepath, fakefilefactory): + fact = FakeFileFactory() + f = fakefilefactory(fakepath) + assert f.path == fakepath + assert f == fakefilefactory[fakepath] + + @pytest.mark.fileio + def test_fake_file_factory_path_persistence( + self, fakepath, fakefilefactory + ): + f1 = fakefilefactory(fakepath) + assert f1 == fakefilefactory(fakepath) except ImportError: pass diff --git a/.config/neomutt/htmlmail.css b/.config/neomutt/htmlmail.css index e60bf8b..ef23086 100644 --- a/.config/neomutt/htmlmail.css +++ b/.config/neomutt/htmlmail.css @@ -1,6 +1,57 @@ +body, .signature, #signature { font-family: Lato, Calibri, Tahoma, sans-serif; } + #signature .sig_separator { display: none; } -body, div.signature { font-family: Lato, "Liberation Sans", sans-serif; } +.admonition { + background-color: transparent; + border-radius: 0.75em; + border: 1px solid #eee; + box-shadow: 0 0.2em 0.5em #999 inset, 0 0 0.0625em #999 inset !important; + margin: 1em; + overflow: hidden; + padding: 0; + font-size: smaller; +} + +.admonition.quote { +} + +.admonition p.admonition-title { + background-color: #eee; + box-shadow: 0 0.2em 0.5em #999, 0 0 0.0625em #999; + font-style: normal; + margin: 0; + padding: 0.25em 0.25em 0.25em 0.5em; + font-style: italic; +} + +.admonition p { + margin: 0.5em; +} + +table, th, td { + border-collapse: collapse; + border: 1px solid #999; +} +th, td { padding: 0.5em; } +.header { + background: #eee; +} +.even { background: #eee; } + +h1, h2, h3, h4, h5, h6 { + color: #666; + background-color: #eee; + padding-left: 0.5em +} +h1 { font-size: 130%; } +h2 { font-size: 120%; } +h3 { font-size: 110%; } +h4 { font-size: 107%; } +h5 { font-size: 103%; } +h6 { font-size: 100%; } + +/* a { text-decoration:none; color:#3a5dae; } a:visited { color:#3a5dae; } a:hover { text-decoration:underline; } @@ -10,7 +61,7 @@ a:hover { text-decoration:underline; } font-style: italic; border-left: 2px solid #eee; } -.quote, blockquote { +blockquote { padding: 0 0.5em; margin: 0; font-style: italic; @@ -22,7 +73,7 @@ a:hover { text-decoration:underline; } margin-bottom: -1em; font-size: 80%; } -.quotechar { display: none; }*/ +.quotechar { display: none; }*x/ .footnote-ref, .footnote-back { text-decoration: none;} /*.signature { color: #999; @@ -30,26 +81,7 @@ a:hover { text-decoration:underline; } white-space: pre; margin: 1em 0 0 0; font-size: 80%; -}*/ -table, th, td { - border-collapse: collapse; - border: 1px solid #999; -} -th, td { padding: 0.5em; } -.header { - background: #eee; -} -.even { background: #eee; } -h1, h2, h3, h4, h5, h6 { - color: #666; - background-color: #eee; - padding-left: 0.5em -} -h1 { font-size: 130%; } -h2 { font-size: 120%; } -h3 { font-size: 110%; } -h4 { font-size: 107%; } -h5 { font-size: 103%; } -h6 { font-size: 100%; } +}*x/ /*p { padding: 0 0.5em; } -pre { padding: 0 1em; }*/ +pre { padding: 0 1em; }*x/ +*/ diff --git a/.config/neomutt/pytest.ini b/.config/neomutt/pytest.ini index 4dedcb2..5a38d80 100644 --- a/.config/neomutt/pytest.ini +++ b/.config/neomutt/pytest.ini @@ -9,3 +9,4 @@ markers = treewalk: test related to MIME tree generation/walking muttctrl: test related to Mutt command interfacing current: test currently being worked on + fileio: test related to file I/O