X-Git-Url: https://git.madduck.net/etc/neomutt.git/blobdiff_plain/9588fa3e1f6fa70aa78061337ac51ef973f8e46e..b08171361024e5fc4ef439c87a8222aa5627c71f:/.config/neomutt/buildmimetree.py diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py index c277e63..2396c49 100755 --- a/.config/neomutt/buildmimetree.py +++ b/.config/neomutt/buildmimetree.py @@ -23,8 +23,8 @@ # - python3-markdown # Optional: # - pytest -# - Pynliner -# - Pygments, if installed, then syntax highlighting is enabled +# - Pynliner, provides --css-file and thus inline styling of HTML output +# - Pygments, then syntax highlighting for fenced code is enabled # # Latest version: # https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py @@ -66,6 +66,16 @@ def parse_cli_args(*args, **kwargs): help="Markdown extension to use (comma-separated list)", ) + if _PYNLINER: + parser.add_argument( + "--css-file", + type=str, + default="", + help="CSS file to merge with the final HTML", + ) + else: + parser.set_defaults(css_file=None) + parser.add_argument( "--only-build", action="store_true", @@ -91,8 +101,10 @@ def parse_cli_args(*args, **kwargs): massage_p.add_argument( "--write-commands-to", + "-o", metavar="PATH", dest="cmdpath", + required=True, help="Temporary file path to write commands to", ) @@ -160,11 +172,15 @@ class InlineImageExtension(Extension): return self._images -def markdown_with_inline_image_support(text, *, extensions=None): +def markdown_with_inline_image_support( + text, *, extensions=None, extension_configs=None +): inline_image_handler = InlineImageExtension() extensions = extensions or [] extensions.append(inline_image_handler) - mdwn = markdown.Markdown(extensions=extensions) + mdwn = markdown.Markdown( + extensions=extensions, extension_configs=extension_configs + ) htmltext = mdwn.convert(text) images = inline_image_handler.get_images() @@ -179,6 +195,38 @@ def markdown_with_inline_image_support(text, *, extensions=None): return text, htmltext, images +# [ CSS STYLING ] ############################################################# + +try: + import pynliner + + _PYNLINER = True + +except ImportError: + _PYNLINER = False + +try: + from pygments.formatters import get_formatter_by_name + + _CODEHILITE_CLASS = "codehilite" + + _PYGMENTS_CSS = get_formatter_by_name( + "html", style="default" + ).get_style_defs(f".{_CODEHILITE_CLASS}") + +except ImportError: + _PYGMENTS_CSS = None + + +def apply_styling(html, css): + return ( + pynliner.Pynliner() + .from_string(html) + .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s)) + .run() + ) + + # [ PARTS GENERATION ] ######################################################## @@ -204,6 +252,9 @@ class Multipart( def __str__(self): return f" children={len(self.children)}" + def __hash__(self): + return hash(str(self.subtype) + "".join(str(self.children))) + def filewriter_fn(path, content, mode="w", **kwargs): with open(path, mode, **kwargs) as out_f: @@ -215,6 +266,9 @@ def collect_inline_images( ): relparts = [] for path, info in images.items(): + if path.startswith("cid:"): + continue + data = request.urlopen(path) mimetype = data.headers["Content-Type"] @@ -240,12 +294,20 @@ def convert_markdown_to_html( origtext, draftpath, *, + cssfile=None, + related_to_html_only=True, filewriter_fn=filewriter_fn, tempdir=None, extensions=None, + extension_configs=None, ): + # TODO extension_configs need to be handled differently + extension_configs = extension_configs or {} + extension_configs.setdefault("pymdownx.highlight", {}) + extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS + origtext, htmltext, images = markdown_with_inline_image_support( - origtext, extensions=extensions + origtext, extensions=extensions, extension_configs=extension_configs ) filewriter_fn(draftpath, origtext, encoding="utf-8") @@ -253,79 +315,106 @@ def convert_markdown_to_html( "text", "plain", draftpath, "Plain-text version", orig=True ) + htmltext = apply_styling(htmltext, cssfile) + htmlpath = draftpath.with_suffix(".html") filewriter_fn( htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace" ) htmlpart = Part("text", "html", htmlpath, "HTML version") - altpart = Multipart( - "alternative", [textpart, htmlpart], "Group of alternative content" - ) - imgparts = collect_inline_images( images, tempdir=tempdir, filewriter_fn=filewriter_fn ) - if imgparts: + + if related_to_html_only: + # If there are inline image part, they will be contained within a + # multipart/related part along with the HTML part only + if imgparts: + # replace htmlpart with a multipart/related container of the HTML + # parts and the images + htmlpart = Multipart( + "relative", [htmlpart] + imgparts, "Group of related content" + ) + return Multipart( - "relative", [altpart] + imgparts, "Group of related content" + "alternative", [textpart, htmlpart], "Group of alternative content" ) + else: - return altpart + # If there are inline image part, they will be siblings to the + # multipart/alternative tree within a multipart/related part + altpart = Multipart( + "alternative", [textpart, htmlpart], "Group of alternative content" + ) + if imgparts: + return Multipart( + "relative", [altpart] + imgparts, "Group of related content" + ) + else: + return altpart class MIMETreeDFWalker: def __init__(self, *, visitor_fn=None, debug=False): - self._visitor_fn = visitor_fn + self._visitor_fn = visitor_fn or self._echovisit self._debug = debug + def _echovisit(self, node, ancestry, debugprint): + debugprint(f"node={node} ancestry={ancestry}") + def walk(self, root, *, visitor_fn=None): """ Recursive function to implement a depth-dirst walk of the MIME-tree rooted at `root`. """ - if isinstance(root, list): - root = Multipart("mixed", children=root) + if len(root) > 1: + root = Multipart("mixed", children=root) + else: + root = root[0] self._walk( root, - stack=[], + ancestry=[], visitor_fn=visitor_fn or self._visitor_fn, ) - def _walk(self, node, *, stack, visitor_fn): + def _walk(self, node, *, ancestry, visitor_fn): # Let's start by enumerating the parts at the current level. At the - # root level, stack will be the empty list, and we expect a multipart/* - # container at this level. Later, e.g. within a mutlipart/alternative - # container, the subtree will just be the alternative parts, while the - # top of the stack will be the multipart/alternative container, which - # we will process after the following loop. - - lead = f"{'| '*len(stack)}|-" + # root level, ancestry will be the empty list, and we expect a + # multipart/* container at this level. Later, e.g. within a + # mutlipart/alternative container, the subtree will just be the + # alternative parts, while the top of the ancestry will be the + # multipart/alternative container, which we will process after the + # following loop. + + lead = f"{'│ '*len(ancestry)}" if isinstance(node, Multipart): self.debugprint( - f"{lead}{node} parents={[s.subtype for s in stack]}" + f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}" ) - # Depth-first, so push the current container onto the stack, - # then descend … - stack.append(node) - self.debugprint("| " * (len(stack) + 1)) + # Depth-first, so push the current container onto the ancestry + # stack, then descend … + ancestry.append(node) + self.debugprint(lead + "│ " * 2) for child in node.children: self._walk( child, - stack=stack, + ancestry=ancestry, visitor_fn=visitor_fn, ) - self.debugprint("| " * len(stack)) - assert stack.pop() == node + assert ancestry.pop() == node else: - self.debugprint(f"{lead}{node}") + self.debugprint(f"{lead}├{node}") + + if False and ancestry: + self.debugprint(lead[:-1] + " │") if visitor_fn: - visitor_fn(node, stack, debugprint=self.debugprint) + visitor_fn(node, ancestry, debugprint=self.debugprint) def debugprint(self, s, **kwargs): if self._debug: @@ -408,6 +497,7 @@ def do_massage( cmd_f, *, extensions=None, + cssfile=None, converter=convert_markdown_to_html, only_build=False, tempdir=None, @@ -443,7 +533,9 @@ def do_massage( mimetree = MIMETreeDFWalker(debug=debug_walk) - def visitor_fn(item, stack, *, debugprint=None): + state = dict(pos=1, tags={}, parts=1) + + def visitor_fn(item, ancestry, *, debugprint=None): """ Visitor function called for every node (part) of the MIME tree, depth-first, and responsible for telling NeoMutt how to assemble @@ -460,13 +552,27 @@ def do_massage( # The original source already exists in the NeoMutt tree, but # the underlying file may have been modified, so we need to # update the encoding, but that's it: + cmds.push("") cmds.push("") + + # We really just need to be able to assume that at this point, + # NeoMutt is at position 1, and that we've processed only this + # part so far. Nevermind about actual attachments, we can + # safely ignore those as they stay at the end. + assert state["pos"] == 1 + assert state["parts"] == 1 else: # … whereas all other parts need to be added, and they're all # considered to be temporary and inline: cmds.push(f"{item.path}") cmds.push("") + # This added a part at the end of the list of parts, and that's + # just how many parts we've seen so far, so it's position in + # the NeoMutt compose list is the count of parts + state["parts"] += 1 + state["pos"] = state["parts"] + # If the item (including the original) comes with additional # information, then we might just as well update the NeoMutt # tree now: @@ -475,29 +581,47 @@ def do_massage( elif isinstance(item, Multipart): # This node has children, but we already visited them (see - # above), and so they have been tagged in NeoMutt's compose - # window. Now it's just a matter of telling NeoMutt to do the - # appropriate grouping: + # above). The tags dictionary of State should contain a list of + # their positions in the NeoMutt compose window, so iterate those + # and tag the parts there: + for tag in state["tags"][item]: + cmds.push(f"{tag}") + if item.subtype == "alternative": cmds.push("") elif item.subtype in ("relative", "related"): cmds.push("") elif item.subtype == "multilingual": cmds.push("") + else: + raise NotImplementedError( + f"Handling of multipart/{item.subtype} is not implemented" + ) + + state["pos"] -= len(state["tags"][item]) - 1 + state["parts"] += 1 + del state["tags"][item] else: # We should never get here - assert not "is valid part" + raise RuntimeError(f"Type {type(item)} is unexpected: {item}") # If the item has a description, we might just as well add it if item.desc: cmds.push(f"{KILL_LINE}{item.desc}") - # Finally, if we're at non-root level, tag the new container, - # as it might itself be part of a container, to be processed - # one level up: - if stack: - cmds.push("") + if ancestry: + # If there's an ancestry, record the current (assumed) position in + # the NeoMutt compose window as needed-to-tag by our direct parent + # (i.e. the last item of the ancestry) + state["tags"].setdefault(ancestry[-1], []).append(state["pos"]) + + lead = "│ " * (len(ancestry) + 1) + "* " + debugprint( + f"{lead}ancestry={[a.subtype for a in ancestry]}\n" + f"{lead}children_positions={state['tags'][ancestry[-1]]}\n" + f"{lead}pos={state['pos']}, parts={state['parts']}" + ) # ----------------- # End of visitor_fn @@ -540,6 +664,7 @@ if __name__ == "__main__": pathlib.Path(args.MAILDRAFT), cmd_f, extensions=args.extensions, + cssfile=args.css_file, only_build=args.only_build, tempdir=args.tempdir, debug_commands=args.debug_commands, @@ -609,7 +734,7 @@ try: assert lines[5] in lines_out[7] @pytest.fixture - def basic_mime_tree(self): + def mime_tree_related_to_alternative(self): return Multipart( "relative", children=[ @@ -634,15 +759,49 @@ try: desc="Related", ) - def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree): + @pytest.fixture + def mime_tree_related_to_html(self): + return Multipart( + "alternative", + children=[ + Part( + "text", + "plain", + "part.txt", + desc="Plain", + orig=True, + ), + Multipart( + "relative", + children=[ + Part("text", "html", "part.html", desc="HTML"), + Part( + "text", + "png", + "logo.png", + cid="logo.png", + desc="Logo", + ), + ], + desc="Related", + ), + ], + desc="Alternative", + ) + + def test_MIMETreeDFWalker_depth_first_walk( + self, mime_tree_related_to_alternative + ): mimetree = MIMETreeDFWalker() items = [] - def visitor_fn(item, stack, debugprint): - items.append((item, len(stack))) + def visitor_fn(item, ancestry, debugprint): + items.append((item, len(ancestry))) - mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn) + mimetree.walk( + mime_tree_related_to_alternative, visitor_fn=visitor_fn + ) assert len(items) == 5 assert items[0][0].subtype == "plain" assert items[0][1] == 2 @@ -655,26 +814,29 @@ try: assert items[4][0].subtype == "relative" assert items[4][1] == 0 - def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree): + def test_MIMETreeDFWalker_list_to_mixed(self, const1): mimetree = MIMETreeDFWalker() items = [] - def visitor_fn(item, stack, debugprint): + def visitor_fn(item, ancestry, debugprint): items.append(item) - mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn) + p = Part("text", "plain", const1) + mimetree.walk([p], visitor_fn=visitor_fn) + assert items[-1].subtype == "plain" + mimetree.walk([p, p], visitor_fn=visitor_fn) assert items[-1].subtype == "mixed" def test_MIMETreeDFWalker_visitor_in_constructor( - self, basic_mime_tree + self, mime_tree_related_to_alternative ): items = [] - def visitor_fn(item, stack, debugprint): + def visitor_fn(item, ancestry, debugprint): items.append(item) mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn) - mimetree.walk(basic_mime_tree) + mimetree.walk(mime_tree_related_to_alternative) assert len(items) == 5 @pytest.fixture @@ -682,7 +844,7 @@ try: return StringIO(text or const1) def test_do_massage_basic(self, const1, string_io, capsys): - def converter(drafttext, draftpath, extensions, tempdir): + def converter(drafttext, draftpath, cssfile, extensions, tempdir): return Part("text", "plain", draftpath, orig=True) do_massage( @@ -700,14 +862,15 @@ try: assert "unset my_edit_headers" == lines.pop(0) assert "send-message" in lines.pop(0) assert "update-encoding" in lines.pop(0) + assert "first-entry" in lines.pop(0) assert "source 'rm -f " in lines.pop(0) assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0) def test_do_massage_fulltree( - self, string_io, const1, basic_mime_tree, capsys + self, string_io, const1, mime_tree_related_to_alternative, capsys ): - def converter(drafttext, draftpath, extensions, tempdir): - return basic_mime_tree + def converter(drafttext, draftpath, cssfile, extensions, tempdir): + return mime_tree_related_to_alternative do_massage( draft_f=string_io, @@ -717,26 +880,28 @@ try: ) captured = capsys.readouterr() - lines = captured.out.splitlines()[4:] - assert "send-message" in lines.pop(0) - assert "Related" in lines.pop(0) - assert "group-related" in lines.pop(0) - assert "tag-entry" in lines.pop(0) - assert "Logo" in lines.pop(0) - assert "content-id" in lines.pop(0) - assert "toggle-unlink" in lines.pop(0) - assert "logo.png" in lines.pop(0) - assert "tag-entry" in lines.pop(0) - assert "Alternative" in lines.pop(0) - assert "group-alternatives" in lines.pop(0) - assert "tag-entry" in lines.pop(0) - assert "HTML" in lines.pop(0) - assert "toggle-unlink" in lines.pop(0) - assert "part.html" in lines.pop(0) - assert "tag-entry" in lines.pop(0) - assert "Plain" in lines.pop(0) - assert "update-encoding" in lines.pop(0) - assert len(lines) == 2 + lines = captured.out.splitlines()[4:-2] + print(lines) + assert "first-entry" in lines.pop() + assert "update-encoding" in lines.pop() + assert "Plain" in lines.pop() + assert "part.html" in lines.pop() + assert "toggle-unlink" in lines.pop() + assert "HTML" in lines.pop() + assert "jump>1" in lines.pop() + assert "jump>2" in lines.pop() + assert "group-alternatives" in lines.pop() + assert "Alternative" in lines.pop() + assert "logo.png" in lines.pop() + assert "toggle-unlink" in lines.pop() + assert "content-id" in lines.pop() + assert "Logo" in lines.pop() + assert "jump>1" in lines.pop() + assert "jump>4" in lines.pop() + assert "group-related" in lines.pop() + assert "Related" in lines.pop() + assert "send-message" in lines.pop() + assert len(lines) == 0 @pytest.fixture def fake_filewriter(self): @@ -756,9 +921,7 @@ try: def markdown_non_converter(self, const1, const2): return lambda s, text: f"{const1}{text}{const2}" - def test_converter_tree_basic( - self, const1, const2, fake_filewriter, markdown_non_converter - ): + def test_converter_tree_basic(self, const1, const2, fake_filewriter): path = pathlib.Path(const2) tree = convert_markdown_to_html( const1, path, filewriter_fn=fake_filewriter @@ -877,15 +1040,53 @@ try: text = f"![inline base64 image]({test_png})" path = pathlib.Path(const1) tree = convert_markdown_to_html( - text, path, filewriter_fn=fake_filewriter + text, + path, + filewriter_fn=fake_filewriter, + related_to_html_only=False, ) - assert tree.subtype == "relative" + assert tree.children[0].subtype == "alternative" assert tree.children[1].subtype == "png" written = fake_filewriter.pop() assert tree.children[1].path == written[0] assert written[1] == request.urlopen(test_png).read() + def test_converter_tree_inline_image_base64_related_to_html( + self, test_png, const1, fake_filewriter + ): + text = f"![inline base64 image]({test_png})" + path = pathlib.Path(const1) + tree = convert_markdown_to_html( + text, + path, + filewriter_fn=fake_filewriter, + related_to_html_only=True, + ) + assert tree.subtype == "alternative" + assert tree.children[1].subtype == "relative" + assert tree.children[1].children[1].subtype == "png" + written = fake_filewriter.pop() + assert tree.children[1].children[1].path == written[0] + assert written[1] == request.urlopen(test_png).read() + + def test_converter_tree_inline_image_cid( + self, const1, fake_filewriter + ): + text = f"![inline base64 image](cid:{const1})" + path = pathlib.Path(const1) + tree = convert_markdown_to_html( + text, + path, + filewriter_fn=fake_filewriter, + related_to_html_only=False, + ) + assert len(tree.children) == 2 + assert tree.children[0].cid != const1 + assert tree.children[0].type != "image" + assert tree.children[1].cid != const1 + assert tree.children[1].type != "image" + def test_inline_image_collection( self, test_png, const1, const2, fake_filewriter ): @@ -902,5 +1103,57 @@ try: assert relparts[0].cid == const1 assert relparts[0].desc.endswith(const2) + def test_apply_stylesheet(self): + if _PYNLINER: + html = "

Hello, world!

" + css = "p { color:red }" + out = apply_styling(html, css) + assert 'p style="color' in out + + def test_apply_stylesheet_pygments(self): + if _PYGMENTS_CSS: + html = ( + f'
' + "
def foo():\n    return
" + ) + out = apply_styling(html, _PYGMENTS_CSS) + assert f'{_CODEHILITE_CLASS}" style="' in out + + def test_mime_tree_relative_within_alternative( + self, string_io, const1, capsys, mime_tree_related_to_html + ): + def converter(drafttext, draftpath, cssfile, extensions, tempdir): + return mime_tree_related_to_html + + do_massage( + draft_f=string_io, + draftpath=const1, + cmd_f=sys.stdout, + converter=converter, + ) + + captured = capsys.readouterr() + lines = captured.out.splitlines()[4:-2] + assert "first-entry" in lines.pop() + assert "update-encoding" in lines.pop() + assert "Plain" in lines.pop() + assert "part.html" in lines.pop() + assert "toggle-unlink" in lines.pop() + assert "HTML" in lines.pop() + assert "logo.png" in lines.pop() + assert "toggle-unlink" in lines.pop() + assert "content-id" in lines.pop() + assert "Logo" in lines.pop() + assert "jump>2" in lines.pop() + assert "jump>3" in lines.pop() + assert "group-related" in lines.pop() + assert "Related" in lines.pop() + assert "jump>1" in lines.pop() + assert "jump>2" in lines.pop() + assert "group-alternative" in lines.pop() + assert "Alternative" in lines.pop() + assert "send-message" in lines.pop() + assert len(lines) == 0 + except ImportError: pass