From: martin f. krafft Date: Tue, 22 Aug 2023 11:01:50 +0000 (+1200) Subject: WIP buildmimetree.py X-Git-Url: https://git.madduck.net/etc/neomutt.git/commitdiff_plain/a6fa49f581a117f8344b700ba5ca522206b278b5?ds=sidebyside;hp=4d1afcd5cc3e282554877a10d6be81931ef0cc03 WIP buildmimetree.py --- diff --git a/.config/neomutt/__pycache__/buildmimetree.cpython-311-pytest-7.4.0.pyc b/.config/neomutt/__pycache__/buildmimetree.cpython-311-pytest-7.4.0.pyc deleted file mode 100644 index 0df1cec..0000000 Binary files a/.config/neomutt/__pycache__/buildmimetree.cpython-311-pytest-7.4.0.pyc and /dev/null differ diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py new file mode 100755 index 0000000..d0493b3 --- /dev/null +++ b/.config/neomutt/buildmimetree.py @@ -0,0 +1,574 @@ +#!/usr/bin/python3 +# +# NeoMutt helper script to create multipart/* emails with Markdown → HTML +# alternative conversion, and handling of inline images, using NeoMutt's +# ability to manually craft MIME trees, but automating this process. +# +# Configuration: +# neomuttrc (needs to be a single line): +# macro compose B "\ +# source '$my_confdir/buildmimetree.py setup|'\ +# sourc e \$my_mdwn_postprocess_cmd_file\ +# " "Convert message into a modern MIME tree with inline images" +# +# (Yes, we need to call source twice, as mutt only starts to process output +# from a source command when the command exits, and since we need to react +# to the output, we need to be invoked again, using a $my_ variable to pass +# information) +# +# Requirements: +# - python3 +# - python3-markdown +# Optional: +# - pytest +# - Pynliner +# - Pygments, if installed, then syntax highlighting is enabled +# +# Latest version: +# https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py +# +# Copyright © 2023 martin f. krafft +# Released under the GPL-2+ licence, just like Mutt itself. +# + +import sys +import pathlib +import markdown +import tempfile +import argparse +from collections import namedtuple + + +def parse_cli_args(*args, **kwargs): + parser = argparse.ArgumentParser( + description=( + "NeoMutt helper to turn text/markdown email parts " + "into full-fledged MIME trees" + ) + ) + parser.epilog = ( + "Copyright © 2022 martin f. krafft .\n" + "Released under the MIT licence" + ) + + subp = parser.add_subparsers(help="Sub-command parsers", dest="mode") + parser_setup = subp.add_parser("setup", help="Setup phase") + parser_massage = subp.add_parser("massage", help="Massaging phase") + + parser_setup.add_argument( + "--debug-commands", + action="store_true", + help="Turn on debug logging of commands generated to stderr", + ) + + parser_setup.add_argument( + "--extension", + "-x", + metavar="EXTENSION", + dest="extensions", + nargs="?", + default=[], + action="append", + help="Markdown extension to add to the list of extensions use", + ) + + parser_massage.add_argument( + "--debug-commands", + action="store_true", + help="Turn on debug logging of commands generated to stderr", + ) + + parser_massage.add_argument( + "--debug-walk", + action="store_true", + help="Turn on debugging to stderr of the MIME tree walk", + ) + + parser_massage.add_argument( + "--extensions", + metavar="EXTENSIONS", + type=str, + default="", + help="Markdown extension to use (comma-separated list)", + ) + + parser_massage.add_argument( + "--write-commands-to", + metavar="PATH", + dest="cmdpath", + help="Temporary file path to write commands to", + ) + + parser_massage.add_argument( + "MAILDRAFT", + nargs="?", + help="If provided, the script is invoked as editor on the mail draft", + ) + + return parser.parse_args(*args, **kwargs) + + +# [ PARTS GENERATION ] ######################################################## + + +class Part( + namedtuple( + "Part", + ["type", "subtype", "path", "desc", "cid", "orig"], + defaults=[None, None, False], + ) +): + def __str__(self): + ret = f"<{self.type}/{self.subtype}>" + if self.cid: + ret = f"{ret} cid:{self.cid}" + if self.orig: + ret = f"{ret} ORIGINAL" + return ret + + +class Multipart( + namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None]) +): + def __str__(self): + return f" children={len(self.children)}" + + +def convert_markdown_to_html(maildraft, *, extensions=None): + draftpath = pathlib.Path(maildraft) + textpart = Part( + "text", "plain", draftpath, "Plain-text version", orig=True + ) + + with open(draftpath, "r", encoding="utf-8") as textmarkdown: + text = textmarkdown.read() + + mdwn = markdown.Markdown(extensions=extensions) + html = mdwn.convert(text) + + htmlpath = draftpath.with_suffix(".html") + htmlpart = Part("text", "html", htmlpath, "HTML version") + + with open( + htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace" + ) as texthtml: + texthtml.write(html) + + logopart = Part( + "image", + "png", + "/usr/share/doc/neomutt/logo/neomutt-256.png", + "Logo", + "neomutt-256.png", + ) + + return Multipart( + "relative", + [ + Multipart( + "alternative", + [textpart, htmlpart], + "Group of alternative content", + ), + logopart, + ], + "Group of related content", + ) + + +class MIMETreeDFWalker: + def __init__(self, *, visitor_fn=None, debug=False): + self._visitor_fn = visitor_fn + self._debug = debug + + def walk(self, root, *, visitor_fn=None): + """ + Recursive function to implement a depth-dirst walk of the MIME-tree + rooted at `root`. + """ + + if isinstance(root, list): + root = Multipart("mixed", children=root) + + self._walk( + root, + stack=[], + visitor_fn=visitor_fn or self._visitor_fn, + ) + + def _walk(self, node, *, stack, visitor_fn): + # Let's start by enumerating the parts at the current level. At the + # root level, stack will be the empty list, and we expect a multipart/* + # container at this level. Later, e.g. within a mutlipart/alternative + # container, the subtree will just be the alternative parts, while the + # top of the stack will be the multipart/alternative container, which + # we will process after the following loop. + + lead = f"{'| '*len(stack)}|-" + if isinstance(node, Multipart): + self.debugprint( + f"{lead}{node} parents={[s.subtype for s in stack]}" + ) + + # Depth-first, so push the current container onto the stack, + # then descend … + stack.append(node) + self.debugprint("| " * (len(stack) + 1)) + for child in node.children: + self._walk( + child, + stack=stack, + visitor_fn=visitor_fn, + ) + self.debugprint("| " * len(stack)) + assert stack.pop() == node + + else: + self.debugprint(f"{lead}{node}") + + if visitor_fn: + visitor_fn(node, stack, debugprint=self.debugprint) + + def debugprint(self, s, **kwargs): + if self._debug: + print(s, file=sys.stderr, **kwargs) + + +# [ RUN MODES ] ############################################################### + + +class MuttCommands: + """ + Stupid class to interface writing out Mutt commands. This is quite a hack + to deal with the fact that Mutt runs "push" commands in reverse order, so + all of a sudden, things become very complicated when mixing with "real" + commands. + + Hence we keep two sets of commands, and one set of pushes. Commands are + added to the first until a push is added, after which commands are added to + the second set of commands. + + On flush(), the first set is printed, followed by the pushes in reverse, + and then the second set is printed. All 3 sets are then cleared. + """ + + def __init__(self, out_f=sys.stdout, *, debug=False): + self._cmd1, self._push, self._cmd2 = [], [], [] + self._out_f = out_f + self._debug = debug + + def cmd(self, s): + self.debugprint(s) + if self._push: + self._cmd2.append(s) + else: + self._cmd1.append(s) + + def push(self, s): + s = s.replace('"', '"') + s = f'push "{s}"' + self.debugprint(s) + self._push.insert(0, s) + + def flush(self): + print( + "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f + ) + self._cmd1, self._push, self._cmd2 = [], [], [] + + def debugprint(self, s, **kwargs): + if self._debug: + print(s, file=sys.stderr, **kwargs) + + +def do_setup( + extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False +): + extensions = extensions or [] + temppath = temppath or pathlib.Path( + tempfile.mkstemp(prefix="muttmdwn-")[1] + ) + cmds = MuttCommands(out_f, debug=debug_commands) + + editor = f"{sys.argv[0]} massage --write-commands-to {temppath}" + if extensions: + editor = f'{editor} --extensions {",".join(extensions)}' + + cmds.cmd('set my_editor="$editor"') + cmds.cmd('set my_edit_headers="$edit_headers"') + cmds.cmd(f'set editor="{editor}"') + cmds.cmd("unset edit_headers") + cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}") + cmds.push("") + cmds.flush() + + +def do_massage( + maildraft, + cmdpath, + *, + extensions=None, + converter=convert_markdown_to_html, + debug_commands=False, + debug_walk=False, +): + # Here's the big picture: we're being invoked as the editor on the email + # draft, and whatever commands we write to the file given as cmdpath will + # be run by the second source command in the macro definition. + + with open(cmdpath, "w") as cmd_f: + # Let's start by cleaning up what the setup did (see above), i.e. we + # restore the $editor and $edit_headers variables, and also unset the + # variable used to identify the command file we're currently writing + # to. + cmds = MuttCommands(cmd_f, debug=debug_commands) + cmds.cmd('set editor="$my_editor"') + cmds.cmd('set edit_headers="$my_edit_headers"') + cmds.cmd("unset my_editor") + cmds.cmd("unset my_edit_headers") + + # let's flush those commands, as there'll be a lot of pushes from now + # on, which need to be run in reverse order + cmds.flush() + + extensions = extensions.split(",") if extensions else [] + tree = converter(maildraft, extensions=extensions) + + mimetree = MIMETreeDFWalker(debug=args.debug_walk) + + def visitor_fn(item, stack, *, debugprint=None): + """ + Visitor function called for every node (part) of the MIME tree, + depth-first, and responsible for telling NeoMutt how to assemble + the tree. + """ + if isinstance(item, Part): + # We've hit a leaf-node, i.e. an alternative or a related part + # with actual content. + + # If the part is not an original part, i.e. doesn't already + # exist, we must first add it. + if not item.orig: + cmds.push(f"{item.path}") + cmds.push("") + if item.cid: + cmds.push( + f"\\Ca\\Ck{item.cid}" + ) + + # If the item (including the original) comes with a + # description, then we might just as well update the NeoMutt + # tree now: + if item.desc: + cmds.push(f"\\Ca\\Ck{item.desc}") + + # Finally, tag the entry that we just processed, so that when + # we're done at this level, as we walk up the stack, the items + # to be grouped will already be tagged and ready. + cmds.push("") + + elif isinstance(item, Multipart): + # This node has children, but we already visited them (see + # above), and so they have been tagged in NeoMutt's compose + # window. Now it's just a matter of telling NeoMutt to do the + # appropriate grouping: + if item.subtype == "alternative": + cmds.push("") + elif item.subtype == "relative": + cmds.push("") + elif item.subtype == "multilingual": + cmds.push("") + + # Again, if there is a description, we might just as well: + if item.desc: + cmds.push(f"\\Ca\\Ck{item.desc}") + + # Finally, if we're at non-root level, tag the new container, + # as it might itself be part of a container, to be processed + # one level up: + if stack: + cmds.push("") + + else: + # We should never get here + assert not "is valid part" + + # ----------------- + # End of visitor_fn + + # Let's walk the tree and visit every node with our fancy visitor + # function + mimetree.walk(tree, visitor_fn=visitor_fn) + + # Finally, cleanup. Since we're responsible for removing the temporary + # file, how's this for a little hack? + cmds.cmd(f"source 'rm -f {args.cmdpath}|'") + cmds.cmd("unset my_mdwn_postprocess_cmd_file") + cmds.flush() + + +# [ CLI ENTRY ] ############################################################### + +if __name__ == "__main__": + args = parse_cli_args() + + if args.mode == "setup": + do_setup(args.extensions, debug_commands=args.debug_commands) + + elif args.mode == "massage": + do_massage( + args.MAILDRAFT, + args.cmdpath, + extensions=args.extensions, + debug_commands=args.debug_commands, + debug_walk=args.debug_walk, + ) + + +# [ TESTS ] ################################################################### + +try: + import pytest + + class Tests: + @pytest.fixture + def const1(self): + return "CONSTANT STRING 1" + + @pytest.fixture + def const2(self): + return "CONSTANT STRING 2" + + # NOTE: tests using the capsys fixture must specify sys.stdout to the + # functions they call, else old stdout is used and not captured + + def test_MuttCommands_cmd(self, const1, const2, capsys): + "Assert order of commands" + cmds = MuttCommands(out_f=sys.stdout) + cmds.cmd(const1) + cmds.cmd(const2) + cmds.flush() + captured = capsys.readouterr() + assert captured.out == "\n".join((const1, const2, "")) + + def test_MuttCommands_push(self, const1, const2, capsys): + "Assert reverse order of pushes" + cmds = MuttCommands(out_f=sys.stdout) + cmds.push(const1) + cmds.push(const2) + cmds.flush() + captured = capsys.readouterr() + assert ( + captured.out + == ('"\npush "'.join(("", const2, const1, "")))[2:-6] + ) + + def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys): + "Assert reverse order of pushes" + cmds = MuttCommands(out_f=sys.stdout) + lines = ["000", "001", "010", "011", "100", "101", "110", "111"] + for i in range(2): + cmds.cmd(lines[4 * i + 0]) + cmds.cmd(lines[4 * i + 1]) + cmds.push(lines[4 * i + 2]) + cmds.push(lines[4 * i + 3]) + cmds.flush() + + captured = capsys.readouterr() + lines_out = captured.out.splitlines() + assert lines[0] in lines_out[0] + assert lines[1] in lines_out[1] + assert lines[7] in lines_out[2] + assert lines[6] in lines_out[3] + assert lines[3] in lines_out[4] + assert lines[2] in lines_out[5] + assert lines[4] in lines_out[6] + assert lines[5] in lines_out[7] + + @pytest.fixture + def basic_mime_tree(self): + return Multipart( + "related", + children=[ + Multipart( + "alternative", + children=[ + Part("text", "plain", "part.txt", desc="Plain"), + Part("text", "html", "part.html", desc="HTML"), + ], + desc="Alternative", + ), + Part( + "text", "png", "logo.png", cid="logo.png", desc="Logo" + ), + ], + desc="Related", + ) + + def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree): + mimetree = MIMETreeDFWalker() + + items = [] + + def visitor_fn(item, stack, debugprint): + items.append((item, len(stack))) + + mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn) + assert len(items) == 5 + assert items[0][0].subtype == "plain" + assert items[0][1] == 2 + assert items[1][0].subtype == "html" + assert items[1][1] == 2 + assert items[2][0].subtype == "alternative" + assert items[2][1] == 1 + assert items[3][0].subtype == "png" + assert items[3][1] == 1 + assert items[4][0].subtype == "related" + assert items[4][1] == 0 + + def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree): + mimetree = MIMETreeDFWalker() + + items = [] + + def visitor_fn(item, stack, debugprint): + items.append(item) + + mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn) + assert items[-1].subtype == "mixed" + + def test_MIMETreeDFWalker_visitor_in_constructor( + self, basic_mime_tree + ): + items = [] + + def visitor_fn(item, stack, debugprint): + items.append(item) + + mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn) + mimetree.walk(basic_mime_tree) + assert len(items) == 5 + + def test_do_setup_no_extensions(self, const1, capsys): + "Assert basics about the setup command output" + do_setup(temppath=const1, out_f=sys.stdout) + captout = capsys.readouterr() + lines = captout.out.splitlines() + assert lines[2].endswith(f'{const1}"') + assert lines[4].endswith(const1) + assert "first-entry" in lines[-1] + assert "edit-file" in lines[-1] + + def test_do_setup_extensions(self, const1, const2, capsys): + "Assert that extensions are passed to editor" + do_setup( + temppath=const1, extensions=[const2, const1], out_f=sys.stdout + ) + captout = capsys.readouterr() + lines = captout.out.splitlines() + # assert comma-separated list of extensions passed + assert lines[2].endswith(f'{const2},{const1}"') + assert lines[4].endswith(const1) + +except ImportError: + pass diff --git a/.config/neomutt/neomuttrc b/.config/neomutt/neomuttrc index 3c02cde..5b29048 100644 --- a/.config/neomutt/neomuttrc +++ b/.config/neomutt/neomuttrc @@ -61,3 +61,5 @@ source "test -f $alias_file && cat $alias_file 2>/dev/null || echo unset alias_f #set index_format="%4C %?GU?%GU& ?%?GR?%GR& ?%?GI?%GI& ? %-10@date@ %-15.15F %4c%?M?/[%M]? %?H?[%H] ?%s" # #set pager_format="<%a> %* %J (%P)" + +macro compose B " source '$my_confdir/buildmimetree.py setup|' source \$my_mdwn_postprocess_cmd_file" "Convert message into a modern MIME tree with inline images"