+#!/usr/bin/python3
+#
+# NeoMutt helper script to create multipart/* emails with Markdown → HTML
+# alternative conversion, and handling of inline images, using NeoMutt's
+# ability to manually craft MIME trees, but automating this process.
+#
+# Configuration:
+# neomuttrc (needs to be a single line):
+# macro compose B "\
+# <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
+# <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
+# " "Convert message into a modern MIME tree with inline images"
+#
+# (Yes, we need to call source twice, as mutt only starts to process output
+# from a source command when the command exits, and since we need to react
+# to the output, we need to be invoked again, using a $my_ variable to pass
+# information)
+#
+# Requirements:
+# - python3
+# - python3-markdown
+# Optional:
+# - pytest
+# - Pynliner
+# - Pygments, if installed, then syntax highlighting is enabled
+#
+# Latest version:
+# https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
+#
+# Copyright © 2023 martin f. krafft <madduck@madduck.net>
+# Released under the GPL-2+ licence, just like Mutt itself.
+#
+
+import sys
+import pathlib
+import markdown
+import tempfile
+import argparse
+from collections import namedtuple
+
+
+def parse_cli_args(*args, **kwargs):
+ parser = argparse.ArgumentParser(
+ description=(
+ "NeoMutt helper to turn text/markdown email parts "
+ "into full-fledged MIME trees"
+ )
+ )
+ parser.epilog = (
+ "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
+ "Released under the MIT licence"
+ )
+
+ subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
+ parser_setup = subp.add_parser("setup", help="Setup phase")
+ parser_massage = subp.add_parser("massage", help="Massaging phase")
+
+ parser_setup.add_argument(
+ "--debug-commands",
+ action="store_true",
+ help="Turn on debug logging of commands generated to stderr",
+ )
+
+ parser_setup.add_argument(
+ "--extension",
+ "-x",
+ metavar="EXTENSION",
+ dest="extensions",
+ nargs="?",
+ default=[],
+ action="append",
+ help="Markdown extension to add to the list of extensions use",
+ )
+
+ parser_massage.add_argument(
+ "--debug-commands",
+ action="store_true",
+ help="Turn on debug logging of commands generated to stderr",
+ )
+
+ parser_massage.add_argument(
+ "--debug-walk",
+ action="store_true",
+ help="Turn on debugging to stderr of the MIME tree walk",
+ )
+
+ parser_massage.add_argument(
+ "--extensions",
+ metavar="EXTENSIONS",
+ type=str,
+ default="",
+ help="Markdown extension to use (comma-separated list)",
+ )
+
+ parser_massage.add_argument(
+ "--write-commands-to",
+ metavar="PATH",
+ dest="cmdpath",
+ help="Temporary file path to write commands to",
+ )
+
+ parser_massage.add_argument(
+ "MAILDRAFT",
+ nargs="?",
+ help="If provided, the script is invoked as editor on the mail draft",
+ )
+
+ return parser.parse_args(*args, **kwargs)
+
+
+# [ PARTS GENERATION ] ########################################################
+
+
+class Part(
+ namedtuple(
+ "Part",
+ ["type", "subtype", "path", "desc", "cid", "orig"],
+ defaults=[None, None, False],
+ )
+):
+ def __str__(self):
+ ret = f"<{self.type}/{self.subtype}>"
+ if self.cid:
+ ret = f"{ret} cid:{self.cid}"
+ if self.orig:
+ ret = f"{ret} ORIGINAL"
+ return ret
+
+
+class Multipart(
+ namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
+):
+ def __str__(self):
+ return f"<multipart/{self.subtype}> children={len(self.children)}"
+
+
+def convert_markdown_to_html(maildraft, *, extensions=None):
+ draftpath = pathlib.Path(maildraft)
+ textpart = Part(
+ "text", "plain", draftpath, "Plain-text version", orig=True
+ )
+
+ with open(draftpath, "r", encoding="utf-8") as textmarkdown:
+ text = textmarkdown.read()
+
+ mdwn = markdown.Markdown(extensions=extensions)
+ html = mdwn.convert(text)
+
+ htmlpath = draftpath.with_suffix(".html")
+ htmlpart = Part("text", "html", htmlpath, "HTML version")
+
+ with open(
+ htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
+ ) as texthtml:
+ texthtml.write(html)
+
+ logopart = Part(
+ "image",
+ "png",
+ "/usr/share/doc/neomutt/logo/neomutt-256.png",
+ "Logo",
+ "neomutt-256.png",
+ )
+
+ return Multipart(
+ "relative",
+ [
+ Multipart(
+ "alternative",
+ [textpart, htmlpart],
+ "Group of alternative content",
+ ),
+ logopart,
+ ],
+ "Group of related content",
+ )
+
+
+class MIMETreeDFWalker:
+ def __init__(self, *, visitor_fn=None, debug=False):
+ self._visitor_fn = visitor_fn
+ self._debug = debug
+
+ def walk(self, root, *, visitor_fn=None):
+ """
+ Recursive function to implement a depth-dirst walk of the MIME-tree
+ rooted at `root`.
+ """
+
+ if isinstance(root, list):
+ root = Multipart("mixed", children=root)
+
+ self._walk(
+ root,
+ stack=[],
+ visitor_fn=visitor_fn or self._visitor_fn,
+ )
+
+ def _walk(self, node, *, stack, visitor_fn):
+ # Let's start by enumerating the parts at the current level. At the
+ # root level, stack will be the empty list, and we expect a multipart/*
+ # container at this level. Later, e.g. within a mutlipart/alternative
+ # container, the subtree will just be the alternative parts, while the
+ # top of the stack will be the multipart/alternative container, which
+ # we will process after the following loop.
+
+ lead = f"{'| '*len(stack)}|-"
+ if isinstance(node, Multipart):
+ self.debugprint(
+ f"{lead}{node} parents={[s.subtype for s in stack]}"
+ )
+
+ # Depth-first, so push the current container onto the stack,
+ # then descend …
+ stack.append(node)
+ self.debugprint("| " * (len(stack) + 1))
+ for child in node.children:
+ self._walk(
+ child,
+ stack=stack,
+ visitor_fn=visitor_fn,
+ )
+ self.debugprint("| " * len(stack))
+ assert stack.pop() == node
+
+ else:
+ self.debugprint(f"{lead}{node}")
+
+ if visitor_fn:
+ visitor_fn(node, stack, debugprint=self.debugprint)
+
+ def debugprint(self, s, **kwargs):
+ if self._debug:
+ print(s, file=sys.stderr, **kwargs)
+
+
+# [ RUN MODES ] ###############################################################
+
+
+class MuttCommands:
+ """
+ Stupid class to interface writing out Mutt commands. This is quite a hack
+ to deal with the fact that Mutt runs "push" commands in reverse order, so
+ all of a sudden, things become very complicated when mixing with "real"
+ commands.
+
+ Hence we keep two sets of commands, and one set of pushes. Commands are
+ added to the first until a push is added, after which commands are added to
+ the second set of commands.
+
+ On flush(), the first set is printed, followed by the pushes in reverse,
+ and then the second set is printed. All 3 sets are then cleared.
+ """
+
+ def __init__(self, out_f=sys.stdout, *, debug=False):
+ self._cmd1, self._push, self._cmd2 = [], [], []
+ self._out_f = out_f
+ self._debug = debug
+
+ def cmd(self, s):
+ self.debugprint(s)
+ if self._push:
+ self._cmd2.append(s)
+ else:
+ self._cmd1.append(s)
+
+ def push(self, s):
+ s = s.replace('"', '"')
+ s = f'push "{s}"'
+ self.debugprint(s)
+ self._push.insert(0, s)
+
+ def flush(self):
+ print(
+ "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
+ )
+ self._cmd1, self._push, self._cmd2 = [], [], []
+
+ def debugprint(self, s, **kwargs):
+ if self._debug:
+ print(s, file=sys.stderr, **kwargs)
+
+
+def do_setup(
+ extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
+):
+ extensions = extensions or []
+ temppath = temppath or pathlib.Path(
+ tempfile.mkstemp(prefix="muttmdwn-")[1]
+ )
+ cmds = MuttCommands(out_f, debug=debug_commands)
+
+ editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
+ if extensions:
+ editor = f'{editor} --extensions {",".join(extensions)}'
+
+ cmds.cmd('set my_editor="$editor"')
+ cmds.cmd('set my_edit_headers="$edit_headers"')
+ cmds.cmd(f'set editor="{editor}"')
+ cmds.cmd("unset edit_headers")
+ cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
+ cmds.push("<first-entry><edit-file>")
+ cmds.flush()
+
+
+def do_massage(
+ maildraft,
+ cmdpath,
+ *,
+ extensions=None,
+ converter=convert_markdown_to_html,
+ debug_commands=False,
+ debug_walk=False,
+):
+ # Here's the big picture: we're being invoked as the editor on the email
+ # draft, and whatever commands we write to the file given as cmdpath will
+ # be run by the second source command in the macro definition.
+
+ with open(cmdpath, "w") as cmd_f:
+ # Let's start by cleaning up what the setup did (see above), i.e. we
+ # restore the $editor and $edit_headers variables, and also unset the
+ # variable used to identify the command file we're currently writing
+ # to.
+ cmds = MuttCommands(cmd_f, debug=debug_commands)
+ cmds.cmd('set editor="$my_editor"')
+ cmds.cmd('set edit_headers="$my_edit_headers"')
+ cmds.cmd("unset my_editor")
+ cmds.cmd("unset my_edit_headers")
+
+ # let's flush those commands, as there'll be a lot of pushes from now
+ # on, which need to be run in reverse order
+ cmds.flush()
+
+ extensions = extensions.split(",") if extensions else []
+ tree = converter(maildraft, extensions=extensions)
+
+ mimetree = MIMETreeDFWalker(debug=args.debug_walk)
+
+ def visitor_fn(item, stack, *, debugprint=None):
+ """
+ Visitor function called for every node (part) of the MIME tree,
+ depth-first, and responsible for telling NeoMutt how to assemble
+ the tree.
+ """
+ if isinstance(item, Part):
+ # We've hit a leaf-node, i.e. an alternative or a related part
+ # with actual content.
+
+ # If the part is not an original part, i.e. doesn't already
+ # exist, we must first add it.
+ if not item.orig:
+ cmds.push(f"<attach-file>{item.path}<enter>")
+ cmds.push("<toggle-unlink><toggle-disposition>")
+ if item.cid:
+ cmds.push(
+ f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>"
+ )
+
+ # If the item (including the original) comes with a
+ # description, then we might just as well update the NeoMutt
+ # tree now:
+ if item.desc:
+ cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
+
+ # Finally, tag the entry that we just processed, so that when
+ # we're done at this level, as we walk up the stack, the items
+ # to be grouped will already be tagged and ready.
+ cmds.push("<tag-entry>")
+
+ elif isinstance(item, Multipart):
+ # This node has children, but we already visited them (see
+ # above), and so they have been tagged in NeoMutt's compose
+ # window. Now it's just a matter of telling NeoMutt to do the
+ # appropriate grouping:
+ if item.subtype == "alternative":
+ cmds.push("<group-alternatives>")
+ elif item.subtype == "relative":
+ cmds.push("<group-related>")
+ elif item.subtype == "multilingual":
+ cmds.push("<group-multilingual>")
+
+ # Again, if there is a description, we might just as well:
+ if item.desc:
+ cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
+
+ # Finally, if we're at non-root level, tag the new container,
+ # as it might itself be part of a container, to be processed
+ # one level up:
+ if stack:
+ cmds.push("<tag-entry>")
+
+ else:
+ # We should never get here
+ assert not "is valid part"
+
+ # -----------------
+ # End of visitor_fn
+
+ # Let's walk the tree and visit every node with our fancy visitor
+ # function
+ mimetree.walk(tree, visitor_fn=visitor_fn)
+
+ # Finally, cleanup. Since we're responsible for removing the temporary
+ # file, how's this for a little hack?
+ cmds.cmd(f"source 'rm -f {args.cmdpath}|'")
+ cmds.cmd("unset my_mdwn_postprocess_cmd_file")
+ cmds.flush()
+
+
+# [ CLI ENTRY ] ###############################################################
+
+if __name__ == "__main__":
+ args = parse_cli_args()
+
+ if args.mode == "setup":
+ do_setup(args.extensions, debug_commands=args.debug_commands)
+
+ elif args.mode == "massage":
+ do_massage(
+ args.MAILDRAFT,
+ args.cmdpath,
+ extensions=args.extensions,
+ debug_commands=args.debug_commands,
+ debug_walk=args.debug_walk,
+ )
+
+
+# [ TESTS ] ###################################################################
+
+try:
+ import pytest
+
+ class Tests:
+ @pytest.fixture
+ def const1(self):
+ return "CONSTANT STRING 1"
+
+ @pytest.fixture
+ def const2(self):
+ return "CONSTANT STRING 2"
+
+ # NOTE: tests using the capsys fixture must specify sys.stdout to the
+ # functions they call, else old stdout is used and not captured
+
+ def test_MuttCommands_cmd(self, const1, const2, capsys):
+ "Assert order of commands"
+ cmds = MuttCommands(out_f=sys.stdout)
+ cmds.cmd(const1)
+ cmds.cmd(const2)
+ cmds.flush()
+ captured = capsys.readouterr()
+ assert captured.out == "\n".join((const1, const2, ""))
+
+ def test_MuttCommands_push(self, const1, const2, capsys):
+ "Assert reverse order of pushes"
+ cmds = MuttCommands(out_f=sys.stdout)
+ cmds.push(const1)
+ cmds.push(const2)
+ cmds.flush()
+ captured = capsys.readouterr()
+ assert (
+ captured.out
+ == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
+ )
+
+ def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
+ "Assert reverse order of pushes"
+ cmds = MuttCommands(out_f=sys.stdout)
+ lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
+ for i in range(2):
+ cmds.cmd(lines[4 * i + 0])
+ cmds.cmd(lines[4 * i + 1])
+ cmds.push(lines[4 * i + 2])
+ cmds.push(lines[4 * i + 3])
+ cmds.flush()
+
+ captured = capsys.readouterr()
+ lines_out = captured.out.splitlines()
+ assert lines[0] in lines_out[0]
+ assert lines[1] in lines_out[1]
+ assert lines[7] in lines_out[2]
+ assert lines[6] in lines_out[3]
+ assert lines[3] in lines_out[4]
+ assert lines[2] in lines_out[5]
+ assert lines[4] in lines_out[6]
+ assert lines[5] in lines_out[7]
+
+ @pytest.fixture
+ def basic_mime_tree(self):
+ return Multipart(
+ "related",
+ children=[
+ Multipart(
+ "alternative",
+ children=[
+ Part("text", "plain", "part.txt", desc="Plain"),
+ Part("text", "html", "part.html", desc="HTML"),
+ ],
+ desc="Alternative",
+ ),
+ Part(
+ "text", "png", "logo.png", cid="logo.png", desc="Logo"
+ ),
+ ],
+ desc="Related",
+ )
+
+ def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
+ mimetree = MIMETreeDFWalker()
+
+ items = []
+
+ def visitor_fn(item, stack, debugprint):
+ items.append((item, len(stack)))
+
+ mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
+ assert len(items) == 5
+ assert items[0][0].subtype == "plain"
+ assert items[0][1] == 2
+ assert items[1][0].subtype == "html"
+ assert items[1][1] == 2
+ assert items[2][0].subtype == "alternative"
+ assert items[2][1] == 1
+ assert items[3][0].subtype == "png"
+ assert items[3][1] == 1
+ assert items[4][0].subtype == "related"
+ assert items[4][1] == 0
+
+ def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
+ mimetree = MIMETreeDFWalker()
+
+ items = []
+
+ def visitor_fn(item, stack, debugprint):
+ items.append(item)
+
+ mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
+ assert items[-1].subtype == "mixed"
+
+ def test_MIMETreeDFWalker_visitor_in_constructor(
+ self, basic_mime_tree
+ ):
+ items = []
+
+ def visitor_fn(item, stack, debugprint):
+ items.append(item)
+
+ mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
+ mimetree.walk(basic_mime_tree)
+ assert len(items) == 5
+
+ def test_do_setup_no_extensions(self, const1, capsys):
+ "Assert basics about the setup command output"
+ do_setup(temppath=const1, out_f=sys.stdout)
+ captout = capsys.readouterr()
+ lines = captout.out.splitlines()
+ assert lines[2].endswith(f'{const1}"')
+ assert lines[4].endswith(const1)
+ assert "first-entry" in lines[-1]
+ assert "edit-file" in lines[-1]
+
+ def test_do_setup_extensions(self, const1, const2, capsys):
+ "Assert that extensions are passed to editor"
+ do_setup(
+ temppath=const1, extensions=[const2, const1], out_f=sys.stdout
+ )
+ captout = capsys.readouterr()
+ lines = captout.out.splitlines()
+ # assert comma-separated list of extensions passed
+ assert lines[2].endswith(f'{const2},{const1}"')
+ assert lines[4].endswith(const1)
+
+except ImportError:
+ pass
projects / etc / neomutt.git / commitdiff