From: martin f. krafft <madduck@madduck.net>
Date: Tue, 22 Aug 2023 11:01:50 +0000 (+1200)
Subject: WIP buildmimetree.py
X-Git-Url: https://git.madduck.net/etc/neomutt.git/commitdiff_plain/a6fa49f581a117f8344b700ba5ca522206b278b5?ds=sidebyside;hp=4d1afcd5cc3e282554877a10d6be81931ef0cc03

WIP buildmimetree.py
---

diff --git a/.config/neomutt/__pycache__/buildmimetree.cpython-311-pytest-7.4.0.pyc b/.config/neomutt/__pycache__/buildmimetree.cpython-311-pytest-7.4.0.pyc
deleted file mode 100644
index 0df1cec..0000000
Binary files a/.config/neomutt/__pycache__/buildmimetree.cpython-311-pytest-7.4.0.pyc and /dev/null differ
diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py
new file mode 100755
index 0000000..d0493b3
--- /dev/null
+++ b/.config/neomutt/buildmimetree.py
@@ -0,0 +1,574 @@
+#!/usr/bin/python3
+#
+# NeoMutt helper script to create multipart/* emails with Markdown → HTML
+# alternative conversion, and handling of inline images, using NeoMutt's
+# ability to manually craft MIME trees, but automating this process.
+#
+# Configuration:
+#   neomuttrc (needs to be a single line):
+#     macro compose B "\
+#       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
+#       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
+#     " "Convert message into a modern MIME tree with inline images"
+#
+#     (Yes, we need to call source twice, as mutt only starts to process output
+#     from a source command when the command exits, and since we need to react
+#     to the output, we need to be invoked again, using a $my_ variable to pass
+#     information)
+#
+# Requirements:
+#   - python3
+#   - python3-markdown
+# Optional:
+#   - pytest
+#   - Pynliner
+#   - Pygments, if installed, then syntax highlighting is enabled
+#
+# Latest version:
+#   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
+#
+# Copyright © 2023 martin f. krafft <madduck@madduck.net>
+# Released under the GPL-2+ licence, just like Mutt itself.
+#
+
+import sys
+import pathlib
+import markdown
+import tempfile
+import argparse
+from collections import namedtuple
+
+
+def parse_cli_args(*args, **kwargs):
+    parser = argparse.ArgumentParser(
+        description=(
+            "NeoMutt helper to turn text/markdown email parts "
+            "into full-fledged MIME trees"
+        )
+    )
+    parser.epilog = (
+        "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
+        "Released under the MIT licence"
+    )
+
+    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
+    parser_setup = subp.add_parser("setup", help="Setup phase")
+    parser_massage = subp.add_parser("massage", help="Massaging phase")
+
+    parser_setup.add_argument(
+        "--debug-commands",
+        action="store_true",
+        help="Turn on debug logging of commands generated to stderr",
+    )
+
+    parser_setup.add_argument(
+        "--extension",
+        "-x",
+        metavar="EXTENSION",
+        dest="extensions",
+        nargs="?",
+        default=[],
+        action="append",
+        help="Markdown extension to add to the list of extensions use",
+    )
+
+    parser_massage.add_argument(
+        "--debug-commands",
+        action="store_true",
+        help="Turn on debug logging of commands generated to stderr",
+    )
+
+    parser_massage.add_argument(
+        "--debug-walk",
+        action="store_true",
+        help="Turn on debugging to stderr of the MIME tree walk",
+    )
+
+    parser_massage.add_argument(
+        "--extensions",
+        metavar="EXTENSIONS",
+        type=str,
+        default="",
+        help="Markdown extension to use (comma-separated list)",
+    )
+
+    parser_massage.add_argument(
+        "--write-commands-to",
+        metavar="PATH",
+        dest="cmdpath",
+        help="Temporary file path to write commands to",
+    )
+
+    parser_massage.add_argument(
+        "MAILDRAFT",
+        nargs="?",
+        help="If provided, the script is invoked as editor on the mail draft",
+    )
+
+    return parser.parse_args(*args, **kwargs)
+
+
+# [ PARTS GENERATION ] ########################################################
+
+
+class Part(
+    namedtuple(
+        "Part",
+        ["type", "subtype", "path", "desc", "cid", "orig"],
+        defaults=[None, None, False],
+    )
+):
+    def __str__(self):
+        ret = f"<{self.type}/{self.subtype}>"
+        if self.cid:
+            ret = f"{ret} cid:{self.cid}"
+        if self.orig:
+            ret = f"{ret} ORIGINAL"
+        return ret
+
+
+class Multipart(
+    namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
+):
+    def __str__(self):
+        return f"<multipart/{self.subtype}> children={len(self.children)}"
+
+
+def convert_markdown_to_html(maildraft, *, extensions=None):
+    draftpath = pathlib.Path(maildraft)
+    textpart = Part(
+        "text", "plain", draftpath, "Plain-text version", orig=True
+    )
+
+    with open(draftpath, "r", encoding="utf-8") as textmarkdown:
+        text = textmarkdown.read()
+
+    mdwn = markdown.Markdown(extensions=extensions)
+    html = mdwn.convert(text)
+
+    htmlpath = draftpath.with_suffix(".html")
+    htmlpart = Part("text", "html", htmlpath, "HTML version")
+
+    with open(
+        htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
+    ) as texthtml:
+        texthtml.write(html)
+
+    logopart = Part(
+        "image",
+        "png",
+        "/usr/share/doc/neomutt/logo/neomutt-256.png",
+        "Logo",
+        "neomutt-256.png",
+    )
+
+    return Multipart(
+        "relative",
+        [
+            Multipart(
+                "alternative",
+                [textpart, htmlpart],
+                "Group of alternative content",
+            ),
+            logopart,
+        ],
+        "Group of related content",
+    )
+
+
+class MIMETreeDFWalker:
+    def __init__(self, *, visitor_fn=None, debug=False):
+        self._visitor_fn = visitor_fn
+        self._debug = debug
+
+    def walk(self, root, *, visitor_fn=None):
+        """
+        Recursive function to implement a depth-dirst walk of the MIME-tree
+        rooted at `root`.
+        """
+
+        if isinstance(root, list):
+            root = Multipart("mixed", children=root)
+
+        self._walk(
+            root,
+            stack=[],
+            visitor_fn=visitor_fn or self._visitor_fn,
+        )
+
+    def _walk(self, node, *, stack, visitor_fn):
+        # Let's start by enumerating the parts at the current level. At the
+        # root level, stack will be the empty list, and we expect a multipart/*
+        # container at this level. Later, e.g. within a mutlipart/alternative
+        # container, the subtree will just be the alternative parts, while the
+        # top of the stack will be the multipart/alternative container, which
+        # we will process after the following loop.
+
+        lead = f"{'| '*len(stack)}|-"
+        if isinstance(node, Multipart):
+            self.debugprint(
+                f"{lead}{node} parents={[s.subtype for s in stack]}"
+            )
+
+            # Depth-first, so push the current container onto the stack,
+            # then descend …
+            stack.append(node)
+            self.debugprint("| " * (len(stack) + 1))
+            for child in node.children:
+                self._walk(
+                    child,
+                    stack=stack,
+                    visitor_fn=visitor_fn,
+                )
+            self.debugprint("| " * len(stack))
+            assert stack.pop() == node
+
+        else:
+            self.debugprint(f"{lead}{node}")
+
+        if visitor_fn:
+            visitor_fn(node, stack, debugprint=self.debugprint)
+
+    def debugprint(self, s, **kwargs):
+        if self._debug:
+            print(s, file=sys.stderr, **kwargs)
+
+
+# [ RUN MODES ] ###############################################################
+
+
+class MuttCommands:
+    """
+    Stupid class to interface writing out Mutt commands. This is quite a hack
+    to deal with the fact that Mutt runs "push" commands in reverse order, so
+    all of a sudden, things become very complicated when mixing with "real"
+    commands.
+
+    Hence we keep two sets of commands, and one set of pushes. Commands are
+    added to the first until a push is added, after which commands are added to
+    the second set of commands.
+
+    On flush(), the first set is printed, followed by the pushes in reverse,
+    and then the second set is printed. All 3 sets are then cleared.
+    """
+
+    def __init__(self, out_f=sys.stdout, *, debug=False):
+        self._cmd1, self._push, self._cmd2 = [], [], []
+        self._out_f = out_f
+        self._debug = debug
+
+    def cmd(self, s):
+        self.debugprint(s)
+        if self._push:
+            self._cmd2.append(s)
+        else:
+            self._cmd1.append(s)
+
+    def push(self, s):
+        s = s.replace('"', '"')
+        s = f'push "{s}"'
+        self.debugprint(s)
+        self._push.insert(0, s)
+
+    def flush(self):
+        print(
+            "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
+        )
+        self._cmd1, self._push, self._cmd2 = [], [], []
+
+    def debugprint(self, s, **kwargs):
+        if self._debug:
+            print(s, file=sys.stderr, **kwargs)
+
+
+def do_setup(
+    extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
+):
+    extensions = extensions or []
+    temppath = temppath or pathlib.Path(
+        tempfile.mkstemp(prefix="muttmdwn-")[1]
+    )
+    cmds = MuttCommands(out_f, debug=debug_commands)
+
+    editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
+    if extensions:
+        editor = f'{editor} --extensions {",".join(extensions)}'
+
+    cmds.cmd('set my_editor="$editor"')
+    cmds.cmd('set my_edit_headers="$edit_headers"')
+    cmds.cmd(f'set editor="{editor}"')
+    cmds.cmd("unset edit_headers")
+    cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
+    cmds.push("<first-entry><edit-file>")
+    cmds.flush()
+
+
+def do_massage(
+    maildraft,
+    cmdpath,
+    *,
+    extensions=None,
+    converter=convert_markdown_to_html,
+    debug_commands=False,
+    debug_walk=False,
+):
+    # Here's the big picture: we're being invoked as the editor on the email
+    # draft, and whatever commands we write to the file given as cmdpath will
+    # be run by the second source command in the macro definition.
+
+    with open(cmdpath, "w") as cmd_f:
+        # Let's start by cleaning up what the setup did (see above), i.e. we
+        # restore the $editor and $edit_headers variables, and also unset the
+        # variable used to identify the command file we're currently writing
+        # to.
+        cmds = MuttCommands(cmd_f, debug=debug_commands)
+        cmds.cmd('set editor="$my_editor"')
+        cmds.cmd('set edit_headers="$my_edit_headers"')
+        cmds.cmd("unset my_editor")
+        cmds.cmd("unset my_edit_headers")
+
+        # let's flush those commands, as there'll be a lot of pushes from now
+        # on, which need to be run in reverse order
+        cmds.flush()
+
+        extensions = extensions.split(",") if extensions else []
+        tree = converter(maildraft, extensions=extensions)
+
+        mimetree = MIMETreeDFWalker(debug=args.debug_walk)
+
+        def visitor_fn(item, stack, *, debugprint=None):
+            """
+            Visitor function called for every node (part) of the MIME tree,
+            depth-first, and responsible for telling NeoMutt how to assemble
+            the tree.
+            """
+            if isinstance(item, Part):
+                # We've hit a leaf-node, i.e. an alternative or a related part
+                # with actual content.
+
+                # If the part is not an original part, i.e. doesn't already
+                # exist, we must first add it.
+                if not item.orig:
+                    cmds.push(f"<attach-file>{item.path}<enter>")
+                    cmds.push("<toggle-unlink><toggle-disposition>")
+                    if item.cid:
+                        cmds.push(
+                            f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>"
+                        )
+
+                # If the item (including the original) comes with a
+                # description, then we might just as well update the NeoMutt
+                # tree now:
+                if item.desc:
+                    cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
+
+                # Finally, tag the entry that we just processed, so that when
+                # we're done at this level, as we walk up the stack, the items
+                # to be grouped will already be tagged and ready.
+                cmds.push("<tag-entry>")
+
+            elif isinstance(item, Multipart):
+                # This node has children, but we already visited them (see
+                # above), and so they have been tagged in NeoMutt's compose
+                # window. Now it's just a matter of telling NeoMutt to do the
+                # appropriate grouping:
+                if item.subtype == "alternative":
+                    cmds.push("<group-alternatives>")
+                elif item.subtype == "relative":
+                    cmds.push("<group-related>")
+                elif item.subtype == "multilingual":
+                    cmds.push("<group-multilingual>")
+
+                # Again, if there is a description, we might just as well:
+                if item.desc:
+                    cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
+
+                # Finally, if we're at non-root level, tag the new container,
+                # as it might itself be part of a container, to be processed
+                # one level up:
+                if stack:
+                    cmds.push("<tag-entry>")
+
+            else:
+                # We should never get here
+                assert not "is valid part"
+
+        # -----------------
+        # End of visitor_fn
+
+        # Let's walk the tree and visit every node with our fancy visitor
+        # function
+        mimetree.walk(tree, visitor_fn=visitor_fn)
+
+        # Finally, cleanup. Since we're responsible for removing the temporary
+        # file, how's this for a little hack?
+        cmds.cmd(f"source 'rm -f {args.cmdpath}|'")
+        cmds.cmd("unset my_mdwn_postprocess_cmd_file")
+        cmds.flush()
+
+
+# [ CLI ENTRY ] ###############################################################
+
+if __name__ == "__main__":
+    args = parse_cli_args()
+
+    if args.mode == "setup":
+        do_setup(args.extensions, debug_commands=args.debug_commands)
+
+    elif args.mode == "massage":
+        do_massage(
+            args.MAILDRAFT,
+            args.cmdpath,
+            extensions=args.extensions,
+            debug_commands=args.debug_commands,
+            debug_walk=args.debug_walk,
+        )
+
+
+# [ TESTS ] ###################################################################
+
+try:
+    import pytest
+
+    class Tests:
+        @pytest.fixture
+        def const1(self):
+            return "CONSTANT STRING 1"
+
+        @pytest.fixture
+        def const2(self):
+            return "CONSTANT STRING 2"
+
+        # NOTE: tests using the capsys fixture must specify sys.stdout to the
+        # functions they call, else old stdout is used and not captured
+
+        def test_MuttCommands_cmd(self, const1, const2, capsys):
+            "Assert order of commands"
+            cmds = MuttCommands(out_f=sys.stdout)
+            cmds.cmd(const1)
+            cmds.cmd(const2)
+            cmds.flush()
+            captured = capsys.readouterr()
+            assert captured.out == "\n".join((const1, const2, ""))
+
+        def test_MuttCommands_push(self, const1, const2, capsys):
+            "Assert reverse order of pushes"
+            cmds = MuttCommands(out_f=sys.stdout)
+            cmds.push(const1)
+            cmds.push(const2)
+            cmds.flush()
+            captured = capsys.readouterr()
+            assert (
+                captured.out
+                == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
+            )
+
+        def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
+            "Assert reverse order of pushes"
+            cmds = MuttCommands(out_f=sys.stdout)
+            lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
+            for i in range(2):
+                cmds.cmd(lines[4 * i + 0])
+                cmds.cmd(lines[4 * i + 1])
+                cmds.push(lines[4 * i + 2])
+                cmds.push(lines[4 * i + 3])
+            cmds.flush()
+
+            captured = capsys.readouterr()
+            lines_out = captured.out.splitlines()
+            assert lines[0] in lines_out[0]
+            assert lines[1] in lines_out[1]
+            assert lines[7] in lines_out[2]
+            assert lines[6] in lines_out[3]
+            assert lines[3] in lines_out[4]
+            assert lines[2] in lines_out[5]
+            assert lines[4] in lines_out[6]
+            assert lines[5] in lines_out[7]
+
+        @pytest.fixture
+        def basic_mime_tree(self):
+            return Multipart(
+                "related",
+                children=[
+                    Multipart(
+                        "alternative",
+                        children=[
+                            Part("text", "plain", "part.txt", desc="Plain"),
+                            Part("text", "html", "part.html", desc="HTML"),
+                        ],
+                        desc="Alternative",
+                    ),
+                    Part(
+                        "text", "png", "logo.png", cid="logo.png", desc="Logo"
+                    ),
+                ],
+                desc="Related",
+            )
+
+        def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
+            mimetree = MIMETreeDFWalker()
+
+            items = []
+
+            def visitor_fn(item, stack, debugprint):
+                items.append((item, len(stack)))
+
+            mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
+            assert len(items) == 5
+            assert items[0][0].subtype == "plain"
+            assert items[0][1] == 2
+            assert items[1][0].subtype == "html"
+            assert items[1][1] == 2
+            assert items[2][0].subtype == "alternative"
+            assert items[2][1] == 1
+            assert items[3][0].subtype == "png"
+            assert items[3][1] == 1
+            assert items[4][0].subtype == "related"
+            assert items[4][1] == 0
+
+        def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
+            mimetree = MIMETreeDFWalker()
+
+            items = []
+
+            def visitor_fn(item, stack, debugprint):
+                items.append(item)
+
+            mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
+            assert items[-1].subtype == "mixed"
+
+        def test_MIMETreeDFWalker_visitor_in_constructor(
+            self, basic_mime_tree
+        ):
+            items = []
+
+            def visitor_fn(item, stack, debugprint):
+                items.append(item)
+
+            mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
+            mimetree.walk(basic_mime_tree)
+            assert len(items) == 5
+
+        def test_do_setup_no_extensions(self, const1, capsys):
+            "Assert basics about the setup command output"
+            do_setup(temppath=const1, out_f=sys.stdout)
+            captout = capsys.readouterr()
+            lines = captout.out.splitlines()
+            assert lines[2].endswith(f'{const1}"')
+            assert lines[4].endswith(const1)
+            assert "first-entry" in lines[-1]
+            assert "edit-file" in lines[-1]
+
+        def test_do_setup_extensions(self, const1, const2, capsys):
+            "Assert that extensions are passed to editor"
+            do_setup(
+                temppath=const1, extensions=[const2, const1], out_f=sys.stdout
+            )
+            captout = capsys.readouterr()
+            lines = captout.out.splitlines()
+            # assert comma-separated list of extensions passed
+            assert lines[2].endswith(f'{const2},{const1}"')
+            assert lines[4].endswith(const1)
+
+except ImportError:
+    pass
diff --git a/.config/neomutt/neomuttrc b/.config/neomutt/neomuttrc
index 3c02cde..5b29048 100644
--- a/.config/neomutt/neomuttrc
+++ b/.config/neomutt/neomuttrc
@@ -61,3 +61,5 @@ source "test -f $alias_file && cat $alias_file 2>/dev/null || echo unset alias_f
 #set index_format="%4C %?GU?%GU& ?%?GR?%GR& ?%?GI?%GI& ? %-10@date@ %-15.15F %4c%?M?/[%M]? %?H?[%H] ?%s"
 #
 #set pager_format="<%a> %*  %J (%P)"
+
+macro compose B "<enter-command> source '$my_confdir/buildmimetree.py setup|'<enter><enter-command> source \$my_mdwn_postprocess_cmd_file<enter>" "Convert message into a modern MIME tree with inline images"