buildhtmltree.py: test massage

[etc/neomutt.git] / .config / neomutt / buildmimetree.py

#!/usr/bin/python3
#
# NeoMutt helper script to create multipart/* emails with Markdown → HTML
# alternative conversion, and handling of inline images, using NeoMutt's
# ability to manually craft MIME trees, but automating this process.
#
# Configuration:
#   neomuttrc (needs to be a single line):
#     macro compose B "\
#       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
#       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
#     " "Convert message into a modern MIME tree with inline images"
#
#     (Yes, we need to call source twice, as mutt only starts to process output
#     from a source command when the command exits, and since we need to react
#     to the output, we need to be invoked again, using a $my_ variable to pass
#     information)
#
# Requirements:
#   - python3
#   - python3-markdown
# Optional:
#   - pytest
#   - Pynliner
#   - Pygments, if installed, then syntax highlighting is enabled
#
# Latest version:
#   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
#
# Copyright © 2023 martin f. krafft <madduck@madduck.net>
# Released under the GPL-2+ licence, just like Mutt itself.
#

import sys
import pathlib
import markdown
import tempfile
import argparse
from collections import namedtuple


def parse_cli_args(*args, **kwargs):
    parser = argparse.ArgumentParser(
        description=(
            "NeoMutt helper to turn text/markdown email parts "
            "into full-fledged MIME trees"
        )
    )
    parser.epilog = (
        "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
        "Released under the MIT licence"
    )

    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
    parser_setup = subp.add_parser("setup", help="Setup phase")
    parser_massage = subp.add_parser("massage", help="Massaging phase")

    parser_setup.add_argument(
        "--debug-commands",
        action="store_true",
        help="Turn on debug logging of commands generated to stderr",
    )

    parser_setup.add_argument(
        "--extension",
        "-x",
        metavar="EXTENSION",
        dest="extensions",
        nargs="?",
        default=[],
        action="append",
        help="Markdown extension to add to the list of extensions use",
    )

    parser_massage.add_argument(
        "--debug-commands",
        action="store_true",
        help="Turn on debug logging of commands generated to stderr",
    )

    parser_massage.add_argument(
        "--debug-walk",
        action="store_true",
        help="Turn on debugging to stderr of the MIME tree walk",
    )

    parser_massage.add_argument(
        "--extensions",
        metavar="EXTENSIONS",
        type=str,
        default="",
        help="Markdown extension to use (comma-separated list)",
    )

    parser_massage.add_argument(
        "--write-commands-to",
        metavar="PATH",
        dest="cmdpath",
        help="Temporary file path to write commands to",
    )

    parser_massage.add_argument(
        "MAILDRAFT",
        nargs="?",
        help="If provided, the script is invoked as editor on the mail draft",
    )

    return parser.parse_args(*args, **kwargs)


# [ PARTS GENERATION ] ########################################################


class Part(
    namedtuple(
        "Part",
        ["type", "subtype", "path", "desc", "cid", "orig"],
        defaults=[None, None, False],
    )
):
    def __str__(self):
        ret = f"<{self.type}/{self.subtype}>"
        if self.cid:
            ret = f"{ret} cid:{self.cid}"
        if self.orig:
            ret = f"{ret} ORIGINAL"
        return ret


class Multipart(
    namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
):
    def __str__(self):
        return f"<multipart/{self.subtype}> children={len(self.children)}"


def convert_markdown_to_html(maildraft, *, extensions=None):
    draftpath = pathlib.Path(maildraft)
    textpart = Part(
        "text", "plain", draftpath, "Plain-text version", orig=True
    )

    with open(draftpath, "r", encoding="utf-8") as textmarkdown:
        text = textmarkdown.read()

    mdwn = markdown.Markdown(extensions=extensions)
    html = mdwn.convert(text)

    htmlpath = draftpath.with_suffix(".html")
    htmlpart = Part("text", "html", htmlpath, "HTML version")

    with open(
        htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
    ) as texthtml:
        texthtml.write(html)

    logopart = Part(
        "image",
        "png",
        "/usr/share/doc/neomutt/logo/neomutt-256.png",
        "Logo",
        "neomutt-256.png",
    )

    return Multipart(
        "relative",
        [
            Multipart(
                "alternative",
                [textpart, htmlpart],
                "Group of alternative content",
            ),
            logopart,
        ],
        "Group of related content",
    )


class MIMETreeDFWalker:
    def __init__(self, *, visitor_fn=None, debug=False):
        self._visitor_fn = visitor_fn
        self._debug = debug

    def walk(self, root, *, visitor_fn=None):
        """
        Recursive function to implement a depth-dirst walk of the MIME-tree
        rooted at `root`.
        """

        if isinstance(root, list):
            root = Multipart("mixed", children=root)

        self._walk(
            root,
            stack=[],
            visitor_fn=visitor_fn or self._visitor_fn,
        )

    def _walk(self, node, *, stack, visitor_fn):
        # Let's start by enumerating the parts at the current level. At the
        # root level, stack will be the empty list, and we expect a multipart/*
        # container at this level. Later, e.g. within a mutlipart/alternative
        # container, the subtree will just be the alternative parts, while the
        # top of the stack will be the multipart/alternative container, which
        # we will process after the following loop.

        lead = f"{'| '*len(stack)}|-"
        if isinstance(node, Multipart):
            self.debugprint(
                f"{lead}{node} parents={[s.subtype for s in stack]}"
            )

            # Depth-first, so push the current container onto the stack,
            # then descend …
            stack.append(node)
            self.debugprint("| " * (len(stack) + 1))
            for child in node.children:
                self._walk(
                    child,
                    stack=stack,
                    visitor_fn=visitor_fn,
                )
            self.debugprint("| " * len(stack))
            assert stack.pop() == node

        else:
            self.debugprint(f"{lead}{node}")

        if visitor_fn:
            visitor_fn(node, stack, debugprint=self.debugprint)

    def debugprint(self, s, **kwargs):
        if self._debug:
            print(s, file=sys.stderr, **kwargs)


# [ RUN MODES ] ###############################################################


class MuttCommands:
    """
    Stupid class to interface writing out Mutt commands. This is quite a hack
    to deal with the fact that Mutt runs "push" commands in reverse order, so
    all of a sudden, things become very complicated when mixing with "real"
    commands.

    Hence we keep two sets of commands, and one set of pushes. Commands are
    added to the first until a push is added, after which commands are added to
    the second set of commands.

    On flush(), the first set is printed, followed by the pushes in reverse,
    and then the second set is printed. All 3 sets are then cleared.
    """

    def __init__(self, out_f=sys.stdout, *, debug=False):
        self._cmd1, self._push, self._cmd2 = [], [], []
        self._out_f = out_f
        self._debug = debug

    def cmd(self, s):
        self.debugprint(s)
        if self._push:
            self._cmd2.append(s)
        else:
            self._cmd1.append(s)

    def push(self, s):
        s = s.replace('"', '"')
        s = f'push "{s}"'
        self.debugprint(s)
        self._push.insert(0, s)

    def flush(self):
        print(
            "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
        )
        self._cmd1, self._push, self._cmd2 = [], [], []

    def debugprint(self, s, **kwargs):
        if self._debug:
            print(s, file=sys.stderr, **kwargs)


def do_setup(
    extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
):
    extensions = extensions or []
    temppath = temppath or pathlib.Path(
        tempfile.mkstemp(prefix="muttmdwn-")[1]
    )
    cmds = MuttCommands(out_f, debug=debug_commands)

    editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
    if extensions:
        editor = f'{editor} --extensions {",".join(extensions)}'

    cmds.cmd('set my_editor="$editor"')
    cmds.cmd('set my_edit_headers="$edit_headers"')
    cmds.cmd(f'set editor="{editor}"')
    cmds.cmd("unset edit_headers")
    cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
    cmds.push("<first-entry><edit-file>")
    cmds.flush()


def do_massage(
    maildraft,
    cmd_f,
    *,
    extensions=None,
    converter=convert_markdown_to_html,
    debug_commands=False,
    debug_walk=False,
):
    # Here's the big picture: we're being invoked as the editor on the email
    # draft, and whatever commands we write to the file given as cmdpath will
    # be run by the second source command in the macro definition.

    # Let's start by cleaning up what the setup did (see above), i.e. we
    # restore the $editor and $edit_headers variables, and also unset the
    # variable used to identify the command file we're currently writing
    # to.
    cmds = MuttCommands(cmd_f, debug=debug_commands)
    cmds.cmd('set editor="$my_editor"')
    cmds.cmd('set edit_headers="$my_edit_headers"')
    cmds.cmd("unset my_editor")
    cmds.cmd("unset my_edit_headers")

    # let's flush those commands, as there'll be a lot of pushes from now
    # on, which need to be run in reverse order
    cmds.flush()

    extensions = extensions.split(",") if extensions else []
    tree = converter(maildraft, extensions=extensions)

    mimetree = MIMETreeDFWalker(debug=debug_walk)

    def visitor_fn(item, stack, *, debugprint=None):
        """
        Visitor function called for every node (part) of the MIME tree,
        depth-first, and responsible for telling NeoMutt how to assemble
        the tree.
        """
        if isinstance(item, Part):
            # We've hit a leaf-node, i.e. an alternative or a related part
            # with actual content.

            # If the part is not an original part, i.e. doesn't already
            # exist, we must first add it.
            if not item.orig:
                cmds.push(f"<attach-file>{item.path}<enter>")
                cmds.push("<toggle-unlink><toggle-disposition>")
                if item.cid:
                    cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")

            # If the item (including the original) comes with a
            # description, then we might just as well update the NeoMutt
            # tree now:
            if item.desc:
                cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")

        elif isinstance(item, Multipart):
            # This node has children, but we already visited them (see
            # above), and so they have been tagged in NeoMutt's compose
            # window. Now it's just a matter of telling NeoMutt to do the
            # appropriate grouping:
            if item.subtype == "alternative":
                cmds.push("<group-alternatives>")
            elif item.subtype == "relative":
                cmds.push("<group-related>")
            elif item.subtype == "multilingual":
                cmds.push("<group-multilingual>")

            # Again, if there is a description, we might just as well:
            if item.desc:
                cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")

        else:
            # We should never get here
            assert not "is valid part"

        # Finally, if we're at non-root level, tag the new container,
        # as it might itself be part of a container, to be processed
        # one level up:
        if stack:
            cmds.push("<tag-entry>")

    # -----------------
    # End of visitor_fn

    # Let's walk the tree and visit every node with our fancy visitor
    # function
    mimetree.walk(tree, visitor_fn=visitor_fn)

    # Finally, cleanup. Since we're responsible for removing the temporary
    # file, how's this for a little hack?
    try:
        filename = cmd_f.name
    except AttributeError:
        filename = "pytest_internal_file"
    cmds.cmd(f"source 'rm -f {filename}|'")
    cmds.cmd("unset my_mdwn_postprocess_cmd_file")
    cmds.flush()


# [ CLI ENTRY ] ###############################################################

if __name__ == "__main__":
    args = parse_cli_args()

    if args.mode == "setup":
        do_setup(args.extensions, debug_commands=args.debug_commands)

    elif args.mode == "massage":
        with open(args.cmdpath, "w") as cmd_f:
            do_massage(
                args.MAILDRAFT,
                cmd_f,
                extensions=args.extensions,
                debug_commands=args.debug_commands,
                debug_walk=args.debug_walk,
            )


# [ TESTS ] ###################################################################

try:
    import pytest

    class Tests:
        @pytest.fixture
        def const1(self):
            return "CONSTANT STRING 1"

        @pytest.fixture
        def const2(self):
            return "CONSTANT STRING 2"

        # NOTE: tests using the capsys fixture must specify sys.stdout to the
        # functions they call, else old stdout is used and not captured

        def test_MuttCommands_cmd(self, const1, const2, capsys):
            "Assert order of commands"
            cmds = MuttCommands(out_f=sys.stdout)
            cmds.cmd(const1)
            cmds.cmd(const2)
            cmds.flush()
            captured = capsys.readouterr()
            assert captured.out == "\n".join((const1, const2, ""))

        def test_MuttCommands_push(self, const1, const2, capsys):
            "Assert reverse order of pushes"
            cmds = MuttCommands(out_f=sys.stdout)
            cmds.push(const1)
            cmds.push(const2)
            cmds.flush()
            captured = capsys.readouterr()
            assert (
                captured.out
                == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
            )

        def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
            "Assert reverse order of pushes"
            cmds = MuttCommands(out_f=sys.stdout)
            lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
            for i in range(2):
                cmds.cmd(lines[4 * i + 0])
                cmds.cmd(lines[4 * i + 1])
                cmds.push(lines[4 * i + 2])
                cmds.push(lines[4 * i + 3])
            cmds.flush()

            captured = capsys.readouterr()
            lines_out = captured.out.splitlines()
            assert lines[0] in lines_out[0]
            assert lines[1] in lines_out[1]
            assert lines[7] in lines_out[2]
            assert lines[6] in lines_out[3]
            assert lines[3] in lines_out[4]
            assert lines[2] in lines_out[5]
            assert lines[4] in lines_out[6]
            assert lines[5] in lines_out[7]

        @pytest.fixture
        def basic_mime_tree(self):
            return Multipart(
                "relative",
                children=[
                    Multipart(
                        "alternative",
                        children=[
                            Part(
                                "text",
                                "plain",
                                "part.txt",
                                desc="Plain",
                                orig=True,
                            ),
                            Part("text", "html", "part.html", desc="HTML"),
                        ],
                        desc="Alternative",
                    ),
                    Part(
                        "text", "png", "logo.png", cid="logo.png", desc="Logo"
                    ),
                ],
                desc="Related",
            )

        def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
            mimetree = MIMETreeDFWalker()

            items = []

            def visitor_fn(item, stack, debugprint):
                items.append((item, len(stack)))

            mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
            assert len(items) == 5
            assert items[0][0].subtype == "plain"
            assert items[0][1] == 2
            assert items[1][0].subtype == "html"
            assert items[1][1] == 2
            assert items[2][0].subtype == "alternative"
            assert items[2][1] == 1
            assert items[3][0].subtype == "png"
            assert items[3][1] == 1
            assert items[4][0].subtype == "relative"
            assert items[4][1] == 0

        def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
            mimetree = MIMETreeDFWalker()

            items = []

            def visitor_fn(item, stack, debugprint):
                items.append(item)

            mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
            assert items[-1].subtype == "mixed"

        def test_MIMETreeDFWalker_visitor_in_constructor(
            self, basic_mime_tree
        ):
            items = []

            def visitor_fn(item, stack, debugprint):
                items.append(item)

            mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
            mimetree.walk(basic_mime_tree)
            assert len(items) == 5

        def test_do_setup_no_extensions(self, const1, capsys):
            "Assert basics about the setup command output"
            do_setup(temppath=const1, out_f=sys.stdout)
            captout = capsys.readouterr()
            lines = captout.out.splitlines()
            assert lines[2].endswith(f'{const1}"')
            assert lines[4].endswith(const1)
            assert "first-entry" in lines[-1]
            assert "edit-file" in lines[-1]

        def test_do_setup_extensions(self, const1, const2, capsys):
            "Assert that extensions are passed to editor"
            do_setup(
                temppath=const1, extensions=[const2, const1], out_f=sys.stdout
            )
            captout = capsys.readouterr()
            lines = captout.out.splitlines()
            # assert comma-separated list of extensions passed
            assert lines[2].endswith(f'{const2},{const1}"')
            assert lines[4].endswith(const1)

        def test_do_massage_basic(self, const1, capsys):
            def converter(maildraft, extensions):
                return Part("text", "plain", "/dev/null", orig=True)

            do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
            captured = capsys.readouterr()
            assert (
                captured.out.strip()
                == """\
            set editor="$my_editor"
            set edit_headers="$my_edit_headers"
            unset my_editor
            unset my_edit_headers
            source 'rm -f pytest_internal_file|'
            unset my_mdwn_postprocess_cmd_file
            """.replace(
                    "            ", ""
                ).strip()
            )

        def test_do_massage_fulltree(self, const1, basic_mime_tree, capsys):
            def converter(maildraft, extensions):
                return basic_mime_tree

            do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
            captured = capsys.readouterr()
            lines = captured.out.splitlines()[4:][::-1]
            assert "Related" in lines.pop()
            assert "group-related" in lines.pop()
            assert "tag-entry" in lines.pop()
            assert "Logo" in lines.pop()
            assert "content-id" in lines.pop()
            assert "toggle-unlink" in lines.pop()
            assert "logo.png" in lines.pop()
            assert "tag-entry" in lines.pop()
            assert "Alternative" in lines.pop()
            assert "group-alternatives" in lines.pop()
            assert "tag-entry" in lines.pop()
            assert "HTML" in lines.pop()
            assert "toggle-unlink" in lines.pop()
            assert "part.html" in lines.pop()
            assert "tag-entry" in lines.pop()
            assert "Plain" in lines.pop()

except ImportError:
    pass