X-Git-Url: https://git.madduck.net/etc/neomutt.git/blobdiff_plain/ba392f61872a76df38acaf633843cf00c9f6e139..aec6bc12a30a4e2f100b5bb1e72c6e53e2076f23:/.config/neomutt/buildmimetree.py?ds=sidebyside

diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py
index bad8bbe..fbc8040 100755
--- a/.config/neomutt/buildmimetree.py
+++ b/.config/neomutt/buildmimetree.py
@@ -6,8 +6,11 @@
 #
 # Configuration:
 #   neomuttrc (needs to be a single line):
+#     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
 #     macro compose B "\
-#       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
+#       <enter-command> source '$my_confdir/buildmimetree.py \
+#       --tempdir $tempdir --extensions $my_mdwn_extensions \
+#       --css-file $my_confdir/htmlmail.css |'<enter>\
 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
 #     " "Convert message into a modern MIME tree with inline images"
 #
@@ -19,10 +22,11 @@
 # Requirements:
 #   - python3
 #   - python3-markdown
+#   - python3-beautifulsoup4
 # Optional:
 #   - pytest
-#   - Pynliner
-#   - Pygments, if installed, then syntax highlighting is enabled
+#   - Pynliner, provides --css-file and thus inline styling of HTML output
+#   - Pygments, then syntax highlighting for fenced code is enabled
 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
@@ -32,11 +36,20 @@
 #
 
 import sys
+import os
+import os.path
 import pathlib
 import markdown
 import tempfile
 import argparse
-from collections import namedtuple
+import re
+import mimetypes
+import bs4
+from collections import namedtuple, OrderedDict
+from markdown.extensions import Extension
+from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
+from email.utils import make_msgid
+from urllib import request
 
 
 def parse_cli_args(*args, **kwargs):
@@ -47,67 +60,241 @@ def parse_cli_args(*args, **kwargs):
         )
     )
     parser.epilog = (
-        "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
+        "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
         "Released under the MIT licence"
     )
 
-    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
-    parser_setup = subp.add_parser("setup", help="Setup phase")
-    parser_massage = subp.add_parser("massage", help="Massaging phase")
+    parser.add_argument(
+        "--extensions",
+        metavar="EXT[,EXT[,EXT]]",
+        type=str,
+        default="",
+        help="Markdown extension to use (comma-separated list)",
+    )
 
-    parser_setup.add_argument(
-        "--debug-commands",
+    if _PYNLINER:
+        parser.add_argument(
+            "--css-file",
+            metavar="FILE",
+            type=pathlib.Path,
+            default=os.devnull,
+            help="CSS file to merge with the final HTML",
+        )
+    else:
+        parser.set_defaults(css_file=None)
+
+    parser.add_argument(
+        "--related-to-html-only",
         action="store_true",
-        help="Turn on debug logging of commands generated to stderr",
+        help="Make related content be sibling to HTML parts only",
     )
 
-    parser_setup.add_argument(
-        "--extension",
-        "-x",
-        metavar="EXTENSION",
-        dest="extensions",
-        nargs="?",
-        default=[],
-        action="append",
-        help="Markdown extension to add to the list of extensions use",
+    def positive_integer(value):
+        try:
+            if int(value) > 0:
+                return int(value)
+
+        except ValueError:
+            pass
+
+        raise ValueError("Must be a positive integer")
+
+    parser.add_argument(
+        "--max-number-other-attachments",
+        metavar="INTEGER",
+        type=positive_integer,
+        default=20,
+        help="Maximum number of other attachments to expect",
+    )
+
+    parser.add_argument(
+        "--only-build",
+        "--just-build",
+        action="store_true",
+        help="Only build, don't send the message",
+    )
+
+    parser.add_argument(
+        "--tempdir",
+        metavar="DIR",
+        type=pathlib.Path,
+        help="Specify temporary directory to use for attachments",
     )
 
-    parser_massage.add_argument(
+    parser.add_argument(
         "--debug-commands",
         action="store_true",
         help="Turn on debug logging of commands generated to stderr",
     )
 
-    parser_massage.add_argument(
+    parser.add_argument(
         "--debug-walk",
         action="store_true",
         help="Turn on debugging to stderr of the MIME tree walk",
     )
 
-    parser_massage.add_argument(
-        "--extensions",
-        metavar="EXTENSIONS",
-        type=str,
-        default="",
-        help="Markdown extension to use (comma-separated list)",
+    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
+    massage_p = subp.add_parser(
+        "massage", help="Massaging phase (internal use)"
     )
 
-    parser_massage.add_argument(
+    massage_p.add_argument(
         "--write-commands-to",
-        metavar="PATH",
+        "-o",
+        metavar="FILE",
         dest="cmdpath",
+        type=pathlib.Path,
+        required=True,
         help="Temporary file path to write commands to",
     )
 
-    parser_massage.add_argument(
+    massage_p.add_argument(
         "MAILDRAFT",
         nargs="?",
+        type=pathlib.Path,
         help="If provided, the script is invoked as editor on the mail draft",
     )
 
     return parser.parse_args(*args, **kwargs)
 
 
+# [ IMAGE HANDLING ] ##########################################################
+
+
+InlineImageInfo = namedtuple(
+    "InlineImageInfo", ["cid", "desc"], defaults=[None]
+)
+
+
+class ImageRegistry:
+    def __init__(self):
+        self._images = OrderedDict()
+
+    def register(self, path, description=None):
+        path = os.path.expanduser(path)
+        if path.startswith("/"):
+            path = f"file://{path}"
+        cid = make_msgid()[1:-1]
+        self._images[path] = InlineImageInfo(cid, description)
+        return cid
+
+    def __iter__(self):
+        return self._images.__iter__()
+
+    def __getitem__(self, idx):
+        return self._images.__getitem__(idx)
+
+    def __len__(self):
+        return self._images.__len__()
+
+    def items(self):
+        return self._images.items()
+
+    def __repr__(self):
+        return f"<ImageRegistry(items={len(self._images)})>"
+
+    def __str__(self):
+        return self._images.__str__()
+
+
+class InlineImageExtension(Extension):
+    class RelatedImageInlineProcessor(ImageInlineProcessor):
+        def __init__(self, re, md, registry):
+            super().__init__(re, md)
+            self._registry = registry
+
+        def handleMatch(self, m, data):
+            el, start, end = super().handleMatch(m, data)
+            if "src" in el.attrib:
+                src = el.attrib["src"]
+                if "://" not in src or src.startswith("file://"):
+                    # We only inline local content
+                    cid = self._registry.register(
+                        el.attrib["src"],
+                        el.attrib.get("title", el.attrib.get("alt")),
+                    )
+                    el.attrib["src"] = f"cid:{cid}"
+            return el, start, end
+
+    def __init__(self, registry):
+        super().__init__()
+        self._image_registry = registry
+
+    INLINE_PATTERN_NAME = "image_link"
+
+    def extendMarkdown(self, md):
+        md.registerExtension(self)
+        inline_image_proc = self.RelatedImageInlineProcessor(
+            IMAGE_LINK_RE, md, self._image_registry
+        )
+        md.inlinePatterns.register(
+            inline_image_proc, InlineImageExtension.INLINE_PATTERN_NAME, 150
+        )
+
+
+def markdown_with_inline_image_support(
+    text,
+    *,
+    mdwn=None,
+    image_registry=None,
+    extensions=None,
+    extension_configs=None,
+):
+    registry = (
+        image_registry if image_registry is not None else ImageRegistry()
+    )
+    inline_image_handler = InlineImageExtension(registry=registry)
+    extensions = extensions or []
+    extensions.append(inline_image_handler)
+    mdwn = markdown.Markdown(
+        extensions=extensions, extension_configs=extension_configs
+    )
+
+    htmltext = mdwn.convert(text)
+
+    def replace_image_with_cid(matchobj):
+        for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
+            if m in registry:
+                return f"(cid:{registry[m].cid}"
+        return matchobj.group(0)
+
+    text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
+    return text, htmltext, registry, mdwn
+
+
+# [ CSS STYLING ] #############################################################
+
+
+try:
+    import pynliner
+
+    _PYNLINER = True
+
+except ImportError:
+    _PYNLINER = False
+
+try:
+    from pygments.formatters import get_formatter_by_name
+
+    _CODEHILITE_CLASS = "codehilite"
+
+    _PYGMENTS_CSS = get_formatter_by_name(
+        "html", style="default"
+    ).get_style_defs(f".{_CODEHILITE_CLASS}")
+
+except ImportError:
+    _PYGMENTS_CSS = None
+
+
+def apply_styling(html, css):
+    return (
+        pynliner.Pynliner()
+        .from_string(html)
+        .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
+        .run()
+    )
+
+
 # [ PARTS GENERATION ] ########################################################
 
 
@@ -133,101 +320,292 @@ class Multipart(
     def __str__(self):
         return f"<multipart/{self.subtype}> children={len(self.children)}"
 
+    def __hash__(self):
+        return hash(str(self.subtype) + "".join(str(self.children)))
+
+
+def filereader_fn(path, mode="r", **kwargs):
+    with open(path, mode, **kwargs) as in_f:
+        return in_f.read()
+
+
+def filewriter_fn(path, content, mode="w", **kwargs):
+    with open(path, mode, **kwargs) as out_f:
+        out_f.write(content)
+
+
+def collect_inline_images(
+    image_registry, *, tempdir=None, filewriter_fn=filewriter_fn
+):
+    relparts = []
+    for path, info in image_registry.items():
+        if path.startswith("cid:"):
+            continue
+
+        data = request.urlopen(path)
 
-def convert_markdown_to_html(maildraft, *, extensions=None):
-    draftpath = pathlib.Path(maildraft)
+        mimetype = data.headers["Content-Type"]
+        ext = mimetypes.guess_extension(mimetype)
+        tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
+        path = pathlib.Path(tempfilename[1])
+
+        filewriter_fn(path, data.read(), "w+b")
+
+        desc = (
+            f'Inline image: "{info.desc}"'
+            if info.desc
+            else f"Inline image {str(len(relparts)+1)}"
+        )
+        relparts.append(
+            Part(*mimetype.split("/"), path, cid=info.cid, desc=desc)
+        )
+
+    return relparts
+
+
+EMAIL_SIG_SEP = "\n-- \n"
+HTML_SIG_MARKER = "=htmlsig "
+
+
+def make_html_doc(body, sig=None):
+    ret = (
+        "<!DOCTYPE html>\n"
+        "<html>\n"
+        "<head>\n"
+        '<meta http-equiv="content-type" content="text/html; charset=UTF-8">\n'  # noqa: E501
+        '<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'  # noqa: E501
+        "</head>\n"
+        "<body>\n"
+        f"{body}\n"
+    )
+
+    if sig:
+        nl = "\n"
+        ret = (
+            f'{ret}<div id="signature"><span class="sig_separator">{EMAIL_SIG_SEP.strip(nl)}</span>\n'  # noqa: E501
+            f"{sig}\n"
+            "</div>"
+        )
+
+    return f"{ret}\n  </body>\n</html>"
+
+
+def make_text_mail(text, sig=None):
+    return EMAIL_SIG_SEP.join((text, sig)) if sig else text
+
+
+def extract_signature(text, *, filereader_fn=filereader_fn):
+    parts = text.split(EMAIL_SIG_SEP, 1)
+    if len(parts) == 1:
+        return text, None, None
+
+    lines = parts[1].splitlines()
+    if lines[0].startswith(HTML_SIG_MARKER):
+        path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1])
+        textsig = "\n".join(lines)
+
+        sig_input = filereader_fn(path.expanduser())
+        soup = bs4.BeautifulSoup(sig_input, "html.parser")
+
+        style = str(soup.style.extract()) if soup.style else ""
+        for sig_selector in (
+            "#signature",
+            "#signatur",
+            "#emailsig",
+            ".signature",
+            ".signatur",
+            ".emailsig",
+            "body",
+            "div",
+        ):
+            sig = soup.select_one(sig_selector)
+            if sig:
+                break
+
+        if not sig:
+            return parts[0], textsig, style + sig_input
+
+        if sig.attrs.get("id") == "signature":
+            sig = "".join(str(c) for c in sig.children)
+
+        return parts[0], textsig, style + str(sig)
+
+    return parts[0], parts[1], None
+
+
+def convert_markdown_to_html(
+    origtext,
+    draftpath,
+    *,
+    related_to_html_only=False,
+    css=None,
+    filewriter_fn=filewriter_fn,
+    filereader_fn=filereader_fn,
+    tempdir=None,
+    extensions=None,
+    extension_configs=None,
+):
+    # TODO extension_configs need to be handled differently
+    extension_configs = extension_configs or {}
+    extension_configs.setdefault("pymdownx.highlight", {})
+    extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
+
+    origtext, textsig, htmlsig = extract_signature(
+        origtext, filereader_fn=filereader_fn
+    )
+
+    (
+        origtext,
+        htmltext,
+        image_registry,
+        mdwn,
+    ) = markdown_with_inline_image_support(
+        origtext, extensions=extensions, extension_configs=extension_configs
+    )
+
+    if htmlsig:
+        if not textsig:
+            # TODO: decide what to do if there is no plain-text version
+            raise NotImplementedError("HTML signature but no text alternative")
+
+        soup = bs4.BeautifulSoup(htmlsig, "html.parser")
+        for img in soup.find_all("img"):
+            uri = img.attrs["src"]
+            desc = img.attrs.get("title", img.attrs.get("alt"))
+            cid = image_registry.register(uri, desc)
+            img.attrs["src"] = f"cid:{cid}"
+
+        htmlsig = str(soup)
+
+    elif textsig:
+        (
+            textsig,
+            htmlsig,
+            image_registry,
+            mdwn,
+        ) = markdown_with_inline_image_support(
+            textsig,
+            extensions=extensions,
+            extension_configs=extension_configs,
+            image_registry=image_registry,
+            mdwn=mdwn,
+        )
+
+    origtext = make_text_mail(origtext, textsig)
+
+    filewriter_fn(draftpath, origtext, encoding="utf-8")
     textpart = Part(
         "text", "plain", draftpath, "Plain-text version", orig=True
     )
 
-    with open(draftpath, "r", encoding="utf-8") as textmarkdown:
-        text = textmarkdown.read()
-
-    mdwn = markdown.Markdown(extensions=extensions)
-    html = mdwn.convert(text)
+    htmltext = make_html_doc(htmltext, htmlsig)
+    htmltext = apply_styling(htmltext, css)
 
     htmlpath = draftpath.with_suffix(".html")
+    filewriter_fn(
+        htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
+    )
     htmlpart = Part("text", "html", htmlpath, "HTML version")
 
-    with open(
-        htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
-    ) as texthtml:
-        texthtml.write(html)
-
-    logopart = Part(
-        "image",
-        "png",
-        "/usr/share/doc/neomutt/logo/neomutt-256.png",
-        "Logo",
-        "neomutt-256.png",
+    imgparts = collect_inline_images(
+        image_registry, tempdir=tempdir, filewriter_fn=filewriter_fn
     )
 
-    return Multipart(
-        "relative",
-        [
-            Multipart(
-                "alternative",
-                [textpart, htmlpart],
-                "Group of alternative content",
-            ),
-            logopart,
-        ],
-        "Group of related content",
-    )
+    if related_to_html_only:
+        # If there are inline image part, they will be contained within a
+        # multipart/related part along with the HTML part only
+        if imgparts:
+            # replace htmlpart with a multipart/related container of the HTML
+            # parts and the images
+            htmlpart = Multipart(
+                "relative", [htmlpart] + imgparts, "Group of related content"
+            )
+
+        return Multipart(
+            "alternative", [textpart, htmlpart], "Group of alternative content"
+        )
+
+    else:
+        # If there are inline image part, they will be siblings to the
+        # multipart/alternative tree within a multipart/related part
+        altpart = Multipart(
+            "alternative", [textpart, htmlpart], "Group of alternative content"
+        )
+        if imgparts:
+            return Multipart(
+                "relative", [altpart] + imgparts, "Group of related content"
+            )
+        else:
+            return altpart
 
 
 class MIMETreeDFWalker:
     def __init__(self, *, visitor_fn=None, debug=False):
-        self._visitor_fn = visitor_fn
+        self._visitor_fn = visitor_fn or self._echovisit
         self._debug = debug
 
+    def _echovisit(self, node, ancestry, debugprint):
+        debugprint(f"node={node} ancestry={ancestry}")
+
     def walk(self, root, *, visitor_fn=None):
         """
         Recursive function to implement a depth-dirst walk of the MIME-tree
         rooted at `root`.
         """
-
         if isinstance(root, list):
-            root = Multipart("mixed", children=root)
+            if len(root) > 1:
+                root = Multipart("mixed", children=root)
+            else:
+                root = root[0]
 
         self._walk(
             root,
-            stack=[],
+            ancestry=[],
+            descendents=[],
             visitor_fn=visitor_fn or self._visitor_fn,
         )
 
-    def _walk(self, node, *, stack, visitor_fn):
+    def _walk(self, node, *, ancestry, descendents, visitor_fn):
         # Let's start by enumerating the parts at the current level. At the
-        # root level, stack will be the empty list, and we expect a multipart/*
-        # container at this level. Later, e.g. within a mutlipart/alternative
-        # container, the subtree will just be the alternative parts, while the
-        # top of the stack will be the multipart/alternative container, which
-        # we will process after the following loop.
-
-        lead = f"{'| '*len(stack)}|-"
+        # root level, ancestry will be the empty list, and we expect a
+        # multipart/* container at this level. Later, e.g. within a
+        # mutlipart/alternative container, the subtree will just be the
+        # alternative parts, while the top of the ancestry will be the
+        # multipart/alternative container, which we will process after the
+        # following loop.
+
+        lead = f"{'│ '*len(ancestry)}"
         if isinstance(node, Multipart):
             self.debugprint(
-                f"{lead}{node} parents={[s.subtype for s in stack]}"
+                f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
             )
 
-            # Depth-first, so push the current container onto the stack,
-            # then descend …
-            stack.append(node)
-            self.debugprint("| " * (len(stack) + 1))
+            # Depth-first, so push the current container onto the ancestry
+            # stack, then descend …
+            ancestry.append(node)
+            self.debugprint(lead + "│ " * 2)
             for child in node.children:
                 self._walk(
                     child,
-                    stack=stack,
+                    ancestry=ancestry,
+                    descendents=descendents,
                     visitor_fn=visitor_fn,
                 )
-            self.debugprint("| " * len(stack))
-            assert stack.pop() == node
+            assert ancestry.pop() == node
+            sibling_descendents = descendents
+            descendents.extend(node.children)
 
         else:
-            self.debugprint(f"{lead}{node}")
+            self.debugprint(f"{lead}├{node}")
+            sibling_descendents = descendents
+
+        if False and ancestry:
+            self.debugprint(lead[:-1] + " │")
 
         if visitor_fn:
-            visitor_fn(node, stack, debugprint=self.debugprint)
+            visitor_fn(
+                node, ancestry, sibling_descendents, debugprint=self.debugprint
+            )
 
     def debugprint(self, s, **kwargs):
         if self._debug:
@@ -265,7 +643,7 @@ class MuttCommands:
             self._cmd1.append(s)
 
     def push(self, s):
-        s = s.replace('"', '"')
+        s = s.replace('"', r"\"")
         s = f'push "{s}"'
         self.debugprint(s)
         self._push.insert(0, s)
@@ -282,17 +660,18 @@ class MuttCommands:
 
 
 def do_setup(
-    extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
+    *,
+    out_f=sys.stdout,
+    temppath=None,
+    tempdir=None,
+    debug_commands=False,
 ):
-    extensions = extensions or []
     temppath = temppath or pathlib.Path(
-        tempfile.mkstemp(prefix="muttmdwn-")[1]
+        tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
     )
     cmds = MuttCommands(out_f, debug=debug_commands)
 
-    editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
-    if extensions:
-        editor = f'{editor} --extensions {",".join(extensions)}'
+    editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 
     cmds.cmd('set my_editor="$editor"')
     cmds.cmd('set my_edit_headers="$edit_headers"')
@@ -304,11 +683,17 @@ def do_setup(
 
 
 def do_massage(
-    maildraft,
+    draft_f,
+    draftpath,
     cmd_f,
     *,
     extensions=None,
+    css_f=None,
     converter=convert_markdown_to_html,
+    related_to_html_only=True,
+    only_build=False,
+    max_other_attachments=20,
+    tempdir=None,
     debug_commands=False,
     debug_walk=False,
 ):
@@ -331,59 +716,131 @@ def do_massage(
     cmds.flush()
 
     extensions = extensions.split(",") if extensions else []
-    tree = converter(maildraft, extensions=extensions)
+    tree = converter(
+        draft_f.read(),
+        draftpath,
+        css=css_f.read() if css_f else None,
+        related_to_html_only=related_to_html_only,
+        tempdir=tempdir,
+        extensions=extensions,
+    )
 
     mimetree = MIMETreeDFWalker(debug=debug_walk)
 
-    def visitor_fn(item, stack, *, debugprint=None):
+    state = dict(pos=1, tags={}, parts=1)
+
+    def visitor_fn(item, ancestry, descendents, *, debugprint=None):
         """
         Visitor function called for every node (part) of the MIME tree,
         depth-first, and responsible for telling NeoMutt how to assemble
         the tree.
         """
+        KILL_LINE = r"\Ca\Ck"
+
         if isinstance(item, Part):
             # We've hit a leaf-node, i.e. an alternative or a related part
             # with actual content.
 
-            # If the part is not an original part, i.e. doesn't already
-            # exist, we must first add it.
-            if not item.orig:
+            # Let's add the part
+            if item.orig:
+                # The original source already exists in the NeoMutt tree, but
+                # the underlying file may have been modified, so we need to
+                # update the encoding, but that's it:
+                cmds.push("<first-entry>")
+                cmds.push("<update-encoding>")
+
+                # We really just need to be able to assume that at this point,
+                # NeoMutt is at position 1, and that we've processed only this
+                # part so far. Nevermind about actual attachments, we can
+                # safely ignore those as they stay at the end.
+                assert state["pos"] == 1
+                assert state["parts"] == 1
+            else:
+                # … whereas all other parts need to be added, and they're all
+                # considered to be temporary and inline:
                 cmds.push(f"<attach-file>{item.path}<enter>")
                 cmds.push("<toggle-unlink><toggle-disposition>")
-                if item.cid:
-                    cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
 
-            # If the item (including the original) comes with a
-            # description, then we might just as well update the NeoMutt
+                # This added a part at the end of the list of parts, and that's
+                # just how many parts we've seen so far, so it's position in
+                # the NeoMutt compose list is the count of parts
+                state["parts"] += 1
+                state["pos"] = state["parts"]
+
+            # If the item (including the original) comes with additional
+            # information, then we might just as well update the NeoMutt
             # tree now:
-            if item.desc:
-                cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
+            if item.cid:
+                cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
+
+            # Now for the biggest hack in this script, which is to handle
+            # attachments, such as PDFs, that aren't related or alternatives.
+            # The problem is that when we add an inline image, it always gets
+            # appended to the list, i.e. inserted *after* other attachments.
+            # Since we don't know the number of attachments, we also cannot
+            # infer the postition of the new attachment. Therefore, we bubble
+            # it all the way to the top, only to then move it down again:
+            if state["pos"] > 1:  # skip for the first part
+                for i in range(max_other_attachments):
+                    # could use any number here, but has to be larger than the
+                    # number of possible attachments. The performance
+                    # difference of using a high number is negligible.
+                    # Bubble up the new part
+                    cmds.push("<move-up>")
+
+                # As we push the part to the right position in the list (i.e.
+                # the last of the subset of attachments this script added), we
+                # must handle the situation that subtrees are skipped by
+                # NeoMutt. Hence, the actual number of positions to move down
+                # is decremented by the number of descendents so far
+                # encountered.
+                for i in range(1, state["pos"] - len(descendents)):
+                    cmds.push("<move-down>")
 
         elif isinstance(item, Multipart):
             # This node has children, but we already visited them (see
-            # above), and so they have been tagged in NeoMutt's compose
-            # window. Now it's just a matter of telling NeoMutt to do the
-            # appropriate grouping:
+            # above). The tags dictionary of State should contain a list of
+            # their positions in the NeoMutt compose window, so iterate those
+            # and tag the parts there:
+            n_tags = len(state["tags"][item])
+            for tag in state["tags"][item]:
+                cmds.push(f"<jump>{tag}<enter><tag-entry>")
+
             if item.subtype == "alternative":
                 cmds.push("<group-alternatives>")
-            elif item.subtype == "relative":
+            elif item.subtype in ("relative", "related"):
                 cmds.push("<group-related>")
             elif item.subtype == "multilingual":
                 cmds.push("<group-multilingual>")
+            else:
+                raise NotImplementedError(
+                    f"Handling of multipart/{item.subtype} is not implemented"
+                )
 
-            # Again, if there is a description, we might just as well:
-            if item.desc:
-                cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
+            state["pos"] -= n_tags - 1
+            state["parts"] += 1
 
         else:
             # We should never get here
-            assert not "is valid part"
-
-        # Finally, if we're at non-root level, tag the new container,
-        # as it might itself be part of a container, to be processed
-        # one level up:
-        if stack:
-            cmds.push("<tag-entry>")
+            raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
+
+        # If the item has a description, we might just as well add it
+        if item.desc:
+            cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
+
+        if ancestry:
+            # If there's an ancestry, record the current (assumed) position in
+            # the NeoMutt compose window as needed-to-tag by our direct parent
+            # (i.e. the last item of the ancestry)
+            state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
+
+            lead = "│ " * (len(ancestry) + 1) + "* "
+            debugprint(
+                f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
+                f"{lead}descendents={[d.subtype for d in descendents]}\n"
+                f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
+                f"{lead}pos={state['pos']}, parts={state['parts']}"
+            )
 
     # -----------------
     # End of visitor_fn
@@ -392,6 +849,9 @@ def do_massage(
     # function
     mimetree.walk(tree, visitor_fn=visitor_fn)
 
+    if not only_build:
+        cmds.push("<send-message>")
+
     # Finally, cleanup. Since we're responsible for removing the temporary
     # file, how's this for a little hack?
     try:
@@ -408,15 +868,26 @@ def do_massage(
 if __name__ == "__main__":
     args = parse_cli_args()
 
-    if args.mode == "setup":
-        do_setup(args.extensions, debug_commands=args.debug_commands)
+    if args.mode is None:
+        do_setup(
+            tempdir=args.tempdir,
+            debug_commands=args.debug_commands,
+        )
 
     elif args.mode == "massage":
-        with open(args.cmdpath, "w") as cmd_f:
+        with open(args.MAILDRAFT, "r") as draft_f, open(
+            args.cmdpath, "w"
+        ) as cmd_f, open(args.css_file, "r") as css_f:
             do_massage(
+                draft_f,
                 args.MAILDRAFT,
                 cmd_f,
                 extensions=args.extensions,
+                css_f=css_f,
+                related_to_html_only=args.related_to_html_only,
+                max_other_attachments=args.max_number_other_attachments,
+                only_build=args.only_build,
+                tempdir=args.tempdir,
                 debug_commands=args.debug_commands,
                 debug_walk=args.debug_walk,
             )
@@ -426,6 +897,7 @@ if __name__ == "__main__":
 
 try:
     import pytest
+    from io import StringIO
 
     class Tests:
         @pytest.fixture
@@ -439,6 +911,7 @@ try:
         # NOTE: tests using the capsys fixture must specify sys.stdout to the
         # functions they call, else old stdout is used and not captured
 
+        @pytest.mark.muttctrl
         def test_MuttCommands_cmd(self, const1, const2, capsys):
             "Assert order of commands"
             cmds = MuttCommands(out_f=sys.stdout)
@@ -448,6 +921,7 @@ try:
             captured = capsys.readouterr()
             assert captured.out == "\n".join((const1, const2, ""))
 
+        @pytest.mark.muttctrl
         def test_MuttCommands_push(self, const1, const2, capsys):
             "Assert reverse order of pushes"
             cmds = MuttCommands(out_f=sys.stdout)
@@ -460,6 +934,15 @@ try:
                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
             )
 
+        @pytest.mark.muttctrl
+        def test_MuttCommands_push_escape(self, const1, const2, capsys):
+            cmds = MuttCommands(out_f=sys.stdout)
+            cmds.push(f'"{const1}"')
+            cmds.flush()
+            captured = capsys.readouterr()
+            assert f'"\\"{const1}\\""' in captured.out
+
+        @pytest.mark.muttctrl
         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
             "Assert reverse order of pushes"
             cmds = MuttCommands(out_f=sys.stdout)
@@ -483,7 +966,7 @@ try:
             assert lines[5] in lines_out[7]
 
         @pytest.fixture
-        def basic_mime_tree(self):
+        def mime_tree_related_to_alternative(self):
             return Multipart(
                 "relative",
                 children=[
@@ -508,114 +991,813 @@ try:
                 desc="Related",
             )
 
-        def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
+        @pytest.fixture
+        def mime_tree_related_to_html(self):
+            return Multipart(
+                "alternative",
+                children=[
+                    Part(
+                        "text",
+                        "plain",
+                        "part.txt",
+                        desc="Plain",
+                        orig=True,
+                    ),
+                    Multipart(
+                        "relative",
+                        children=[
+                            Part("text", "html", "part.html", desc="HTML"),
+                            Part(
+                                "text",
+                                "png",
+                                "logo.png",
+                                cid="logo.png",
+                                desc="Logo",
+                            ),
+                        ],
+                        desc="Related",
+                    ),
+                ],
+                desc="Alternative",
+            )
+
+        @pytest.mark.treewalk
+        def test_MIMETreeDFWalker_depth_first_walk(
+            self, mime_tree_related_to_alternative
+        ):
             mimetree = MIMETreeDFWalker()
 
             items = []
 
-            def visitor_fn(item, stack, debugprint):
-                items.append((item, len(stack)))
+            def visitor_fn(item, ancestry, descendents, debugprint):
+                items.append((item, len(ancestry), len(descendents)))
 
-            mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
+            mimetree.walk(
+                mime_tree_related_to_alternative, visitor_fn=visitor_fn
+            )
             assert len(items) == 5
             assert items[0][0].subtype == "plain"
             assert items[0][1] == 2
+            assert items[0][2] == 0
             assert items[1][0].subtype == "html"
             assert items[1][1] == 2
+            assert items[1][2] == 0
             assert items[2][0].subtype == "alternative"
             assert items[2][1] == 1
+            assert items[2][2] == 2
             assert items[3][0].subtype == "png"
             assert items[3][1] == 1
+            assert items[3][2] == 2
             assert items[4][0].subtype == "relative"
             assert items[4][1] == 0
+            assert items[4][2] == 4
 
-        def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
+        @pytest.mark.treewalk
+        def test_MIMETreeDFWalker_list_to_mixed(self, const1):
             mimetree = MIMETreeDFWalker()
-
             items = []
 
-            def visitor_fn(item, stack, debugprint):
+            def visitor_fn(item, ancestry, descendents, debugprint):
                 items.append(item)
 
-            mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
+            p = Part("text", "plain", const1)
+            mimetree.walk([p], visitor_fn=visitor_fn)
+            assert items[-1].subtype == "plain"
+            mimetree.walk([p, p], visitor_fn=visitor_fn)
             assert items[-1].subtype == "mixed"
 
+        @pytest.mark.treewalk
         def test_MIMETreeDFWalker_visitor_in_constructor(
-            self, basic_mime_tree
+            self, mime_tree_related_to_alternative
         ):
             items = []
 
-            def visitor_fn(item, stack, debugprint):
+            def visitor_fn(item, ancestry, descendents, debugprint):
                 items.append(item)
 
             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
-            mimetree.walk(basic_mime_tree)
+            mimetree.walk(mime_tree_related_to_alternative)
             assert len(items) == 5
 
-        def test_do_setup_no_extensions(self, const1, capsys):
-            "Assert basics about the setup command output"
-            do_setup(temppath=const1, out_f=sys.stdout)
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            assert lines[2].endswith(f'{const1}"')
-            assert lines[4].endswith(const1)
-            assert "first-entry" in lines[-1]
-            assert "edit-file" in lines[-1]
-
-        def test_do_setup_extensions(self, const1, const2, capsys):
-            "Assert that extensions are passed to editor"
-            do_setup(
-                temppath=const1, extensions=[const2, const1], out_f=sys.stdout
+        @pytest.fixture
+        def string_io(self, const1, text=None):
+            return StringIO(text or const1)
+
+        @pytest.mark.massage
+        def test_do_massage_basic(self, const1, string_io, capsys):
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
+                return Part("text", "plain", draftpath, orig=True)
+
+            do_massage(
+                draft_f=string_io,
+                draftpath=const1,
+                cmd_f=sys.stdout,
+                converter=converter,
             )
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            # assert comma-separated list of extensions passed
-            assert lines[2].endswith(f'{const2},{const1}"')
-            assert lines[4].endswith(const1)
 
-        def test_do_massage_basic(self, const1, capsys):
-            def converter(maildraft, extensions):
-                return Part("text", "plain", "/dev/null", orig=True)
+            captured = capsys.readouterr()
+            lines = captured.out.splitlines()
+            assert '="$my_editor"' in lines.pop(0)
+            assert '="$my_edit_headers"' in lines.pop(0)
+            assert "unset my_editor" == lines.pop(0)
+            assert "unset my_edit_headers" == lines.pop(0)
+            assert "send-message" in lines.pop(0)
+            assert "update-encoding" in lines.pop(0)
+            assert "first-entry" in lines.pop(0)
+            assert "source 'rm -f " in lines.pop(0)
+            assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
+
+        @pytest.mark.massage
+        def test_do_massage_fulltree(
+            self, string_io, const1, mime_tree_related_to_alternative, capsys
+        ):
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
+                return mime_tree_related_to_alternative
+
+            max_attachments = 5
+            do_massage(
+                draft_f=string_io,
+                draftpath=const1,
+                cmd_f=sys.stdout,
+                max_other_attachments=max_attachments,
+                converter=converter,
+            )
 
-            do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
             captured = capsys.readouterr()
-            assert (
-                captured.out.strip()
-                == """\
-            set editor="$my_editor"
-            set edit_headers="$my_edit_headers"
-            unset my_editor
-            unset my_edit_headers
-            source 'rm -f pytest_internal_file|'
-            unset my_mdwn_postprocess_cmd_file
-            """.replace(
-                    "            ", ""
-                ).strip()
+            lines = captured.out.splitlines()[4:-2]
+            assert "first-entry" in lines.pop()
+            assert "update-encoding" in lines.pop()
+            assert "Plain" in lines.pop()
+            assert "part.html" in lines.pop()
+            assert "toggle-unlink" in lines.pop()
+            for i in range(max_attachments):
+                assert "move-up" in lines.pop()
+            assert "move-down" in lines.pop()
+            assert "HTML" in lines.pop()
+            assert "jump>1" in lines.pop()
+            assert "jump>2" in lines.pop()
+            assert "group-alternatives" in lines.pop()
+            assert "Alternative" in lines.pop()
+            assert "logo.png" in lines.pop()
+            assert "toggle-unlink" in lines.pop()
+            assert "content-id" in lines.pop()
+            for i in range(max_attachments):
+                assert "move-up" in lines.pop()
+            assert "move-down" in lines.pop()
+            assert "Logo" in lines.pop()
+            assert "jump>1" in lines.pop()
+            assert "jump>4" in lines.pop()
+            assert "group-related" in lines.pop()
+            assert "Related" in lines.pop()
+            assert "send-message" in lines.pop()
+            assert len(lines) == 0
+
+        @pytest.fixture
+        def fake_filewriter(self):
+            class FileWriter:
+                def __init__(self):
+                    self._writes = []
+
+                def __call__(self, path, content, mode="w", **kwargs):
+                    self._writes.append((path, content))
+
+                def pop(self, index=-1):
+                    return self._writes.pop(index)
+
+            return FileWriter()
+
+        @pytest.fixture
+        def markdown_non_converter(self, const1, const2):
+            return lambda s, text: f"{const1}{text}{const2}"
+
+        @pytest.mark.converter
+        def test_converter_tree_basic(self, const1, const2, fake_filewriter):
+            path = pathlib.Path(const2)
+            tree = convert_markdown_to_html(
+                const1, path, filewriter_fn=fake_filewriter
+            )
+
+            assert tree.subtype == "alternative"
+            assert len(tree.children) == 2
+            assert tree.children[0].subtype == "plain"
+            assert tree.children[0].path == path
+            assert tree.children[0].orig
+            assert tree.children[1].subtype == "html"
+            assert tree.children[1].path == path.with_suffix(".html")
+
+        def test_converter_writes(
+            self,
+            const1,
+            const2,
+            fake_filewriter,
+            monkeypatch,
+            markdown_non_converter,
+        ):
+            path = pathlib.Path(const2)
+
+            with monkeypatch.context() as m:
+                m.setattr(markdown.Markdown, "convert", markdown_non_converter)
+                convert_markdown_to_html(
+                    const1, path, filewriter_fn=fake_filewriter
+                )
+
+            assert (path, const1) == fake_filewriter.pop(0)
+            written = fake_filewriter.pop(0)
+            assert path.with_suffix(".html") == written[0]
+            assert const1 in written[1]
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_processor(self):
+            imgpath1 = "file:/path/to/image.png"
+            imgpath2 = "file:///path/to/image.png?url=params"
+            imgpath3 = "/path/to/image.png"
+            text = f"""![inline local image]({imgpath1})
+                       ![image inlined
+                         with newline]({imgpath2})
+                       ![image local path]({imgpath3})"""
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+
+            # local paths have been normalised to URLs:
+            imgpath3 = f"file://{imgpath3}"
+
+            assert 'src="cid:' in html
+            assert "](cid:" in text
+            assert len(images) == 3
+            assert imgpath1 in images
+            assert imgpath2 in images
+            assert imgpath3 in images
+            assert images[imgpath1].cid != images[imgpath2].cid
+            assert images[imgpath1].cid != images[imgpath3].cid
+            assert images[imgpath2].cid != images[imgpath3].cid
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_processor_title_to_desc(self, const1):
+            imgpath = "file:///path/to/image.png"
+            text = f'![inline local image]({imgpath} "{const1}")'
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+            assert images[imgpath].desc == const1
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_processor_alt_to_desc(self, const1):
+            imgpath = "file:///path/to/image.png"
+            text = f"![{const1}]({imgpath})"
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+            assert images[imgpath].desc == const1
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_processor_title_over_alt_desc(
+            self, const1, const2
+        ):
+            imgpath = "file:///path/to/image.png"
+            text = f'![{const1}]({imgpath} "{const2}")'
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+            assert images[imgpath].desc == const2
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_not_external(self):
+            imgpath = "https://path/to/image.png"
+            text = f"![inline image]({imgpath})"
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+
+            assert 'src="cid:' not in html
+            assert "](cid:" not in text
+            assert len(images) == 0
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_local_file(self):
+            imgpath = "/path/to/image.png"
+            text = f"![inline image]({imgpath})"
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+
+            for k, v in images.items():
+                assert k == f"file://{imgpath}"
+                break
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_expanduser(self):
+            imgpath = pathlib.Path("~/image.png")
+            text = f"![inline image]({imgpath})"
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+
+            for k, v in images.items():
+                assert k == f"file://{imgpath.expanduser()}"
+                break
+
+        @pytest.fixture
+        def test_png(self):
+            return (
+                "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
+                "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
+            )
+
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_processor_base64(self, test_png):
+            text = f"![1px white inlined]({test_png})"
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+
+            assert 'src="cid:' in html
+            assert "](cid:" in text
+            assert len(images) == 1
+            assert test_png in images
+
+        @pytest.mark.converter
+        def test_converter_tree_inline_image_base64(
+            self, test_png, const1, fake_filewriter
+        ):
+            text = f"![inline base64 image]({test_png})"
+            path = pathlib.Path(const1)
+            tree = convert_markdown_to_html(
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=False,
+            )
+            assert tree.subtype == "relative"
+            assert tree.children[0].subtype == "alternative"
+            assert tree.children[1].subtype == "png"
+            written = fake_filewriter.pop()
+            assert tree.children[1].path == written[0]
+            assert written[1] == request.urlopen(test_png).read()
+
+        @pytest.mark.converter
+        def test_converter_tree_inline_image_base64_related_to_html(
+            self, test_png, const1, fake_filewriter
+        ):
+            text = f"![inline base64 image]({test_png})"
+            path = pathlib.Path(const1)
+            tree = convert_markdown_to_html(
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=True,
+            )
+            assert tree.subtype == "alternative"
+            assert tree.children[1].subtype == "relative"
+            assert tree.children[1].children[1].subtype == "png"
+            written = fake_filewriter.pop()
+            assert tree.children[1].children[1].path == written[0]
+            assert written[1] == request.urlopen(test_png).read()
+
+        @pytest.mark.converter
+        def test_converter_tree_inline_image_cid(
+            self, const1, fake_filewriter
+        ):
+            text = f"![inline base64 image](cid:{const1})"
+            path = pathlib.Path(const1)
+            tree = convert_markdown_to_html(
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=False,
+            )
+            assert len(tree.children) == 2
+            assert tree.children[0].cid != const1
+            assert tree.children[0].type != "image"
+            assert tree.children[1].cid != const1
+            assert tree.children[1].type != "image"
+
+        @pytest.mark.imgcoll
+        def test_inline_image_collection(
+            self, test_png, const1, const2, fake_filewriter
+        ):
+            test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
+            relparts = collect_inline_images(
+                test_images, filewriter_fn=fake_filewriter
             )
 
-        def test_do_massage_fulltree(self, const1, basic_mime_tree, capsys):
-            def converter(maildraft, extensions):
-                return basic_mime_tree
+            written = fake_filewriter.pop()
+            assert b"PNG" in written[1]
+
+            assert relparts[0].subtype == "png"
+            assert relparts[0].path == written[0]
+            assert relparts[0].cid == const1
+            assert relparts[0].desc.endswith(const2)
+
+        if _PYNLINER:
+
+            @pytest.mark.styling
+            def test_apply_stylesheet(self):
+                html = "<p>Hello, world!</p>"
+                css = "p { color:red }"
+                out = apply_styling(html, css)
+                assert 'p style="color' in out
+
+            @pytest.mark.styling
+            def test_massage_styling_to_converter(self, string_io, const1):
+                css = "p { color:red }"
+                css_f = StringIO(css)
+                out_f = StringIO()
+                css_applied = []
+
+                def converter(
+                    drafttext,
+                    draftpath,
+                    css,
+                    related_to_html_only,
+                    extensions,
+                    tempdir,
+                ):
+                    css_applied.append(css)
+                    return Part("text", "plain", draftpath, orig=True)
+
+                do_massage(
+                    draft_f=string_io,
+                    draftpath=const1,
+                    cmd_f=out_f,
+                    css_f=css_f,
+                    converter=converter,
+                )
+                assert css_applied[0] == css
+
+            @pytest.mark.converter
+            def test_converter_apply_styles(
+                self, const1, fake_filewriter, monkeypatch
+            ):
+                path = pathlib.Path(const1)
+                text = "Hello, world!"
+                css = "p { color:red }"
+                with monkeypatch.context() as m:
+                    m.setattr(
+                        markdown.Markdown,
+                        "convert",
+                        lambda s, t: f"<p>{t}</p>",
+                    )
+                    convert_markdown_to_html(
+                        text, path, css=css, filewriter_fn=fake_filewriter
+                    )
+                assert "color: red" in fake_filewriter.pop()[1]
+
+        if _PYGMENTS_CSS:
+
+            @pytest.mark.styling
+            def test_apply_stylesheet_pygments(self):
+                html = (
+                    f'<div class="{_CODEHILITE_CLASS}">'
+                    "<pre>def foo():\n    return</pre></div>"
+                )
+                out = apply_styling(html, _PYGMENTS_CSS)
+                assert f'{_CODEHILITE_CLASS}" style="' in out
+
+        @pytest.mark.massage
+        def test_mime_tree_relative_within_alternative(
+            self, string_io, const1, capsys, mime_tree_related_to_html
+        ):
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
+                return mime_tree_related_to_html
+
+            do_massage(
+                draft_f=string_io,
+                draftpath=const1,
+                cmd_f=sys.stdout,
+                converter=converter,
+            )
 
-            do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
             captured = capsys.readouterr()
-            lines = captured.out.splitlines()[4:][::-1]
-            assert "Related" in lines.pop()
-            assert "group-related" in lines.pop()
-            assert "tag-entry" in lines.pop()
-            assert "Logo" in lines.pop()
-            assert "content-id" in lines.pop()
+            lines = captured.out.splitlines()[4:-2]
+            assert "first-entry" in lines.pop()
+            assert "update-encoding" in lines.pop()
+            assert "Plain" in lines.pop()
+            assert "part.html" in lines.pop()
             assert "toggle-unlink" in lines.pop()
-            assert "logo.png" in lines.pop()
-            assert "tag-entry" in lines.pop()
-            assert "Alternative" in lines.pop()
-            assert "group-alternatives" in lines.pop()
-            assert "tag-entry" in lines.pop()
+            assert "move-up" in lines.pop()
+            while True:
+                top = lines.pop()
+                if "move-up" not in top:
+                    break
+            assert "move-down" in top
             assert "HTML" in lines.pop()
+            assert "logo.png" in lines.pop()
             assert "toggle-unlink" in lines.pop()
-            assert "part.html" in lines.pop()
-            assert "tag-entry" in lines.pop()
-            assert "Plain" in lines.pop()
+            assert "content-id" in lines.pop()
+            assert "move-up" in lines.pop()
+            while True:
+                top = lines.pop()
+                if "move-up" not in top:
+                    break
+            assert "move-down" in top
+            assert "move-down" in lines.pop()
+            assert "Logo" in lines.pop()
+            assert "jump>2" in lines.pop()
+            assert "jump>3" in lines.pop()
+            assert "group-related" in lines.pop()
+            assert "Related" in lines.pop()
+            assert "jump>1" in lines.pop()
+            assert "jump>2" in lines.pop()
+            assert "group-alternative" in lines.pop()
+            assert "Alternative" in lines.pop()
+            assert "send-message" in lines.pop()
+            assert len(lines) == 0
+
+        @pytest.mark.massage
+        def test_mime_tree_nested_trees_does_not_break_positioning(
+            self, string_io, const1, capsys
+        ):
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
+                return Multipart(
+                    "relative",
+                    children=[
+                        Multipart(
+                            "alternative",
+                            children=[
+                                Part(
+                                    "text",
+                                    "plain",
+                                    "part.txt",
+                                    desc="Plain",
+                                    orig=True,
+                                ),
+                                Multipart(
+                                    "alternative",
+                                    children=[
+                                        Part(
+                                            "text",
+                                            "plain",
+                                            "part.txt",
+                                            desc="Nested plain",
+                                        ),
+                                        Part(
+                                            "text",
+                                            "html",
+                                            "part.html",
+                                            desc="Nested HTML",
+                                        ),
+                                    ],
+                                    desc="Nested alternative",
+                                ),
+                            ],
+                            desc="Alternative",
+                        ),
+                        Part(
+                            "text",
+                            "png",
+                            "logo.png",
+                            cid="logo.png",
+                            desc="Logo",
+                        ),
+                    ],
+                    desc="Related",
+                )
+
+            do_massage(
+                draft_f=string_io,
+                draftpath=const1,
+                cmd_f=sys.stdout,
+                converter=converter,
+            )
+
+            captured = capsys.readouterr()
+            lines = captured.out.splitlines()
+            while "logo.png" not in lines.pop():
+                pass
+            lines.pop()
+            assert "content-id" in lines.pop()
+            assert "move-up" in lines.pop()
+            while True:
+                top = lines.pop()
+                if "move-up" not in top:
+                    break
+            assert "move-down" in top
+            # Due to the nested trees, the number of descendents of the sibling
+            # actually needs to be considered, not just the nieces. So to move
+            # from position 1 to position 6, it only needs one <move-down>
+            # because that jumps over the entire sibling tree. Thus what
+            # follows next must not be another <move-down>
+            assert "Logo" in lines.pop()
+
+        @pytest.mark.sig
+        def test_signature_extraction_no_signature(self, const1):
+            assert (const1, None, None) == extract_signature(const1)
+
+        @pytest.mark.sig
+        def test_signature_extraction_just_text(self, const1, const2):
+            origtext, textsig, htmlsig = extract_signature(
+                f"{const1}{EMAIL_SIG_SEP}{const2}"
+            )
+            assert origtext == const1
+            assert textsig == const2
+            assert htmlsig is None
+
+        @pytest.mark.sig
+        def test_signature_extraction_html(self, const1, const2):
+            path = pathlib.Path("somepath")
+            sigconst = "HTML signature from {path} but as a string"
+
+            def filereader_fn(path):
+                return (
+                    f'<div id="signature">{sigconst.format(path=path)}</div>'
+                )
+
+            origtext, textsig, htmlsig = extract_signature(
+                f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {path}\n{const2}",
+                filereader_fn=filereader_fn,
+            )
+            assert origtext == const1
+            assert textsig == const2
+            assert htmlsig == sigconst.format(path=path)
+
+        @pytest.mark.sig
+        def test_signature_extraction_file_not_found(self, const1):
+            path = pathlib.Path("/does/not/exist")
+            with pytest.raises(FileNotFoundError):
+                origtext, textsig, htmlsig = extract_signature(
+                    f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{path}\n{const1}"
+                )
+
+        @pytest.mark.imgproc
+        def test_image_registry(self, const1):
+            reg = ImageRegistry()
+            cid = reg.register(const1)
+            assert "@" in cid
+            assert not cid.startswith("<")
+            assert not cid.endswith(">")
+            assert const1 in reg
+
+        @pytest.mark.imgproc
+        def test_image_registry_file_uri(self, const1):
+            reg = ImageRegistry()
+            reg.register("/some/path")
+            for path in reg:
+                assert path.startswith("file://")
+                break
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling(
+            self, const1, fake_filewriter, monkeypatch
+        ):
+            path = pathlib.Path(const1)
+
+            mailparts = (
+                "This is the mail body\n",
+                f"{EMAIL_SIG_SEP}",
+                "This is a plain-text signature only",
+            )
+
+            def filereader_fn(path):
+                return ""
+
+            with monkeypatch.context() as m:
+                m.setattr(markdown.Markdown, "convert", lambda s, t: t)
+                convert_markdown_to_html(
+                    "".join(mailparts),
+                    path,
+                    filewriter_fn=fake_filewriter,
+                    filereader_fn=filereader_fn,
+                )
+
+            soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+            body = soup.body.contents
+
+            assert mailparts[0] in body.pop(0)
+
+            sig = soup.select_one("#signature")
+            assert sig == body.pop(0)
+
+            sep = sig.select_one("span.sig_separator")
+            assert sep == sig.contents[0]
+            assert f"\n{sep.text}\n" == EMAIL_SIG_SEP
+
+            assert mailparts[2] in sig.contents[1]
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling_htmlsig(
+            self, const1, fake_filewriter, monkeypatch
+        ):
+            path = pathlib.Path(const1)
+
+            mailparts = (
+                "This is the mail body",
+                f"{EMAIL_SIG_SEP}",
+                f"{HTML_SIG_MARKER}{path}\n",
+                "This is the plain-text version",
+            )
+
+            htmlsig = "HTML Signature from {path}"
+
+            def filereader_fn(path):
+                return f'<div id="signature">{htmlsig.format(path=path)}</div>'
+
+            def mdwn_fn(t):
+                return t.upper()
+
+            with monkeypatch.context() as m:
+                m.setattr(
+                    markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
+                )
+                convert_markdown_to_html(
+                    "".join(mailparts),
+                    path,
+                    filewriter_fn=fake_filewriter,
+                    filereader_fn=filereader_fn,
+                )
+
+            soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+            sig = soup.select_one("#signature")
+            sig.span.extract()
+
+            assert HTML_SIG_MARKER not in sig.text
+            assert htmlsig.format(path=path) == sig.text.strip()
+
+            plaintext = fake_filewriter.pop()[1]
+            assert plaintext.endswith(EMAIL_SIG_SEP + mailparts[-1])
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling_htmlsig_with_image(
+            self, const1, fake_filewriter, monkeypatch, test_png
+        ):
+            path = pathlib.Path(const1)
+
+            mailparts = (
+                "This is the mail body",
+                f"{EMAIL_SIG_SEP}",
+                f"{HTML_SIG_MARKER}{path}\n",
+                "This is the plain-text version",
+            )
+
+            htmlsig = (
+                "HTML Signature from {path} with image\n"
+                f'<img src="{test_png}">\n'
+            )
+
+            def filereader_fn(path):
+                return f'<div id="signature">{htmlsig.format(path=path)}</div>'
+
+            def mdwn_fn(t):
+                return t.upper()
+
+            with monkeypatch.context() as m:
+                m.setattr(
+                    markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
+                )
+                convert_markdown_to_html(
+                    "".join(mailparts),
+                    path,
+                    filewriter_fn=fake_filewriter,
+                    filereader_fn=filereader_fn,
+                )
+
+            assert fake_filewriter.pop()[0].suffix == ".png"
+
+            soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+            assert soup.img.attrs["src"].startswith("cid:")
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling_textsig_with_image(
+            self, const1, fake_filewriter, test_png
+        ):
+            mailparts = (
+                "This is the mail body",
+                f"{EMAIL_SIG_SEP}",
+                "This is the plain-text version with image\n",
+                f"![Inline]({test_png})",
+
+            )
+            tree = convert_markdown_to_html
+                "".join(mailparts),
+                pathlib.Path(const1),
+                filewriter_fn=fake_filewriter,
+            )
+
+            assert tree.subtype == "relative"
+            assert tree.children[0].subtype == "alternative"
+            assert tree.children[1].subtype == "png"
+            written = fake_filewriter.pop()
+            assert tree.children[1].path == written[0]
+            assert written[1] == request.urlopen(test_png).read()
+
+        def test_converter_attribution_to_admonition(self, fake_filewriter):
+
 
 except ImportError:
     pass