]> git.madduck.net Git - etc/neomutt.git/blobdiff - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

add siby alternate
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
index bd446bf1b196a8662d6a74349c73c5655583e5ed..fbc8040061877b74cc0e9fbc4a8e605d21a8a33c 100755 (executable)
@@ -6,8 +6,11 @@
 #
 # Configuration:
 #   neomuttrc (needs to be a single line):
 #
 # Configuration:
 #   neomuttrc (needs to be a single line):
+#     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
 #     macro compose B "\
 #     macro compose B "\
-#       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
+#       <enter-command> source '$my_confdir/buildmimetree.py \
+#       --tempdir $tempdir --extensions $my_mdwn_extensions \
+#       --css-file $my_confdir/htmlmail.css |'<enter>\
 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
 #     " "Convert message into a modern MIME tree with inline images"
 #
 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
 #     " "Convert message into a modern MIME tree with inline images"
 #
 # Requirements:
 #   - python3
 #   - python3-markdown
 # Requirements:
 #   - python3
 #   - python3-markdown
+#   - python3-beautifulsoup4
 # Optional:
 #   - pytest
 # Optional:
 #   - pytest
-#   - Pynliner
-#   - Pygments, if installed, then syntax highlighting is enabled
+#   - Pynliner, provides --css-file and thus inline styling of HTML output
+#   - Pygments, then syntax highlighting for fenced code is enabled
 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
 #
 
 import sys
 #
 
 import sys
+import os
+import os.path
 import pathlib
 import markdown
 import tempfile
 import argparse
 import re
 import mimetypes
 import pathlib
 import markdown
 import tempfile
 import argparse
 import re
 import mimetypes
+import bs4
 from collections import namedtuple, OrderedDict
 from markdown.extensions import Extension
 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
 from collections import namedtuple, OrderedDict
 from markdown.extensions import Extension
 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
@@ -53,74 +60,105 @@ def parse_cli_args(*args, **kwargs):
         )
     )
     parser.epilog = (
         )
     )
     parser.epilog = (
-        "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
+        "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
         "Released under the MIT licence"
     )
 
         "Released under the MIT licence"
     )
 
-    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
-    parser_setup = subp.add_parser("setup", help="Setup phase")
-    parser_massage = subp.add_parser("massage", help="Massaging phase")
+    parser.add_argument(
+        "--extensions",
+        metavar="EXT[,EXT[,EXT]]",
+        type=str,
+        default="",
+        help="Markdown extension to use (comma-separated list)",
+    )
 
 
-    parser_setup.add_argument(
-        "--debug-commands",
+    if _PYNLINER:
+        parser.add_argument(
+            "--css-file",
+            metavar="FILE",
+            type=pathlib.Path,
+            default=os.devnull,
+            help="CSS file to merge with the final HTML",
+        )
+    else:
+        parser.set_defaults(css_file=None)
+
+    parser.add_argument(
+        "--related-to-html-only",
         action="store_true",
         action="store_true",
-        help="Turn on debug logging of commands generated to stderr",
+        help="Make related content be sibling to HTML parts only",
     )
 
     )
 
-    parser_setup.add_argument(
-        "--extension",
-        "-x",
-        metavar="EXTENSION",
-        dest="extensions",
-        nargs="?",
-        default=[],
-        action="append",
-        help="Markdown extension to add to the list of extensions use",
+    def positive_integer(value):
+        try:
+            if int(value) > 0:
+                return int(value)
+
+        except ValueError:
+            pass
+
+        raise ValueError("Must be a positive integer")
+
+    parser.add_argument(
+        "--max-number-other-attachments",
+        metavar="INTEGER",
+        type=positive_integer,
+        default=20,
+        help="Maximum number of other attachments to expect",
     )
 
     )
 
-    parser_setup.add_argument(
-        "--send-message",
+    parser.add_argument(
+        "--only-build",
+        "--just-build",
         action="store_true",
         action="store_true",
-        help="Generate command(s) to send the message after processing",
+        help="Only build, don't send the message",
+    )
+
+    parser.add_argument(
+        "--tempdir",
+        metavar="DIR",
+        type=pathlib.Path,
+        help="Specify temporary directory to use for attachments",
     )
 
     )
 
-    parser_massage.add_argument(
+    parser.add_argument(
         "--debug-commands",
         action="store_true",
         help="Turn on debug logging of commands generated to stderr",
     )
 
         "--debug-commands",
         action="store_true",
         help="Turn on debug logging of commands generated to stderr",
     )
 
-    parser_massage.add_argument(
+    parser.add_argument(
         "--debug-walk",
         action="store_true",
         help="Turn on debugging to stderr of the MIME tree walk",
     )
 
         "--debug-walk",
         action="store_true",
         help="Turn on debugging to stderr of the MIME tree walk",
     )
 
-    parser_massage.add_argument(
-        "--extensions",
-        metavar="EXTENSIONS",
-        type=str,
-        default="",
-        help="Markdown extension to use (comma-separated list)",
+    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
+    massage_p = subp.add_parser(
+        "massage", help="Massaging phase (internal use)"
     )
 
     )
 
-    parser_massage.add_argument(
+    massage_p.add_argument(
         "--write-commands-to",
         "--write-commands-to",
-        metavar="PATH",
+        "-o",
+        metavar="FILE",
         dest="cmdpath",
         dest="cmdpath",
+        type=pathlib.Path,
+        required=True,
         help="Temporary file path to write commands to",
     )
 
         help="Temporary file path to write commands to",
     )
 
-    parser_massage.add_argument(
+    massage_p.add_argument(
         "MAILDRAFT",
         nargs="?",
         "MAILDRAFT",
         nargs="?",
+        type=pathlib.Path,
         help="If provided, the script is invoked as editor on the mail draft",
     )
 
     return parser.parse_args(*args, **kwargs)
 
 
         help="If provided, the script is invoked as editor on the mail draft",
     )
 
     return parser.parse_args(*args, **kwargs)
 
 
-# [ MARKDOWN WRAPPING ] #######################################################
+# [ IMAGE HANDLING ] ##########################################################
 
 
 InlineImageInfo = namedtuple(
 
 
 InlineImageInfo = namedtuple(
@@ -128,11 +166,42 @@ InlineImageInfo = namedtuple(
 )
 
 
 )
 
 
+class ImageRegistry:
+    def __init__(self):
+        self._images = OrderedDict()
+
+    def register(self, path, description=None):
+        path = os.path.expanduser(path)
+        if path.startswith("/"):
+            path = f"file://{path}"
+        cid = make_msgid()[1:-1]
+        self._images[path] = InlineImageInfo(cid, description)
+        return cid
+
+    def __iter__(self):
+        return self._images.__iter__()
+
+    def __getitem__(self, idx):
+        return self._images.__getitem__(idx)
+
+    def __len__(self):
+        return self._images.__len__()
+
+    def items(self):
+        return self._images.items()
+
+    def __repr__(self):
+        return f"<ImageRegistry(items={len(self._images)})>"
+
+    def __str__(self):
+        return self._images.__str__()
+
+
 class InlineImageExtension(Extension):
     class RelatedImageInlineProcessor(ImageInlineProcessor):
 class InlineImageExtension(Extension):
     class RelatedImageInlineProcessor(ImageInlineProcessor):
-        def __init__(self, re, md, ext):
+        def __init__(self, re, md, registry):
             super().__init__(re, md)
             super().__init__(re, md)
-            self._ext = ext
+            self._registry = registry
 
         def handleMatch(self, m, data):
             el, start, end = super().handleMatch(m, data)
 
         def handleMatch(self, m, data):
             el, start, end = super().handleMatch(m, data)
@@ -140,52 +209,90 @@ class InlineImageExtension(Extension):
                 src = el.attrib["src"]
                 if "://" not in src or src.startswith("file://"):
                     # We only inline local content
                 src = el.attrib["src"]
                 if "://" not in src or src.startswith("file://"):
                     # We only inline local content
-                    cid = self._ext.get_cid_for_image(el.attrib)
+                    cid = self._registry.register(
+                        el.attrib["src"],
+                        el.attrib.get("title", el.attrib.get("alt")),
+                    )
                     el.attrib["src"] = f"cid:{cid}"
             return el, start, end
 
                     el.attrib["src"] = f"cid:{cid}"
             return el, start, end
 
-    def __init__(self):
+    def __init__(self, registry):
         super().__init__()
         super().__init__()
-        self._images = OrderedDict()
+        self._image_registry = registry
+
+    INLINE_PATTERN_NAME = "image_link"
 
     def extendMarkdown(self, md):
         md.registerExtension(self)
         inline_image_proc = self.RelatedImageInlineProcessor(
 
     def extendMarkdown(self, md):
         md.registerExtension(self)
         inline_image_proc = self.RelatedImageInlineProcessor(
-            IMAGE_LINK_RE, md, self
+            IMAGE_LINK_RE, md, self._image_registry
         )
         )
-        md.inlinePatterns.register(inline_image_proc, "image_link", 150)
-
-    def get_cid_for_image(self, attrib):
-        msgid = make_msgid()[1:-1]
-        path = attrib["src"]
-        if path.startswith("/"):
-            path = f"file://{path}"
-        self._images[path] = InlineImageInfo(
-            msgid, attrib.get("title", attrib.get("alt"))
+        md.inlinePatterns.register(
+            inline_image_proc, InlineImageExtension.INLINE_PATTERN_NAME, 150
         )
         )
-        return msgid
 
 
-    def get_images(self):
-        return self._images
 
 
-
-def markdown_with_inline_image_support(text, *, extensions=None):
-    inline_image_handler = InlineImageExtension()
+def markdown_with_inline_image_support(
+    text,
+    *,
+    mdwn=None,
+    image_registry=None,
+    extensions=None,
+    extension_configs=None,
+):
+    registry = (
+        image_registry if image_registry is not None else ImageRegistry()
+    )
+    inline_image_handler = InlineImageExtension(registry=registry)
     extensions = extensions or []
     extensions.append(inline_image_handler)
     extensions = extensions or []
     extensions.append(inline_image_handler)
-    mdwn = markdown.Markdown(extensions=extensions)
-    htmltext = mdwn.convert(text)
+    mdwn = markdown.Markdown(
+        extensions=extensions, extension_configs=extension_configs
+    )
 
 
-    images = inline_image_handler.get_images()
+    htmltext = mdwn.convert(text)
 
     def replace_image_with_cid(matchobj):
         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 
     def replace_image_with_cid(matchobj):
         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
-            if m in images:
-                return f"(cid:{images[m].cid}"
+            if m in registry:
+                return f"(cid:{registry[m].cid}"
         return matchobj.group(0)
 
     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
         return matchobj.group(0)
 
     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
-    return text, htmltext, images
+    return text, htmltext, registry, mdwn
+
+
+# [ CSS STYLING ] #############################################################
+
+
+try:
+    import pynliner
+
+    _PYNLINER = True
+
+except ImportError:
+    _PYNLINER = False
+
+try:
+    from pygments.formatters import get_formatter_by_name
+
+    _CODEHILITE_CLASS = "codehilite"
+
+    _PYGMENTS_CSS = get_formatter_by_name(
+        "html", style="default"
+    ).get_style_defs(f".{_CODEHILITE_CLASS}")
+
+except ImportError:
+    _PYGMENTS_CSS = None
+
+
+def apply_styling(html, css):
+    return (
+        pynliner.Pynliner()
+        .from_string(html)
+        .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
+        .run()
+    )
 
 
 # [ PARTS GENERATION ] ########################################################
 
 
 # [ PARTS GENERATION ] ########################################################
@@ -213,6 +320,14 @@ class Multipart(
     def __str__(self):
         return f"<multipart/{self.subtype}> children={len(self.children)}"
 
     def __str__(self):
         return f"<multipart/{self.subtype}> children={len(self.children)}"
 
+    def __hash__(self):
+        return hash(str(self.subtype) + "".join(str(self.children)))
+
+
+def filereader_fn(path, mode="r", **kwargs):
+    with open(path, mode, **kwargs) as in_f:
+        return in_f.read()
+
 
 def filewriter_fn(path, content, mode="w", **kwargs):
     with open(path, mode, **kwargs) as out_f:
 
 def filewriter_fn(path, content, mode="w", **kwargs):
     with open(path, mode, **kwargs) as out_f:
@@ -220,10 +335,13 @@ def filewriter_fn(path, content, mode="w", **kwargs):
 
 
 def collect_inline_images(
 
 
 def collect_inline_images(
-    images, *, tempdir=None, filewriter_fn=filewriter_fn
+    image_registry, *, tempdir=None, filewriter_fn=filewriter_fn
 ):
     relparts = []
 ):
     relparts = []
-    for path, info in images.items():
+    for path, info in image_registry.items():
+        if path.startswith("cid:"):
+            continue
+
         data = request.urlopen(path)
 
         mimetype = data.headers["Content-Type"]
         data = request.urlopen(path)
 
         mimetype = data.headers["Content-Type"]
@@ -233,96 +351,261 @@ def collect_inline_images(
 
         filewriter_fn(path, data.read(), "w+b")
 
 
         filewriter_fn(path, data.read(), "w+b")
 
+        desc = (
+            f'Inline image: "{info.desc}"'
+            if info.desc
+            else f"Inline image {str(len(relparts)+1)}"
+        )
         relparts.append(
         relparts.append(
-            Part(*mimetype.split("/"), path, cid=info.cid, desc=info.desc)
+            Part(*mimetype.split("/"), path, cid=info.cid, desc=desc)
         )
 
     return relparts
 
 
         )
 
     return relparts
 
 
+EMAIL_SIG_SEP = "\n-- \n"
+HTML_SIG_MARKER = "=htmlsig "
+
+
+def make_html_doc(body, sig=None):
+    ret = (
+        "<!DOCTYPE html>\n"
+        "<html>\n"
+        "<head>\n"
+        '<meta http-equiv="content-type" content="text/html; charset=UTF-8">\n'  # noqa: E501
+        '<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'  # noqa: E501
+        "</head>\n"
+        "<body>\n"
+        f"{body}\n"
+    )
+
+    if sig:
+        nl = "\n"
+        ret = (
+            f'{ret}<div id="signature"><span class="sig_separator">{EMAIL_SIG_SEP.strip(nl)}</span>\n'  # noqa: E501
+            f"{sig}\n"
+            "</div>"
+        )
+
+    return f"{ret}\n  </body>\n</html>"
+
+
+def make_text_mail(text, sig=None):
+    return EMAIL_SIG_SEP.join((text, sig)) if sig else text
+
+
+def extract_signature(text, *, filereader_fn=filereader_fn):
+    parts = text.split(EMAIL_SIG_SEP, 1)
+    if len(parts) == 1:
+        return text, None, None
+
+    lines = parts[1].splitlines()
+    if lines[0].startswith(HTML_SIG_MARKER):
+        path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1])
+        textsig = "\n".join(lines)
+
+        sig_input = filereader_fn(path.expanduser())
+        soup = bs4.BeautifulSoup(sig_input, "html.parser")
+
+        style = str(soup.style.extract()) if soup.style else ""
+        for sig_selector in (
+            "#signature",
+            "#signatur",
+            "#emailsig",
+            ".signature",
+            ".signatur",
+            ".emailsig",
+            "body",
+            "div",
+        ):
+            sig = soup.select_one(sig_selector)
+            if sig:
+                break
+
+        if not sig:
+            return parts[0], textsig, style + sig_input
+
+        if sig.attrs.get("id") == "signature":
+            sig = "".join(str(c) for c in sig.children)
+
+        return parts[0], textsig, style + str(sig)
+
+    return parts[0], parts[1], None
+
+
 def convert_markdown_to_html(
 def convert_markdown_to_html(
-    origtext, draftpath, *, filewriter_fn=filewriter_fn, extensions=None
+    origtext,
+    draftpath,
+    *,
+    related_to_html_only=False,
+    css=None,
+    filewriter_fn=filewriter_fn,
+    filereader_fn=filereader_fn,
+    tempdir=None,
+    extensions=None,
+    extension_configs=None,
 ):
 ):
-    origtext, htmltext, images = markdown_with_inline_image_support(
-        origtext, extensions=extensions
+    # TODO extension_configs need to be handled differently
+    extension_configs = extension_configs or {}
+    extension_configs.setdefault("pymdownx.highlight", {})
+    extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
+
+    origtext, textsig, htmlsig = extract_signature(
+        origtext, filereader_fn=filereader_fn
     )
 
     )
 
+    (
+        origtext,
+        htmltext,
+        image_registry,
+        mdwn,
+    ) = markdown_with_inline_image_support(
+        origtext, extensions=extensions, extension_configs=extension_configs
+    )
+
+    if htmlsig:
+        if not textsig:
+            # TODO: decide what to do if there is no plain-text version
+            raise NotImplementedError("HTML signature but no text alternative")
+
+        soup = bs4.BeautifulSoup(htmlsig, "html.parser")
+        for img in soup.find_all("img"):
+            uri = img.attrs["src"]
+            desc = img.attrs.get("title", img.attrs.get("alt"))
+            cid = image_registry.register(uri, desc)
+            img.attrs["src"] = f"cid:{cid}"
+
+        htmlsig = str(soup)
+
+    elif textsig:
+        (
+            textsig,
+            htmlsig,
+            image_registry,
+            mdwn,
+        ) = markdown_with_inline_image_support(
+            textsig,
+            extensions=extensions,
+            extension_configs=extension_configs,
+            image_registry=image_registry,
+            mdwn=mdwn,
+        )
+
+    origtext = make_text_mail(origtext, textsig)
+
     filewriter_fn(draftpath, origtext, encoding="utf-8")
     textpart = Part(
         "text", "plain", draftpath, "Plain-text version", orig=True
     )
 
     filewriter_fn(draftpath, origtext, encoding="utf-8")
     textpart = Part(
         "text", "plain", draftpath, "Plain-text version", orig=True
     )
 
+    htmltext = make_html_doc(htmltext, htmlsig)
+    htmltext = apply_styling(htmltext, css)
+
     htmlpath = draftpath.with_suffix(".html")
     filewriter_fn(
         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
     )
     htmlpart = Part("text", "html", htmlpath, "HTML version")
 
     htmlpath = draftpath.with_suffix(".html")
     filewriter_fn(
         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
     )
     htmlpart = Part("text", "html", htmlpath, "HTML version")
 
-    altpart = Multipart(
-        "alternative", [textpart, htmlpart], "Group of alternative content"
+    imgparts = collect_inline_images(
+        image_registry, tempdir=tempdir, filewriter_fn=filewriter_fn
     )
 
     )
 
-    imgparts = collect_inline_images(images, filewriter_fn=filewriter_fn)
-    if imgparts:
+    if related_to_html_only:
+        # If there are inline image part, they will be contained within a
+        # multipart/related part along with the HTML part only
+        if imgparts:
+            # replace htmlpart with a multipart/related container of the HTML
+            # parts and the images
+            htmlpart = Multipart(
+                "relative", [htmlpart] + imgparts, "Group of related content"
+            )
+
         return Multipart(
         return Multipart(
-            "relative", [altpart] + imgparts, "Group of related content"
+            "alternative", [textpart, htmlpart], "Group of alternative content"
         )
         )
+
     else:
     else:
-        return altpart
+        # If there are inline image part, they will be siblings to the
+        # multipart/alternative tree within a multipart/related part
+        altpart = Multipart(
+            "alternative", [textpart, htmlpart], "Group of alternative content"
+        )
+        if imgparts:
+            return Multipart(
+                "relative", [altpart] + imgparts, "Group of related content"
+            )
+        else:
+            return altpart
 
 
 class MIMETreeDFWalker:
     def __init__(self, *, visitor_fn=None, debug=False):
 
 
 class MIMETreeDFWalker:
     def __init__(self, *, visitor_fn=None, debug=False):
-        self._visitor_fn = visitor_fn
+        self._visitor_fn = visitor_fn or self._echovisit
         self._debug = debug
 
         self._debug = debug
 
+    def _echovisit(self, node, ancestry, debugprint):
+        debugprint(f"node={node} ancestry={ancestry}")
+
     def walk(self, root, *, visitor_fn=None):
         """
         Recursive function to implement a depth-dirst walk of the MIME-tree
         rooted at `root`.
         """
     def walk(self, root, *, visitor_fn=None):
         """
         Recursive function to implement a depth-dirst walk of the MIME-tree
         rooted at `root`.
         """
-
         if isinstance(root, list):
         if isinstance(root, list):
-            root = Multipart("mixed", children=root)
+            if len(root) > 1:
+                root = Multipart("mixed", children=root)
+            else:
+                root = root[0]
 
         self._walk(
             root,
 
         self._walk(
             root,
-            stack=[],
+            ancestry=[],
+            descendents=[],
             visitor_fn=visitor_fn or self._visitor_fn,
         )
 
             visitor_fn=visitor_fn or self._visitor_fn,
         )
 
-    def _walk(self, node, *, stack, visitor_fn):
+    def _walk(self, node, *, ancestry, descendents, visitor_fn):
         # Let's start by enumerating the parts at the current level. At the
         # Let's start by enumerating the parts at the current level. At the
-        # root level, stack will be the empty list, and we expect a multipart/*
-        # container at this level. Later, e.g. within a mutlipart/alternative
-        # container, the subtree will just be the alternative parts, while the
-        # top of the stack will be the multipart/alternative container, which
-        # we will process after the following loop.
-
-        lead = f"{'| '*len(stack)}|-"
+        # root level, ancestry will be the empty list, and we expect a
+        # multipart/* container at this level. Later, e.g. within a
+        # mutlipart/alternative container, the subtree will just be the
+        # alternative parts, while the top of the ancestry will be the
+        # multipart/alternative container, which we will process after the
+        # following loop.
+
+        lead = f"{'│ '*len(ancestry)}"
         if isinstance(node, Multipart):
             self.debugprint(
         if isinstance(node, Multipart):
             self.debugprint(
-                f"{lead}{node} parents={[s.subtype for s in stack]}"
+                f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
             )
 
             )
 
-            # Depth-first, so push the current container onto the stack,
-            # then descend …
-            stack.append(node)
-            self.debugprint("| " * (len(stack) + 1))
+            # Depth-first, so push the current container onto the ancestry
+            # stack, then descend …
+            ancestry.append(node)
+            self.debugprint(lead + "│ " * 2)
             for child in node.children:
                 self._walk(
                     child,
             for child in node.children:
                 self._walk(
                     child,
-                    stack=stack,
+                    ancestry=ancestry,
+                    descendents=descendents,
                     visitor_fn=visitor_fn,
                 )
                     visitor_fn=visitor_fn,
                 )
-            self.debugprint("| " * len(stack))
-            assert stack.pop() == node
+            assert ancestry.pop() == node
+            sibling_descendents = descendents
+            descendents.extend(node.children)
 
         else:
 
         else:
-            self.debugprint(f"{lead}{node}")
+            self.debugprint(f"{lead}├{node}")
+            sibling_descendents = descendents
+
+        if False and ancestry:
+            self.debugprint(lead[:-1] + " │")
 
         if visitor_fn:
 
         if visitor_fn:
-            visitor_fn(node, stack, debugprint=self.debugprint)
+            visitor_fn(
+                node, ancestry, sibling_descendents, debugprint=self.debugprint
+            )
 
     def debugprint(self, s, **kwargs):
         if self._debug:
 
     def debugprint(self, s, **kwargs):
         if self._debug:
@@ -360,7 +643,7 @@ class MuttCommands:
             self._cmd1.append(s)
 
     def push(self, s):
             self._cmd1.append(s)
 
     def push(self, s):
-        s = s.replace('"', '"')
+        s = s.replace('"', r"\"")
         s = f'push "{s}"'
         self.debugprint(s)
         self._push.insert(0, s)
         s = f'push "{s}"'
         self.debugprint(s)
         self._push.insert(0, s)
@@ -377,19 +660,18 @@ class MuttCommands:
 
 
 def do_setup(
 
 
 def do_setup(
-    extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
+    *,
+    out_f=sys.stdout,
+    temppath=None,
+    tempdir=None,
+    debug_commands=False,
 ):
 ):
-    extensions = extensions or []
     temppath = temppath or pathlib.Path(
     temppath = temppath or pathlib.Path(
-        tempfile.mkstemp(prefix="muttmdwn-")[1]
+        tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
     )
     cmds = MuttCommands(out_f, debug=debug_commands)
 
     )
     cmds = MuttCommands(out_f, debug=debug_commands)
 
-    editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
-    if extensions:
-        editor = f'{editor} --extensions {",".join(extensions)}'
-    if debug_commands:
-        editor = f"{editor} --debug-commands"
+    editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 
     cmds.cmd('set my_editor="$editor"')
     cmds.cmd('set my_edit_headers="$edit_headers"')
 
     cmds.cmd('set my_editor="$editor"')
     cmds.cmd('set my_edit_headers="$edit_headers"')
@@ -406,7 +688,12 @@ def do_massage(
     cmd_f,
     *,
     extensions=None,
     cmd_f,
     *,
     extensions=None,
+    css_f=None,
     converter=convert_markdown_to_html,
     converter=convert_markdown_to_html,
+    related_to_html_only=True,
+    only_build=False,
+    max_other_attachments=20,
+    tempdir=None,
     debug_commands=False,
     debug_walk=False,
 ):
     debug_commands=False,
     debug_walk=False,
 ):
@@ -429,16 +716,27 @@ def do_massage(
     cmds.flush()
 
     extensions = extensions.split(",") if extensions else []
     cmds.flush()
 
     extensions = extensions.split(",") if extensions else []
-    tree = converter(draft_f.read(), draftpath, extensions=extensions)
+    tree = converter(
+        draft_f.read(),
+        draftpath,
+        css=css_f.read() if css_f else None,
+        related_to_html_only=related_to_html_only,
+        tempdir=tempdir,
+        extensions=extensions,
+    )
 
     mimetree = MIMETreeDFWalker(debug=debug_walk)
 
 
     mimetree = MIMETreeDFWalker(debug=debug_walk)
 
-    def visitor_fn(item, stack, *, debugprint=None):
+    state = dict(pos=1, tags={}, parts=1)
+
+    def visitor_fn(item, ancestry, descendents, *, debugprint=None):
         """
         Visitor function called for every node (part) of the MIME tree,
         depth-first, and responsible for telling NeoMutt how to assemble
         the tree.
         """
         """
         Visitor function called for every node (part) of the MIME tree,
         depth-first, and responsible for telling NeoMutt how to assemble
         the tree.
         """
+        KILL_LINE = r"\Ca\Ck"
+
         if isinstance(item, Part):
             # We've hit a leaf-node, i.e. an alternative or a related part
             # with actual content.
         if isinstance(item, Part):
             # We've hit a leaf-node, i.e. an alternative or a related part
             # with actual content.
@@ -448,44 +746,101 @@ def do_massage(
                 # The original source already exists in the NeoMutt tree, but
                 # the underlying file may have been modified, so we need to
                 # update the encoding, but that's it:
                 # The original source already exists in the NeoMutt tree, but
                 # the underlying file may have been modified, so we need to
                 # update the encoding, but that's it:
+                cmds.push("<first-entry>")
                 cmds.push("<update-encoding>")
                 cmds.push("<update-encoding>")
+
+                # We really just need to be able to assume that at this point,
+                # NeoMutt is at position 1, and that we've processed only this
+                # part so far. Nevermind about actual attachments, we can
+                # safely ignore those as they stay at the end.
+                assert state["pos"] == 1
+                assert state["parts"] == 1
             else:
                 # … whereas all other parts need to be added, and they're all
                 # considered to be temporary and inline:
                 cmds.push(f"<attach-file>{item.path}<enter>")
                 cmds.push("<toggle-unlink><toggle-disposition>")
 
             else:
                 # … whereas all other parts need to be added, and they're all
                 # considered to be temporary and inline:
                 cmds.push(f"<attach-file>{item.path}<enter>")
                 cmds.push("<toggle-unlink><toggle-disposition>")
 
+                # This added a part at the end of the list of parts, and that's
+                # just how many parts we've seen so far, so it's position in
+                # the NeoMutt compose list is the count of parts
+                state["parts"] += 1
+                state["pos"] = state["parts"]
+
             # If the item (including the original) comes with additional
             # information, then we might just as well update the NeoMutt
             # tree now:
             if item.cid:
             # If the item (including the original) comes with additional
             # information, then we might just as well update the NeoMutt
             # tree now:
             if item.cid:
-                cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
+                cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
+
+            # Now for the biggest hack in this script, which is to handle
+            # attachments, such as PDFs, that aren't related or alternatives.
+            # The problem is that when we add an inline image, it always gets
+            # appended to the list, i.e. inserted *after* other attachments.
+            # Since we don't know the number of attachments, we also cannot
+            # infer the postition of the new attachment. Therefore, we bubble
+            # it all the way to the top, only to then move it down again:
+            if state["pos"] > 1:  # skip for the first part
+                for i in range(max_other_attachments):
+                    # could use any number here, but has to be larger than the
+                    # number of possible attachments. The performance
+                    # difference of using a high number is negligible.
+                    # Bubble up the new part
+                    cmds.push("<move-up>")
+
+                # As we push the part to the right position in the list (i.e.
+                # the last of the subset of attachments this script added), we
+                # must handle the situation that subtrees are skipped by
+                # NeoMutt. Hence, the actual number of positions to move down
+                # is decremented by the number of descendents so far
+                # encountered.
+                for i in range(1, state["pos"] - len(descendents)):
+                    cmds.push("<move-down>")
 
         elif isinstance(item, Multipart):
             # This node has children, but we already visited them (see
 
         elif isinstance(item, Multipart):
             # This node has children, but we already visited them (see
-            # above), and so they have been tagged in NeoMutt's compose
-            # window. Now it's just a matter of telling NeoMutt to do the
-            # appropriate grouping:
+            # above). The tags dictionary of State should contain a list of
+            # their positions in the NeoMutt compose window, so iterate those
+            # and tag the parts there:
+            n_tags = len(state["tags"][item])
+            for tag in state["tags"][item]:
+                cmds.push(f"<jump>{tag}<enter><tag-entry>")
+
             if item.subtype == "alternative":
                 cmds.push("<group-alternatives>")
             if item.subtype == "alternative":
                 cmds.push("<group-alternatives>")
-            elif item.subtype == "relative":
+            elif item.subtype in ("relative", "related"):
                 cmds.push("<group-related>")
             elif item.subtype == "multilingual":
                 cmds.push("<group-multilingual>")
                 cmds.push("<group-related>")
             elif item.subtype == "multilingual":
                 cmds.push("<group-multilingual>")
+            else:
+                raise NotImplementedError(
+                    f"Handling of multipart/{item.subtype} is not implemented"
+                )
+
+            state["pos"] -= n_tags - 1
+            state["parts"] += 1
 
         else:
             # We should never get here
 
         else:
             # We should never get here
-            assert not "is valid part"
+            raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
 
         # If the item has a description, we might just as well add it
         if item.desc:
 
         # If the item has a description, we might just as well add it
         if item.desc:
-            cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
-
-        # Finally, if we're at non-root level, tag the new container,
-        # as it might itself be part of a container, to be processed
-        # one level up:
-        if stack:
-            cmds.push("<tag-entry>")
+            cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
+
+        if ancestry:
+            # If there's an ancestry, record the current (assumed) position in
+            # the NeoMutt compose window as needed-to-tag by our direct parent
+            # (i.e. the last item of the ancestry)
+            state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
+
+            lead = "│ " * (len(ancestry) + 1) + "* "
+            debugprint(
+                f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
+                f"{lead}descendents={[d.subtype for d in descendents]}\n"
+                f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
+                f"{lead}pos={state['pos']}, parts={state['parts']}"
+            )
 
     # -----------------
     # End of visitor_fn
 
     # -----------------
     # End of visitor_fn
@@ -494,6 +849,9 @@ def do_massage(
     # function
     mimetree.walk(tree, visitor_fn=visitor_fn)
 
     # function
     mimetree.walk(tree, visitor_fn=visitor_fn)
 
+    if not only_build:
+        cmds.push("<send-message>")
+
     # Finally, cleanup. Since we're responsible for removing the temporary
     # file, how's this for a little hack?
     try:
     # Finally, cleanup. Since we're responsible for removing the temporary
     # file, how's this for a little hack?
     try:
@@ -510,21 +868,26 @@ def do_massage(
 if __name__ == "__main__":
     args = parse_cli_args()
 
 if __name__ == "__main__":
     args = parse_cli_args()
 
-    if args.mode == "setup":
-        if args.send_message:
-            raise NotImplementedError()
-
-        do_setup(args.extensions, debug_commands=args.debug_commands)
+    if args.mode is None:
+        do_setup(
+            tempdir=args.tempdir,
+            debug_commands=args.debug_commands,
+        )
 
     elif args.mode == "massage":
         with open(args.MAILDRAFT, "r") as draft_f, open(
             args.cmdpath, "w"
 
     elif args.mode == "massage":
         with open(args.MAILDRAFT, "r") as draft_f, open(
             args.cmdpath, "w"
-        ) as cmd_f:
+        ) as cmd_f, open(args.css_file, "r") as css_f:
             do_massage(
                 draft_f,
             do_massage(
                 draft_f,
-                pathlib.Path(args.MAILDRAFT),
+                args.MAILDRAFT,
                 cmd_f,
                 extensions=args.extensions,
                 cmd_f,
                 extensions=args.extensions,
+                css_f=css_f,
+                related_to_html_only=args.related_to_html_only,
+                max_other_attachments=args.max_number_other_attachments,
+                only_build=args.only_build,
+                tempdir=args.tempdir,
                 debug_commands=args.debug_commands,
                 debug_walk=args.debug_walk,
             )
                 debug_commands=args.debug_commands,
                 debug_walk=args.debug_walk,
             )
@@ -548,6 +911,7 @@ try:
         # NOTE: tests using the capsys fixture must specify sys.stdout to the
         # functions they call, else old stdout is used and not captured
 
         # NOTE: tests using the capsys fixture must specify sys.stdout to the
         # functions they call, else old stdout is used and not captured
 
+        @pytest.mark.muttctrl
         def test_MuttCommands_cmd(self, const1, const2, capsys):
             "Assert order of commands"
             cmds = MuttCommands(out_f=sys.stdout)
         def test_MuttCommands_cmd(self, const1, const2, capsys):
             "Assert order of commands"
             cmds = MuttCommands(out_f=sys.stdout)
@@ -557,6 +921,7 @@ try:
             captured = capsys.readouterr()
             assert captured.out == "\n".join((const1, const2, ""))
 
             captured = capsys.readouterr()
             assert captured.out == "\n".join((const1, const2, ""))
 
+        @pytest.mark.muttctrl
         def test_MuttCommands_push(self, const1, const2, capsys):
             "Assert reverse order of pushes"
             cmds = MuttCommands(out_f=sys.stdout)
         def test_MuttCommands_push(self, const1, const2, capsys):
             "Assert reverse order of pushes"
             cmds = MuttCommands(out_f=sys.stdout)
@@ -569,6 +934,15 @@ try:
                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
             )
 
                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
             )
 
+        @pytest.mark.muttctrl
+        def test_MuttCommands_push_escape(self, const1, const2, capsys):
+            cmds = MuttCommands(out_f=sys.stdout)
+            cmds.push(f'"{const1}"')
+            cmds.flush()
+            captured = capsys.readouterr()
+            assert f'"\\"{const1}\\""' in captured.out
+
+        @pytest.mark.muttctrl
         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
             "Assert reverse order of pushes"
             cmds = MuttCommands(out_f=sys.stdout)
         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
             "Assert reverse order of pushes"
             cmds = MuttCommands(out_f=sys.stdout)
@@ -592,7 +966,7 @@ try:
             assert lines[5] in lines_out[7]
 
         @pytest.fixture
             assert lines[5] in lines_out[7]
 
         @pytest.fixture
-        def basic_mime_tree(self):
+        def mime_tree_related_to_alternative(self):
             return Multipart(
                 "relative",
                 children=[
             return Multipart(
                 "relative",
                 children=[
@@ -617,76 +991,108 @@ try:
                 desc="Related",
             )
 
                 desc="Related",
             )
 
-        def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
+        @pytest.fixture
+        def mime_tree_related_to_html(self):
+            return Multipart(
+                "alternative",
+                children=[
+                    Part(
+                        "text",
+                        "plain",
+                        "part.txt",
+                        desc="Plain",
+                        orig=True,
+                    ),
+                    Multipart(
+                        "relative",
+                        children=[
+                            Part("text", "html", "part.html", desc="HTML"),
+                            Part(
+                                "text",
+                                "png",
+                                "logo.png",
+                                cid="logo.png",
+                                desc="Logo",
+                            ),
+                        ],
+                        desc="Related",
+                    ),
+                ],
+                desc="Alternative",
+            )
+
+        @pytest.mark.treewalk
+        def test_MIMETreeDFWalker_depth_first_walk(
+            self, mime_tree_related_to_alternative
+        ):
             mimetree = MIMETreeDFWalker()
 
             items = []
 
             mimetree = MIMETreeDFWalker()
 
             items = []
 
-            def visitor_fn(item, stack, debugprint):
-                items.append((item, len(stack)))
+            def visitor_fn(item, ancestry, descendents, debugprint):
+                items.append((item, len(ancestry), len(descendents)))
 
 
-            mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
+            mimetree.walk(
+                mime_tree_related_to_alternative, visitor_fn=visitor_fn
+            )
             assert len(items) == 5
             assert items[0][0].subtype == "plain"
             assert items[0][1] == 2
             assert len(items) == 5
             assert items[0][0].subtype == "plain"
             assert items[0][1] == 2
+            assert items[0][2] == 0
             assert items[1][0].subtype == "html"
             assert items[1][1] == 2
             assert items[1][0].subtype == "html"
             assert items[1][1] == 2
+            assert items[1][2] == 0
             assert items[2][0].subtype == "alternative"
             assert items[2][1] == 1
             assert items[2][0].subtype == "alternative"
             assert items[2][1] == 1
+            assert items[2][2] == 2
             assert items[3][0].subtype == "png"
             assert items[3][1] == 1
             assert items[3][0].subtype == "png"
             assert items[3][1] == 1
+            assert items[3][2] == 2
             assert items[4][0].subtype == "relative"
             assert items[4][1] == 0
             assert items[4][0].subtype == "relative"
             assert items[4][1] == 0
+            assert items[4][2] == 4
 
 
-        def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
+        @pytest.mark.treewalk
+        def test_MIMETreeDFWalker_list_to_mixed(self, const1):
             mimetree = MIMETreeDFWalker()
             items = []
 
             mimetree = MIMETreeDFWalker()
             items = []
 
-            def visitor_fn(item, stack, debugprint):
+            def visitor_fn(item, ancestry, descendents, debugprint):
                 items.append(item)
 
                 items.append(item)
 
-            mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
+            p = Part("text", "plain", const1)
+            mimetree.walk([p], visitor_fn=visitor_fn)
+            assert items[-1].subtype == "plain"
+            mimetree.walk([p, p], visitor_fn=visitor_fn)
             assert items[-1].subtype == "mixed"
 
             assert items[-1].subtype == "mixed"
 
+        @pytest.mark.treewalk
         def test_MIMETreeDFWalker_visitor_in_constructor(
         def test_MIMETreeDFWalker_visitor_in_constructor(
-            self, basic_mime_tree
+            self, mime_tree_related_to_alternative
         ):
             items = []
 
         ):
             items = []
 
-            def visitor_fn(item, stack, debugprint):
+            def visitor_fn(item, ancestry, descendents, debugprint):
                 items.append(item)
 
             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
                 items.append(item)
 
             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
-            mimetree.walk(basic_mime_tree)
+            mimetree.walk(mime_tree_related_to_alternative)
             assert len(items) == 5
 
             assert len(items) == 5
 
-        def test_do_setup_no_extensions(self, const1, capsys):
-            "Assert basics about the setup command output"
-            do_setup(temppath=const1, out_f=sys.stdout)
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            assert lines[2].endswith(f'{const1}"')
-            assert lines[4].endswith(const1)
-            assert "first-entry" in lines[-1]
-            assert "edit-file" in lines[-1]
-
-        def test_do_setup_extensions(self, const1, const2, capsys):
-            "Assert that extensions are passed to editor"
-            do_setup(
-                temppath=const1, extensions=[const2, const1], out_f=sys.stdout
-            )
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            # assert comma-separated list of extensions passed
-            assert lines[2].endswith(f'{const2},{const1}"')
-            assert lines[4].endswith(const1)
-
         @pytest.fixture
         def string_io(self, const1, text=None):
             return StringIO(text or const1)
 
         @pytest.fixture
         def string_io(self, const1, text=None):
             return StringIO(text or const1)
 
+        @pytest.mark.massage
         def test_do_massage_basic(self, const1, string_io, capsys):
         def test_do_massage_basic(self, const1, string_io, capsys):
-            def converter(drafttext, draftpath, extensions):
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
                 return Part("text", "plain", draftpath, orig=True)
 
             do_massage(
                 return Part("text", "plain", draftpath, orig=True)
 
             do_massage(
@@ -702,43 +1108,63 @@ try:
             assert '="$my_edit_headers"' in lines.pop(0)
             assert "unset my_editor" == lines.pop(0)
             assert "unset my_edit_headers" == lines.pop(0)
             assert '="$my_edit_headers"' in lines.pop(0)
             assert "unset my_editor" == lines.pop(0)
             assert "unset my_edit_headers" == lines.pop(0)
+            assert "send-message" in lines.pop(0)
             assert "update-encoding" in lines.pop(0)
             assert "update-encoding" in lines.pop(0)
+            assert "first-entry" in lines.pop(0)
             assert "source 'rm -f " in lines.pop(0)
             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 
             assert "source 'rm -f " in lines.pop(0)
             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 
+        @pytest.mark.massage
         def test_do_massage_fulltree(
         def test_do_massage_fulltree(
-            self, string_io, const1, basic_mime_tree, capsys
+            self, string_io, const1, mime_tree_related_to_alternative, capsys
         ):
         ):
-            def converter(drafttext, draftpath, extensions):
-                return basic_mime_tree
-
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
+                return mime_tree_related_to_alternative
+
+            max_attachments = 5
             do_massage(
                 draft_f=string_io,
                 draftpath=const1,
                 cmd_f=sys.stdout,
             do_massage(
                 draft_f=string_io,
                 draftpath=const1,
                 cmd_f=sys.stdout,
+                max_other_attachments=max_attachments,
                 converter=converter,
             )
 
             captured = capsys.readouterr()
                 converter=converter,
             )
 
             captured = capsys.readouterr()
-            lines = captured.out.splitlines()[4:]
-            assert "Related" in lines.pop(0)
-            assert "group-related" in lines.pop(0)
-            assert "tag-entry" in lines.pop(0)
-            assert "Logo" in lines.pop(0)
-            assert "content-id" in lines.pop(0)
-            assert "toggle-unlink" in lines.pop(0)
-            assert "logo.png" in lines.pop(0)
-            assert "tag-entry" in lines.pop(0)
-            assert "Alternative" in lines.pop(0)
-            assert "group-alternatives" in lines.pop(0)
-            assert "tag-entry" in lines.pop(0)
-            assert "HTML" in lines.pop(0)
-            assert "toggle-unlink" in lines.pop(0)
-            assert "part.html" in lines.pop(0)
-            assert "tag-entry" in lines.pop(0)
-            assert "Plain" in lines.pop(0)
-            assert "update-encoding" in lines.pop(0)
-            assert len(lines) == 2
+            lines = captured.out.splitlines()[4:-2]
+            assert "first-entry" in lines.pop()
+            assert "update-encoding" in lines.pop()
+            assert "Plain" in lines.pop()
+            assert "part.html" in lines.pop()
+            assert "toggle-unlink" in lines.pop()
+            for i in range(max_attachments):
+                assert "move-up" in lines.pop()
+            assert "move-down" in lines.pop()
+            assert "HTML" in lines.pop()
+            assert "jump>1" in lines.pop()
+            assert "jump>2" in lines.pop()
+            assert "group-alternatives" in lines.pop()
+            assert "Alternative" in lines.pop()
+            assert "logo.png" in lines.pop()
+            assert "toggle-unlink" in lines.pop()
+            assert "content-id" in lines.pop()
+            for i in range(max_attachments):
+                assert "move-up" in lines.pop()
+            assert "move-down" in lines.pop()
+            assert "Logo" in lines.pop()
+            assert "jump>1" in lines.pop()
+            assert "jump>4" in lines.pop()
+            assert "group-related" in lines.pop()
+            assert "Related" in lines.pop()
+            assert "send-message" in lines.pop()
+            assert len(lines) == 0
 
         @pytest.fixture
         def fake_filewriter(self):
 
         @pytest.fixture
         def fake_filewriter(self):
@@ -758,9 +1184,8 @@ try:
         def markdown_non_converter(self, const1, const2):
             return lambda s, text: f"{const1}{text}{const2}"
 
         def markdown_non_converter(self, const1, const2):
             return lambda s, text: f"{const1}{text}{const2}"
 
-        def test_converter_tree_basic(
-            self, const1, const2, fake_filewriter, markdown_non_converter
-        ):
+        @pytest.mark.converter
+        def test_converter_tree_basic(self, const1, const2, fake_filewriter):
             path = pathlib.Path(const2)
             tree = convert_markdown_to_html(
                 const1, path, filewriter_fn=fake_filewriter
             path = pathlib.Path(const2)
             tree = convert_markdown_to_html(
                 const1, path, filewriter_fn=fake_filewriter
@@ -791,11 +1216,11 @@ try:
                 )
 
             assert (path, const1) == fake_filewriter.pop(0)
                 )
 
             assert (path, const1) == fake_filewriter.pop(0)
-            assert (
-                path.with_suffix(".html"),
-                markdown_non_converter(None, const1),
-            ) == fake_filewriter.pop(0)
+            written = fake_filewriter.pop(0)
+            assert path.with_suffix(".html") == written[0]
+            assert const1 in written[1]
 
 
+        @pytest.mark.imgproc
         def test_markdown_inline_image_processor(self):
             imgpath1 = "file:/path/to/image.png"
             imgpath2 = "file:///path/to/image.png?url=params"
         def test_markdown_inline_image_processor(self):
             imgpath1 = "file:/path/to/image.png"
             imgpath2 = "file:///path/to/image.png?url=params"
@@ -804,7 +1229,7 @@ try:
                        ![image inlined
                          with newline]({imgpath2})
                        ![image local path]({imgpath3})"""
                        ![image inlined
                          with newline]({imgpath2})
                        ![image local path]({imgpath3})"""
-            text, html, images = markdown_with_inline_image_support(text)
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
 
             # local paths have been normalised to URLs:
             imgpath3 = f"file://{imgpath3}"
 
             # local paths have been normalised to URLs:
             imgpath3 = f"file://{imgpath3}"
@@ -819,44 +1244,59 @@ try:
             assert images[imgpath1].cid != images[imgpath3].cid
             assert images[imgpath2].cid != images[imgpath3].cid
 
             assert images[imgpath1].cid != images[imgpath3].cid
             assert images[imgpath2].cid != images[imgpath3].cid
 
+        @pytest.mark.imgproc
         def test_markdown_inline_image_processor_title_to_desc(self, const1):
             imgpath = "file:///path/to/image.png"
             text = f'![inline local image]({imgpath} "{const1}")'
         def test_markdown_inline_image_processor_title_to_desc(self, const1):
             imgpath = "file:///path/to/image.png"
             text = f'![inline local image]({imgpath} "{const1}")'
-            text, html, images = markdown_with_inline_image_support(text)
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
             assert images[imgpath].desc == const1
 
             assert images[imgpath].desc == const1
 
+        @pytest.mark.imgproc
         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
             imgpath = "file:///path/to/image.png"
             text = f"![{const1}]({imgpath})"
         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
             imgpath = "file:///path/to/image.png"
             text = f"![{const1}]({imgpath})"
-            text, html, images = markdown_with_inline_image_support(text)
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
             assert images[imgpath].desc == const1
 
             assert images[imgpath].desc == const1
 
+        @pytest.mark.imgproc
         def test_markdown_inline_image_processor_title_over_alt_desc(
             self, const1, const2
         ):
             imgpath = "file:///path/to/image.png"
             text = f'![{const1}]({imgpath} "{const2}")'
         def test_markdown_inline_image_processor_title_over_alt_desc(
             self, const1, const2
         ):
             imgpath = "file:///path/to/image.png"
             text = f'![{const1}]({imgpath} "{const2}")'
-            text, html, images = markdown_with_inline_image_support(text)
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
             assert images[imgpath].desc == const2
 
             assert images[imgpath].desc == const2
 
+        @pytest.mark.imgproc
         def test_markdown_inline_image_not_external(self):
             imgpath = "https://path/to/image.png"
             text = f"![inline image]({imgpath})"
         def test_markdown_inline_image_not_external(self):
             imgpath = "https://path/to/image.png"
             text = f"![inline image]({imgpath})"
-            text, html, images = markdown_with_inline_image_support(text)
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
 
             assert 'src="cid:' not in html
             assert "](cid:" not in text
             assert len(images) == 0
 
 
             assert 'src="cid:' not in html
             assert "](cid:" not in text
             assert len(images) == 0
 
+        @pytest.mark.imgproc
         def test_markdown_inline_image_local_file(self):
             imgpath = "/path/to/image.png"
             text = f"![inline image]({imgpath})"
         def test_markdown_inline_image_local_file(self):
             imgpath = "/path/to/image.png"
             text = f"![inline image]({imgpath})"
-            text, html, images = markdown_with_inline_image_support(text)
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
 
             for k, v in images.items():
                 assert k == f"file://{imgpath}"
                 break
 
 
             for k, v in images.items():
                 assert k == f"file://{imgpath}"
                 break
 
+        @pytest.mark.imgproc
+        def test_markdown_inline_image_expanduser(self):
+            imgpath = pathlib.Path("~/image.png")
+            text = f"![inline image]({imgpath})"
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
+
+            for k, v in images.items():
+                assert k == f"file://{imgpath.expanduser()}"
+                break
+
         @pytest.fixture
         def test_png(self):
             return (
         @pytest.fixture
         def test_png(self):
             return (
@@ -864,30 +1304,500 @@ try:
                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
             )
 
                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
             )
 
-
+        @pytest.mark.imgproc
         def test_markdown_inline_image_processor_base64(self, test_png):
             text = f"![1px white inlined]({test_png})"
         def test_markdown_inline_image_processor_base64(self, test_png):
             text = f"![1px white inlined]({test_png})"
-            text, html, images = markdown_with_inline_image_support(text)
+            text, html, images, mdwn = markdown_with_inline_image_support(text)
 
             assert 'src="cid:' in html
             assert "](cid:" in text
             assert len(images) == 1
             assert test_png in images
 
 
             assert 'src="cid:' in html
             assert "](cid:" in text
             assert len(images) == 1
             assert test_png in images
 
+        @pytest.mark.converter
         def test_converter_tree_inline_image_base64(
             self, test_png, const1, fake_filewriter
         ):
             text = f"![inline base64 image]({test_png})"
             path = pathlib.Path(const1)
             tree = convert_markdown_to_html(
         def test_converter_tree_inline_image_base64(
             self, test_png, const1, fake_filewriter
         ):
             text = f"![inline base64 image]({test_png})"
             path = pathlib.Path(const1)
             tree = convert_markdown_to_html(
-                text, path, filewriter_fn=fake_filewriter
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=False,
+            )
+            assert tree.subtype == "relative"
+            assert tree.children[0].subtype == "alternative"
+            assert tree.children[1].subtype == "png"
+            written = fake_filewriter.pop()
+            assert tree.children[1].path == written[0]
+            assert written[1] == request.urlopen(test_png).read()
+
+        @pytest.mark.converter
+        def test_converter_tree_inline_image_base64_related_to_html(
+            self, test_png, const1, fake_filewriter
+        ):
+            text = f"![inline base64 image]({test_png})"
+            path = pathlib.Path(const1)
+            tree = convert_markdown_to_html(
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=True,
+            )
+            assert tree.subtype == "alternative"
+            assert tree.children[1].subtype == "relative"
+            assert tree.children[1].children[1].subtype == "png"
+            written = fake_filewriter.pop()
+            assert tree.children[1].children[1].path == written[0]
+            assert written[1] == request.urlopen(test_png).read()
+
+        @pytest.mark.converter
+        def test_converter_tree_inline_image_cid(
+            self, const1, fake_filewriter
+        ):
+            text = f"![inline base64 image](cid:{const1})"
+            path = pathlib.Path(const1)
+            tree = convert_markdown_to_html(
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=False,
+            )
+            assert len(tree.children) == 2
+            assert tree.children[0].cid != const1
+            assert tree.children[0].type != "image"
+            assert tree.children[1].cid != const1
+            assert tree.children[1].type != "image"
+
+        @pytest.mark.imgcoll
+        def test_inline_image_collection(
+            self, test_png, const1, const2, fake_filewriter
+        ):
+            test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
+            relparts = collect_inline_images(
+                test_images, filewriter_fn=fake_filewriter
+            )
+
+            written = fake_filewriter.pop()
+            assert b"PNG" in written[1]
+
+            assert relparts[0].subtype == "png"
+            assert relparts[0].path == written[0]
+            assert relparts[0].cid == const1
+            assert relparts[0].desc.endswith(const2)
+
+        if _PYNLINER:
+
+            @pytest.mark.styling
+            def test_apply_stylesheet(self):
+                html = "<p>Hello, world!</p>"
+                css = "p { color:red }"
+                out = apply_styling(html, css)
+                assert 'p style="color' in out
+
+            @pytest.mark.styling
+            def test_massage_styling_to_converter(self, string_io, const1):
+                css = "p { color:red }"
+                css_f = StringIO(css)
+                out_f = StringIO()
+                css_applied = []
+
+                def converter(
+                    drafttext,
+                    draftpath,
+                    css,
+                    related_to_html_only,
+                    extensions,
+                    tempdir,
+                ):
+                    css_applied.append(css)
+                    return Part("text", "plain", draftpath, orig=True)
+
+                do_massage(
+                    draft_f=string_io,
+                    draftpath=const1,
+                    cmd_f=out_f,
+                    css_f=css_f,
+                    converter=converter,
+                )
+                assert css_applied[0] == css
+
+            @pytest.mark.converter
+            def test_converter_apply_styles(
+                self, const1, fake_filewriter, monkeypatch
+            ):
+                path = pathlib.Path(const1)
+                text = "Hello, world!"
+                css = "p { color:red }"
+                with monkeypatch.context() as m:
+                    m.setattr(
+                        markdown.Markdown,
+                        "convert",
+                        lambda s, t: f"<p>{t}</p>",
+                    )
+                    convert_markdown_to_html(
+                        text, path, css=css, filewriter_fn=fake_filewriter
+                    )
+                assert "color: red" in fake_filewriter.pop()[1]
+
+        if _PYGMENTS_CSS:
+
+            @pytest.mark.styling
+            def test_apply_stylesheet_pygments(self):
+                html = (
+                    f'<div class="{_CODEHILITE_CLASS}">'
+                    "<pre>def foo():\n    return</pre></div>"
+                )
+                out = apply_styling(html, _PYGMENTS_CSS)
+                assert f'{_CODEHILITE_CLASS}" style="' in out
+
+        @pytest.mark.massage
+        def test_mime_tree_relative_within_alternative(
+            self, string_io, const1, capsys, mime_tree_related_to_html
+        ):
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
+                return mime_tree_related_to_html
+
+            do_massage(
+                draft_f=string_io,
+                draftpath=const1,
+                cmd_f=sys.stdout,
+                converter=converter,
+            )
+
+            captured = capsys.readouterr()
+            lines = captured.out.splitlines()[4:-2]
+            assert "first-entry" in lines.pop()
+            assert "update-encoding" in lines.pop()
+            assert "Plain" in lines.pop()
+            assert "part.html" in lines.pop()
+            assert "toggle-unlink" in lines.pop()
+            assert "move-up" in lines.pop()
+            while True:
+                top = lines.pop()
+                if "move-up" not in top:
+                    break
+            assert "move-down" in top
+            assert "HTML" in lines.pop()
+            assert "logo.png" in lines.pop()
+            assert "toggle-unlink" in lines.pop()
+            assert "content-id" in lines.pop()
+            assert "move-up" in lines.pop()
+            while True:
+                top = lines.pop()
+                if "move-up" not in top:
+                    break
+            assert "move-down" in top
+            assert "move-down" in lines.pop()
+            assert "Logo" in lines.pop()
+            assert "jump>2" in lines.pop()
+            assert "jump>3" in lines.pop()
+            assert "group-related" in lines.pop()
+            assert "Related" in lines.pop()
+            assert "jump>1" in lines.pop()
+            assert "jump>2" in lines.pop()
+            assert "group-alternative" in lines.pop()
+            assert "Alternative" in lines.pop()
+            assert "send-message" in lines.pop()
+            assert len(lines) == 0
+
+        @pytest.mark.massage
+        def test_mime_tree_nested_trees_does_not_break_positioning(
+            self, string_io, const1, capsys
+        ):
+            def converter(
+                drafttext,
+                draftpath,
+                css,
+                related_to_html_only,
+                extensions,
+                tempdir,
+            ):
+                return Multipart(
+                    "relative",
+                    children=[
+                        Multipart(
+                            "alternative",
+                            children=[
+                                Part(
+                                    "text",
+                                    "plain",
+                                    "part.txt",
+                                    desc="Plain",
+                                    orig=True,
+                                ),
+                                Multipart(
+                                    "alternative",
+                                    children=[
+                                        Part(
+                                            "text",
+                                            "plain",
+                                            "part.txt",
+                                            desc="Nested plain",
+                                        ),
+                                        Part(
+                                            "text",
+                                            "html",
+                                            "part.html",
+                                            desc="Nested HTML",
+                                        ),
+                                    ],
+                                    desc="Nested alternative",
+                                ),
+                            ],
+                            desc="Alternative",
+                        ),
+                        Part(
+                            "text",
+                            "png",
+                            "logo.png",
+                            cid="logo.png",
+                            desc="Logo",
+                        ),
+                    ],
+                    desc="Related",
+                )
+
+            do_massage(
+                draft_f=string_io,
+                draftpath=const1,
+                cmd_f=sys.stdout,
+                converter=converter,
+            )
+
+            captured = capsys.readouterr()
+            lines = captured.out.splitlines()
+            while "logo.png" not in lines.pop():
+                pass
+            lines.pop()
+            assert "content-id" in lines.pop()
+            assert "move-up" in lines.pop()
+            while True:
+                top = lines.pop()
+                if "move-up" not in top:
+                    break
+            assert "move-down" in top
+            # Due to the nested trees, the number of descendents of the sibling
+            # actually needs to be considered, not just the nieces. So to move
+            # from position 1 to position 6, it only needs one <move-down>
+            # because that jumps over the entire sibling tree. Thus what
+            # follows next must not be another <move-down>
+            assert "Logo" in lines.pop()
+
+        @pytest.mark.sig
+        def test_signature_extraction_no_signature(self, const1):
+            assert (const1, None, None) == extract_signature(const1)
+
+        @pytest.mark.sig
+        def test_signature_extraction_just_text(self, const1, const2):
+            origtext, textsig, htmlsig = extract_signature(
+                f"{const1}{EMAIL_SIG_SEP}{const2}"
+            )
+            assert origtext == const1
+            assert textsig == const2
+            assert htmlsig is None
+
+        @pytest.mark.sig
+        def test_signature_extraction_html(self, const1, const2):
+            path = pathlib.Path("somepath")
+            sigconst = "HTML signature from {path} but as a string"
+
+            def filereader_fn(path):
+                return (
+                    f'<div id="signature">{sigconst.format(path=path)}</div>'
+                )
+
+            origtext, textsig, htmlsig = extract_signature(
+                f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {path}\n{const2}",
+                filereader_fn=filereader_fn,
+            )
+            assert origtext == const1
+            assert textsig == const2
+            assert htmlsig == sigconst.format(path=path)
+
+        @pytest.mark.sig
+        def test_signature_extraction_file_not_found(self, const1):
+            path = pathlib.Path("/does/not/exist")
+            with pytest.raises(FileNotFoundError):
+                origtext, textsig, htmlsig = extract_signature(
+                    f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{path}\n{const1}"
+                )
+
+        @pytest.mark.imgproc
+        def test_image_registry(self, const1):
+            reg = ImageRegistry()
+            cid = reg.register(const1)
+            assert "@" in cid
+            assert not cid.startswith("<")
+            assert not cid.endswith(">")
+            assert const1 in reg
+
+        @pytest.mark.imgproc
+        def test_image_registry_file_uri(self, const1):
+            reg = ImageRegistry()
+            reg.register("/some/path")
+            for path in reg:
+                assert path.startswith("file://")
+                break
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling(
+            self, const1, fake_filewriter, monkeypatch
+        ):
+            path = pathlib.Path(const1)
+
+            mailparts = (
+                "This is the mail body\n",
+                f"{EMAIL_SIG_SEP}",
+                "This is a plain-text signature only",
+            )
+
+            def filereader_fn(path):
+                return ""
+
+            with monkeypatch.context() as m:
+                m.setattr(markdown.Markdown, "convert", lambda s, t: t)
+                convert_markdown_to_html(
+                    "".join(mailparts),
+                    path,
+                    filewriter_fn=fake_filewriter,
+                    filereader_fn=filereader_fn,
+                )
+
+            soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+            body = soup.body.contents
+
+            assert mailparts[0] in body.pop(0)
+
+            sig = soup.select_one("#signature")
+            assert sig == body.pop(0)
+
+            sep = sig.select_one("span.sig_separator")
+            assert sep == sig.contents[0]
+            assert f"\n{sep.text}\n" == EMAIL_SIG_SEP
+
+            assert mailparts[2] in sig.contents[1]
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling_htmlsig(
+            self, const1, fake_filewriter, monkeypatch
+        ):
+            path = pathlib.Path(const1)
+
+            mailparts = (
+                "This is the mail body",
+                f"{EMAIL_SIG_SEP}",
+                f"{HTML_SIG_MARKER}{path}\n",
+                "This is the plain-text version",
+            )
+
+            htmlsig = "HTML Signature from {path}"
+
+            def filereader_fn(path):
+                return f'<div id="signature">{htmlsig.format(path=path)}</div>'
+
+            def mdwn_fn(t):
+                return t.upper()
+
+            with monkeypatch.context() as m:
+                m.setattr(
+                    markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
+                )
+                convert_markdown_to_html(
+                    "".join(mailparts),
+                    path,
+                    filewriter_fn=fake_filewriter,
+                    filereader_fn=filereader_fn,
+                )
+
+            soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+            sig = soup.select_one("#signature")
+            sig.span.extract()
+
+            assert HTML_SIG_MARKER not in sig.text
+            assert htmlsig.format(path=path) == sig.text.strip()
+
+            plaintext = fake_filewriter.pop()[1]
+            assert plaintext.endswith(EMAIL_SIG_SEP + mailparts[-1])
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling_htmlsig_with_image(
+            self, const1, fake_filewriter, monkeypatch, test_png
+        ):
+            path = pathlib.Path(const1)
+
+            mailparts = (
+                "This is the mail body",
+                f"{EMAIL_SIG_SEP}",
+                f"{HTML_SIG_MARKER}{path}\n",
+                "This is the plain-text version",
+            )
+
+            htmlsig = (
+                "HTML Signature from {path} with image\n"
+                f'<img src="{test_png}">\n'
+            )
+
+            def filereader_fn(path):
+                return f'<div id="signature">{htmlsig.format(path=path)}</div>'
+
+            def mdwn_fn(t):
+                return t.upper()
+
+            with monkeypatch.context() as m:
+                m.setattr(
+                    markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
+                )
+                convert_markdown_to_html(
+                    "".join(mailparts),
+                    path,
+                    filewriter_fn=fake_filewriter,
+                    filereader_fn=filereader_fn,
+                )
+
+            assert fake_filewriter.pop()[0].suffix == ".png"
+
+            soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+            assert soup.img.attrs["src"].startswith("cid:")
+
+        @pytest.mark.converter
+        @pytest.mark.sig
+        def test_converter_signature_handling_textsig_with_image(
+            self, const1, fake_filewriter, test_png
+        ):
+            mailparts = (
+                "This is the mail body",
+                f"{EMAIL_SIG_SEP}",
+                "This is the plain-text version with image\n",
+                f"![Inline]({test_png})",
+
+            )
+            tree = convert_markdown_to_html
+                "".join(mailparts),
+                pathlib.Path(const1),
+                filewriter_fn=fake_filewriter,
             )
 
             assert tree.subtype == "relative"
             )
 
             assert tree.subtype == "relative"
+            assert tree.children[0].subtype == "alternative"
             assert tree.children[1].subtype == "png"
             written = fake_filewriter.pop()
             assert tree.children[1].path == written[0]
             assert written[1] == request.urlopen(test_png).read()
 
             assert tree.children[1].subtype == "png"
             written = fake_filewriter.pop()
             assert tree.children[1].path == written[0]
             assert written[1] == request.urlopen(test_png).read()
 
+        def test_converter_attribution_to_admonition(self, fake_filewriter):
+
+
 except ImportError:
     pass
 except ImportError:
     pass