.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=pathlib.Path,
  73             help="CSS file to merge with the final HTML",
  74         )
  75     else:
  76         parser.set_defaults(css_file=None)
  77
  78     parser.add_argument(
  79         "--related-to-html-only",
  80         action="store_true",
  81         help="Make related content be sibling to HTML parts only",
  82     )
  83
  84     def positive_integer(value):
  85         try:
  86             if int(value) > 0:
  87                 return int(value)
  88
  89         except ValueError:
  90             pass
  91
  92         raise ValueError(f"Must be a positive integer")
  93
  94     parser.add_argument(
  95         "--max-number-other-attachments",
  96         type=positive_integer,
  97         help="Make related content be sibling to HTML parts only",
  98     )
  99
 100     parser.add_argument(
 101         "--only-build",
 102         action="store_true",
 103         help="Only build, don't send the message",
 104     )
 105
 106     parser.add_argument(
 107         "--tempdir",
 108         type=pathlib.Path,
 109         help="Specify temporary directory to use for attachments",
 110     )
 111
 112     parser.add_argument(
 113         "--debug-commands",
 114         action="store_true",
 115         help="Turn on debug logging of commands generated to stderr",
 116     )
 117
 118     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
 119     massage_p = subp.add_parser(
 120         "massage", help="Massaging phase (internal use)"
 121     )
 122
 123     massage_p.add_argument(
 124         "--write-commands-to",
 125         "-o",
 126         metavar="PATH",
 127         dest="cmdpath",
 128         type=pathlib.Path,
 129         required=True,
 130         help="Temporary file path to write commands to",
 131     )
 132
 133     massage_p.add_argument(
 134         "--debug-walk",
 135         action="store_true",
 136         help="Turn on debugging to stderr of the MIME tree walk",
 137     )
 138
 139     massage_p.add_argument(
 140         "MAILDRAFT",
 141         nargs="?",
 142         type=pathlib.Path,
 143         help="If provided, the script is invoked as editor on the mail draft",
 144     )
 145
 146     return parser.parse_args(*args, **kwargs)
 147
 148
 149 # [ MARKDOWN WRAPPING ] #######################################################
 150
 151
 152 InlineImageInfo = namedtuple(
 153     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 154 )
 155
 156
 157 class InlineImageExtension(Extension):
 158     class RelatedImageInlineProcessor(ImageInlineProcessor):
 159         def __init__(self, re, md, ext):
 160             super().__init__(re, md)
 161             self._ext = ext
 162
 163         def handleMatch(self, m, data):
 164             el, start, end = super().handleMatch(m, data)
 165             if "src" in el.attrib:
 166                 src = el.attrib["src"]
 167                 if "://" not in src or src.startswith("file://"):
 168                     # We only inline local content
 169                     cid = self._ext.get_cid_for_image(el.attrib)
 170                     el.attrib["src"] = f"cid:{cid}"
 171             return el, start, end
 172
 173     def __init__(self):
 174         super().__init__()
 175         self._images = OrderedDict()
 176
 177     def extendMarkdown(self, md):
 178         md.registerExtension(self)
 179         inline_image_proc = self.RelatedImageInlineProcessor(
 180             IMAGE_LINK_RE, md, self
 181         )
 182         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 183
 184     def get_cid_for_image(self, attrib):
 185         msgid = make_msgid()[1:-1]
 186         path = attrib["src"]
 187         if path.startswith("/"):
 188             path = f"file://{path}"
 189         self._images[path] = InlineImageInfo(
 190             msgid, attrib.get("title", attrib.get("alt"))
 191         )
 192         return msgid
 193
 194     def get_images(self):
 195         return self._images
 196
 197
 198 def markdown_with_inline_image_support(
 199     text, *, extensions=None, extension_configs=None
 200 ):
 201     inline_image_handler = InlineImageExtension()
 202     extensions = extensions or []
 203     extensions.append(inline_image_handler)
 204     mdwn = markdown.Markdown(
 205         extensions=extensions, extension_configs=extension_configs
 206     )
 207     htmltext = mdwn.convert(text)
 208
 209     images = inline_image_handler.get_images()
 210
 211     def replace_image_with_cid(matchobj):
 212         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 213             if m in images:
 214                 return f"(cid:{images[m].cid}"
 215         return matchobj.group(0)
 216
 217     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 218     return text, htmltext, images
 219
 220
 221 # [ CSS STYLING ] #############################################################
 222
 223 try:
 224     import pynliner
 225
 226     _PYNLINER = True
 227
 228 except ImportError:
 229     _PYNLINER = False
 230
 231 try:
 232     from pygments.formatters import get_formatter_by_name
 233
 234     _CODEHILITE_CLASS = "codehilite"
 235
 236     _PYGMENTS_CSS = get_formatter_by_name(
 237         "html", style="default"
 238     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 239
 240 except ImportError:
 241     _PYGMENTS_CSS = None
 242
 243
 244 def apply_styling(html, css):
 245     return (
 246         pynliner.Pynliner()
 247         .from_string(html)
 248         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 249         .run()
 250     )
 251
 252
 253 # [ PARTS GENERATION ] ########################################################
 254
 255
 256 class Part(
 257     namedtuple(
 258         "Part",
 259         ["type", "subtype", "path", "desc", "cid", "orig"],
 260         defaults=[None, None, False],
 261     )
 262 ):
 263     def __str__(self):
 264         ret = f"<{self.type}/{self.subtype}>"
 265         if self.cid:
 266             ret = f"{ret} cid:{self.cid}"
 267         if self.orig:
 268             ret = f"{ret} ORIGINAL"
 269         return ret
 270
 271
 272 class Multipart(
 273     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 274 ):
 275     def __str__(self):
 276         return f"<multipart/{self.subtype}> children={len(self.children)}"
 277
 278     def __hash__(self):
 279         return hash(str(self.subtype) + "".join(str(self.children)))
 280
 281
 282 def filewriter_fn(path, content, mode="w", **kwargs):
 283     with open(path, mode, **kwargs) as out_f:
 284         out_f.write(content)
 285
 286
 287 def collect_inline_images(
 288     images, *, tempdir=None, filewriter_fn=filewriter_fn
 289 ):
 290     relparts = []
 291     for path, info in images.items():
 292         if path.startswith("cid:"):
 293             continue
 294
 295         data = request.urlopen(path)
 296
 297         mimetype = data.headers["Content-Type"]
 298         ext = mimetypes.guess_extension(mimetype)
 299         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 300         path = pathlib.Path(tempfilename[1])
 301
 302         filewriter_fn(path, data.read(), "w+b")
 303
 304         relparts.append(
 305             Part(
 306                 *mimetype.split("/"),
 307                 path,
 308                 cid=info.cid,
 309                 desc=f"Image: {info.desc}",
 310             )
 311         )
 312
 313     return relparts
 314
 315
 316 def convert_markdown_to_html(
 317     origtext,
 318     draftpath,
 319     *,
 320     related_to_html_only=False,
 321     cssfile=None,
 322     filewriter_fn=filewriter_fn,
 323     tempdir=None,
 324     extensions=None,
 325     extension_configs=None,
 326 ):
 327     # TODO extension_configs need to be handled differently
 328     extension_configs = extension_configs or {}
 329     extension_configs.setdefault("pymdownx.highlight", {})
 330     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 331
 332     origtext, htmltext, images = markdown_with_inline_image_support(
 333         origtext, extensions=extensions, extension_configs=extension_configs
 334     )
 335
 336     filewriter_fn(draftpath, origtext, encoding="utf-8")
 337     textpart = Part(
 338         "text", "plain", draftpath, "Plain-text version", orig=True
 339     )
 340
 341     htmltext = apply_styling(htmltext, cssfile)
 342
 343     htmlpath = draftpath.with_suffix(".html")
 344     filewriter_fn(
 345         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 346     )
 347     htmlpart = Part("text", "html", htmlpath, "HTML version")
 348
 349     imgparts = collect_inline_images(
 350         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 351     )
 352
 353     if related_to_html_only:
 354         # If there are inline image part, they will be contained within a
 355         # multipart/related part along with the HTML part only
 356         if imgparts:
 357             # replace htmlpart with a multipart/related container of the HTML
 358             # parts and the images
 359             htmlpart = Multipart(
 360                 "relative", [htmlpart] + imgparts, "Group of related content"
 361             )
 362
 363         return Multipart(
 364             "alternative", [textpart, htmlpart], "Group of alternative content"
 365         )
 366
 367     else:
 368         # If there are inline image part, they will be siblings to the
 369         # multipart/alternative tree within a multipart/related part
 370         altpart = Multipart(
 371             "alternative", [textpart, htmlpart], "Group of alternative content"
 372         )
 373         if imgparts:
 374             return Multipart(
 375                 "relative", [altpart] + imgparts, "Group of related content"
 376             )
 377         else:
 378             return altpart
 379
 380
 381 class MIMETreeDFWalker:
 382     def __init__(self, *, visitor_fn=None, debug=False):
 383         self._visitor_fn = visitor_fn or self._echovisit
 384         self._debug = debug
 385
 386     def _echovisit(self, node, ancestry, debugprint):
 387         debugprint(f"node={node} ancestry={ancestry}")
 388
 389     def walk(self, root, *, visitor_fn=None):
 390         """
 391         Recursive function to implement a depth-dirst walk of the MIME-tree
 392         rooted at `root`.
 393         """
 394         if isinstance(root, list):
 395             if len(root) > 1:
 396                 root = Multipart("mixed", children=root)
 397             else:
 398                 root = root[0]
 399
 400         self._walk(
 401             root,
 402             ancestry=[],
 403             descendents=[],
 404             visitor_fn=visitor_fn or self._visitor_fn,
 405         )
 406
 407     def _walk(self, node, *, ancestry, descendents, visitor_fn):
 408         # Let's start by enumerating the parts at the current level. At the
 409         # root level, ancestry will be the empty list, and we expect a
 410         # multipart/* container at this level. Later, e.g. within a
 411         # mutlipart/alternative container, the subtree will just be the
 412         # alternative parts, while the top of the ancestry will be the
 413         # multipart/alternative container, which we will process after the
 414         # following loop.
 415
 416         lead = f"{'│ '*len(ancestry)}"
 417         if isinstance(node, Multipart):
 418             self.debugprint(
 419                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
 420             )
 421
 422             # Depth-first, so push the current container onto the ancestry
 423             # stack, then descend …
 424             ancestry.append(node)
 425             self.debugprint(lead + "│ " * 2)
 426             for child in node.children:
 427                 self._walk(
 428                     child,
 429                     ancestry=ancestry,
 430                     descendents=descendents,
 431                     visitor_fn=visitor_fn,
 432                 )
 433             assert ancestry.pop() == node
 434             sibling_descendents = descendents
 435             descendents.extend(node.children)
 436
 437         else:
 438             self.debugprint(f"{lead}├{node}")
 439             sibling_descendents = descendents
 440
 441         if False and ancestry:
 442             self.debugprint(lead[:-1] + " │")
 443
 444         if visitor_fn:
 445             visitor_fn(
 446                 node, ancestry, sibling_descendents, debugprint=self.debugprint
 447             )
 448
 449     def debugprint(self, s, **kwargs):
 450         if self._debug:
 451             print(s, file=sys.stderr, **kwargs)
 452
 453
 454 # [ RUN MODES ] ###############################################################
 455
 456
 457 class MuttCommands:
 458     """
 459     Stupid class to interface writing out Mutt commands. This is quite a hack
 460     to deal with the fact that Mutt runs "push" commands in reverse order, so
 461     all of a sudden, things become very complicated when mixing with "real"
 462     commands.
 463
 464     Hence we keep two sets of commands, and one set of pushes. Commands are
 465     added to the first until a push is added, after which commands are added to
 466     the second set of commands.
 467
 468     On flush(), the first set is printed, followed by the pushes in reverse,
 469     and then the second set is printed. All 3 sets are then cleared.
 470     """
 471
 472     def __init__(self, out_f=sys.stdout, *, debug=False):
 473         self._cmd1, self._push, self._cmd2 = [], [], []
 474         self._out_f = out_f
 475         self._debug = debug
 476
 477     def cmd(self, s):
 478         self.debugprint(s)
 479         if self._push:
 480             self._cmd2.append(s)
 481         else:
 482             self._cmd1.append(s)
 483
 484     def push(self, s):
 485         s = s.replace('"', '"')
 486         s = f'push "{s}"'
 487         self.debugprint(s)
 488         self._push.insert(0, s)
 489
 490     def flush(self):
 491         print(
 492             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 493         )
 494         self._cmd1, self._push, self._cmd2 = [], [], []
 495
 496     def debugprint(self, s, **kwargs):
 497         if self._debug:
 498             print(s, file=sys.stderr, **kwargs)
 499
 500
 501 def do_setup(
 502     *,
 503     out_f=sys.stdout,
 504     temppath=None,
 505     tempdir=None,
 506     debug_commands=False,
 507 ):
 508     temppath = temppath or pathlib.Path(
 509         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 510     )
 511     cmds = MuttCommands(out_f, debug=debug_commands)
 512
 513     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 514
 515     cmds.cmd('set my_editor="$editor"')
 516     cmds.cmd('set my_edit_headers="$edit_headers"')
 517     cmds.cmd(f'set editor="{editor}"')
 518     cmds.cmd("unset edit_headers")
 519     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 520     cmds.push("<first-entry><edit-file>")
 521     cmds.flush()
 522
 523
 524 def do_massage(
 525     draft_f,
 526     draftpath,
 527     cmd_f,
 528     *,
 529     extensions=None,
 530     cssfile=None,
 531     converter=convert_markdown_to_html,
 532     related_to_html_only=True,
 533     only_build=False,
 534     max_other_attachments=20,
 535     tempdir=None,
 536     debug_commands=False,
 537     debug_walk=False,
 538 ):
 539     # Here's the big picture: we're being invoked as the editor on the email
 540     # draft, and whatever commands we write to the file given as cmdpath will
 541     # be run by the second source command in the macro definition.
 542
 543     # Let's start by cleaning up what the setup did (see above), i.e. we
 544     # restore the $editor and $edit_headers variables, and also unset the
 545     # variable used to identify the command file we're currently writing
 546     # to.
 547     cmds = MuttCommands(cmd_f, debug=debug_commands)
 548     cmds.cmd('set editor="$my_editor"')
 549     cmds.cmd('set edit_headers="$my_edit_headers"')
 550     cmds.cmd("unset my_editor")
 551     cmds.cmd("unset my_edit_headers")
 552
 553     # let's flush those commands, as there'll be a lot of pushes from now
 554     # on, which need to be run in reverse order
 555     cmds.flush()
 556
 557     extensions = extensions.split(",") if extensions else []
 558     tree = converter(
 559         draft_f.read(),
 560         draftpath,
 561         cssfile=cssfile,
 562         related_to_html_only=related_to_html_only,
 563         tempdir=tempdir,
 564         extensions=extensions,
 565     )
 566
 567     mimetree = MIMETreeDFWalker(debug=debug_walk)
 568
 569     state = dict(pos=1, tags={}, parts=1)
 570
 571     def visitor_fn(item, ancestry, descendents, *, debugprint=None):
 572         """
 573         Visitor function called for every node (part) of the MIME tree,
 574         depth-first, and responsible for telling NeoMutt how to assemble
 575         the tree.
 576         """
 577         KILL_LINE = r"\Ca\Ck"
 578
 579         if isinstance(item, Part):
 580             # We've hit a leaf-node, i.e. an alternative or a related part
 581             # with actual content.
 582
 583             # Let's add the part
 584             if item.orig:
 585                 # The original source already exists in the NeoMutt tree, but
 586                 # the underlying file may have been modified, so we need to
 587                 # update the encoding, but that's it:
 588                 cmds.push("<first-entry>")
 589                 cmds.push("<update-encoding>")
 590
 591                 # We really just need to be able to assume that at this point,
 592                 # NeoMutt is at position 1, and that we've processed only this
 593                 # part so far. Nevermind about actual attachments, we can
 594                 # safely ignore those as they stay at the end.
 595                 assert state["pos"] == 1
 596                 assert state["parts"] == 1
 597             else:
 598                 # … whereas all other parts need to be added, and they're all
 599                 # considered to be temporary and inline:
 600                 cmds.push(f"<attach-file>{item.path}<enter>")
 601                 cmds.push("<toggle-unlink><toggle-disposition>")
 602
 603                 # This added a part at the end of the list of parts, and that's
 604                 # just how many parts we've seen so far, so it's position in
 605                 # the NeoMutt compose list is the count of parts
 606                 state["parts"] += 1
 607                 state["pos"] = state["parts"]
 608
 609             # If the item (including the original) comes with additional
 610             # information, then we might just as well update the NeoMutt
 611             # tree now:
 612             if item.cid:
 613                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 614
 615             # Now for the biggest hack in this script, which is to handle
 616             # attachments, such as PDFs, that aren't related or alternatives.
 617             # The problem is that when we add an inline image, it always gets
 618             # appended to the list, i.e. inserted *after* other attachments.
 619             # Since we don't know the number of attachments, we also cannot
 620             # infer the postition of the new attachment. Therefore, we bubble
 621             # it all the way to the top, only to then move it down again:
 622             if state["pos"] > 1:  # skip for the first part
 623                 for i in range(max_other_attachments):
 624                     # could use any number here, but has to be larger than the
 625                     # number of possible attachments. The performance
 626                     # difference of using a high number is negligible.
 627                     # Bubble up the new part
 628                     cmds.push(f"<move-up>")
 629
 630                 # As we push the part to the right position in the list (i.e.
 631                 # the last of the subset of attachments this script added), we
 632                 # must handle the situation that subtrees are skipped by
 633                 # NeoMutt. Hence, the actual number of positions to move down
 634                 # is decremented by the number of descendents so far
 635                 # encountered.
 636                 for i in range(1, state["pos"] - len(descendents)):
 637                     cmds.push(f"<move-down>")
 638
 639         elif isinstance(item, Multipart):
 640             # This node has children, but we already visited them (see
 641             # above). The tags dictionary of State should contain a list of
 642             # their positions in the NeoMutt compose window, so iterate those
 643             # and tag the parts there:
 644             n_tags = len(state["tags"][item])
 645             for tag in state["tags"][item]:
 646                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
 647
 648             if item.subtype == "alternative":
 649                 cmds.push("<group-alternatives>")
 650             elif item.subtype in ("relative", "related"):
 651                 cmds.push("<group-related>")
 652             elif item.subtype == "multilingual":
 653                 cmds.push("<group-multilingual>")
 654             else:
 655                 raise NotImplementedError(
 656                     f"Handling of multipart/{item.subtype} is not implemented"
 657                 )
 658
 659             state["pos"] -= n_tags - 1
 660             state["parts"] += 1
 661
 662         else:
 663             # We should never get here
 664             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
 665
 666         # If the item has a description, we might just as well add it
 667         if item.desc:
 668             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 669
 670         if ancestry:
 671             # If there's an ancestry, record the current (assumed) position in
 672             # the NeoMutt compose window as needed-to-tag by our direct parent
 673             # (i.e. the last item of the ancestry)
 674             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
 675
 676             lead = "│ " * (len(ancestry) + 1) + "* "
 677             debugprint(
 678                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
 679                 f"{lead}descendents={[d.subtype for d in descendents]}\n"
 680                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
 681                 f"{lead}pos={state['pos']}, parts={state['parts']}"
 682             )
 683
 684     # -----------------
 685     # End of visitor_fn
 686
 687     # Let's walk the tree and visit every node with our fancy visitor
 688     # function
 689     mimetree.walk(tree, visitor_fn=visitor_fn)
 690
 691     if not only_build:
 692         cmds.push("<send-message>")
 693
 694     # Finally, cleanup. Since we're responsible for removing the temporary
 695     # file, how's this for a little hack?
 696     try:
 697         filename = cmd_f.name
 698     except AttributeError:
 699         filename = "pytest_internal_file"
 700     cmds.cmd(f"source 'rm -f {filename}|'")
 701     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 702     cmds.flush()
 703
 704
 705 # [ CLI ENTRY ] ###############################################################
 706
 707 if __name__ == "__main__":
 708     args = parse_cli_args()
 709
 710     if args.mode is None:
 711         do_setup(
 712             tempdir=args.tempdir,
 713             debug_commands=args.debug_commands,
 714         )
 715
 716     elif args.mode == "massage":
 717         with open(args.MAILDRAFT, "r") as draft_f, open(
 718             args.cmdpath, "w"
 719         ) as cmd_f:
 720             do_massage(
 721                 draft_f,
 722                 args.MAILDRAFT,
 723                 cmd_f,
 724                 extensions=args.extensions,
 725                 cssfile=args.css_file,
 726                 related_to_html_only=args.related_to_html_only,
 727                 max_other_attachments=args.max_number_other_attachments,
 728                 only_build=args.only_build,
 729                 tempdir=args.tempdir,
 730                 debug_commands=args.debug_commands,
 731                 debug_walk=args.debug_walk,
 732             )
 733
 734
 735 # [ TESTS ] ###################################################################
 736
 737 try:
 738     import pytest
 739     from io import StringIO
 740
 741     class Tests:
 742         @pytest.fixture
 743         def const1(self):
 744             return "CONSTANT STRING 1"
 745
 746         @pytest.fixture
 747         def const2(self):
 748             return "CONSTANT STRING 2"
 749
 750         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 751         # functions they call, else old stdout is used and not captured
 752
 753         def test_MuttCommands_cmd(self, const1, const2, capsys):
 754             "Assert order of commands"
 755             cmds = MuttCommands(out_f=sys.stdout)
 756             cmds.cmd(const1)
 757             cmds.cmd(const2)
 758             cmds.flush()
 759             captured = capsys.readouterr()
 760             assert captured.out == "\n".join((const1, const2, ""))
 761
 762         def test_MuttCommands_push(self, const1, const2, capsys):
 763             "Assert reverse order of pushes"
 764             cmds = MuttCommands(out_f=sys.stdout)
 765             cmds.push(const1)
 766             cmds.push(const2)
 767             cmds.flush()
 768             captured = capsys.readouterr()
 769             assert (
 770                 captured.out
 771                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 772             )
 773
 774         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 775             "Assert reverse order of pushes"
 776             cmds = MuttCommands(out_f=sys.stdout)
 777             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 778             for i in range(2):
 779                 cmds.cmd(lines[4 * i + 0])
 780                 cmds.cmd(lines[4 * i + 1])
 781                 cmds.push(lines[4 * i + 2])
 782                 cmds.push(lines[4 * i + 3])
 783             cmds.flush()
 784
 785             captured = capsys.readouterr()
 786             lines_out = captured.out.splitlines()
 787             assert lines[0] in lines_out[0]
 788             assert lines[1] in lines_out[1]
 789             assert lines[7] in lines_out[2]
 790             assert lines[6] in lines_out[3]
 791             assert lines[3] in lines_out[4]
 792             assert lines[2] in lines_out[5]
 793             assert lines[4] in lines_out[6]
 794             assert lines[5] in lines_out[7]
 795
 796         @pytest.fixture
 797         def mime_tree_related_to_alternative(self):
 798             return Multipart(
 799                 "relative",
 800                 children=[
 801                     Multipart(
 802                         "alternative",
 803                         children=[
 804                             Part(
 805                                 "text",
 806                                 "plain",
 807                                 "part.txt",
 808                                 desc="Plain",
 809                                 orig=True,
 810                             ),
 811                             Part("text", "html", "part.html", desc="HTML"),
 812                         ],
 813                         desc="Alternative",
 814                     ),
 815                     Part(
 816                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 817                     ),
 818                 ],
 819                 desc="Related",
 820             )
 821
 822         @pytest.fixture
 823         def mime_tree_related_to_html(self):
 824             return Multipart(
 825                 "alternative",
 826                 children=[
 827                     Part(
 828                         "text",
 829                         "plain",
 830                         "part.txt",
 831                         desc="Plain",
 832                         orig=True,
 833                     ),
 834                     Multipart(
 835                         "relative",
 836                         children=[
 837                             Part("text", "html", "part.html", desc="HTML"),
 838                             Part(
 839                                 "text",
 840                                 "png",
 841                                 "logo.png",
 842                                 cid="logo.png",
 843                                 desc="Logo",
 844                             ),
 845                         ],
 846                         desc="Related",
 847                     ),
 848                 ],
 849                 desc="Alternative",
 850             )
 851
 852         def test_MIMETreeDFWalker_depth_first_walk(
 853             self, mime_tree_related_to_alternative
 854         ):
 855             mimetree = MIMETreeDFWalker()
 856
 857             items = []
 858
 859             def visitor_fn(item, ancestry, descendents, debugprint):
 860                 items.append((item, len(ancestry), len(descendents)))
 861
 862             mimetree.walk(
 863                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
 864             )
 865             assert len(items) == 5
 866             assert items[0][0].subtype == "plain"
 867             assert items[0][1] == 2
 868             assert items[0][2] == 0
 869             assert items[1][0].subtype == "html"
 870             assert items[1][1] == 2
 871             assert items[1][2] == 0
 872             assert items[2][0].subtype == "alternative"
 873             assert items[2][1] == 1
 874             assert items[2][2] == 2
 875             assert items[3][0].subtype == "png"
 876             assert items[3][1] == 1
 877             assert items[3][2] == 2
 878             assert items[4][0].subtype == "relative"
 879             assert items[4][1] == 0
 880             assert items[4][2] == 4
 881
 882         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
 883             mimetree = MIMETreeDFWalker()
 884             items = []
 885
 886             def visitor_fn(item, ancestry, descendents, debugprint):
 887                 items.append(item)
 888
 889             p = Part("text", "plain", const1)
 890             mimetree.walk([p], visitor_fn=visitor_fn)
 891             assert items[-1].subtype == "plain"
 892             mimetree.walk([p, p], visitor_fn=visitor_fn)
 893             assert items[-1].subtype == "mixed"
 894
 895         def test_MIMETreeDFWalker_visitor_in_constructor(
 896             self, mime_tree_related_to_alternative
 897         ):
 898             items = []
 899
 900             def visitor_fn(item, ancestry, descendents, debugprint):
 901                 items.append(item)
 902
 903             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 904             mimetree.walk(mime_tree_related_to_alternative)
 905             assert len(items) == 5
 906
 907         @pytest.fixture
 908         def string_io(self, const1, text=None):
 909             return StringIO(text or const1)
 910
 911         def test_do_massage_basic(self, const1, string_io, capsys):
 912             def converter(
 913                 drafttext,
 914                 draftpath,
 915                 cssfile,
 916                 related_to_html_only,
 917                 extensions,
 918                 tempdir,
 919             ):
 920                 return Part("text", "plain", draftpath, orig=True)
 921
 922             do_massage(
 923                 draft_f=string_io,
 924                 draftpath=const1,
 925                 cmd_f=sys.stdout,
 926                 converter=converter,
 927             )
 928
 929             captured = capsys.readouterr()
 930             lines = captured.out.splitlines()
 931             assert '="$my_editor"' in lines.pop(0)
 932             assert '="$my_edit_headers"' in lines.pop(0)
 933             assert "unset my_editor" == lines.pop(0)
 934             assert "unset my_edit_headers" == lines.pop(0)
 935             assert "send-message" in lines.pop(0)
 936             assert "update-encoding" in lines.pop(0)
 937             assert "first-entry" in lines.pop(0)
 938             assert "source 'rm -f " in lines.pop(0)
 939             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 940
 941         def test_do_massage_fulltree(
 942             self, string_io, const1, mime_tree_related_to_alternative, capsys
 943         ):
 944             def converter(
 945                 drafttext,
 946                 draftpath,
 947                 cssfile,
 948                 related_to_html_only,
 949                 extensions,
 950                 tempdir,
 951             ):
 952                 return mime_tree_related_to_alternative
 953
 954             max_attachments = 5
 955             do_massage(
 956                 draft_f=string_io,
 957                 draftpath=const1,
 958                 cmd_f=sys.stdout,
 959                 max_other_attachments=max_attachments,
 960                 converter=converter,
 961             )
 962
 963             captured = capsys.readouterr()
 964             lines = captured.out.splitlines()[4:-2]
 965             assert "first-entry" in lines.pop()
 966             assert "update-encoding" in lines.pop()
 967             assert "Plain" in lines.pop()
 968             assert "part.html" in lines.pop()
 969             assert "toggle-unlink" in lines.pop()
 970             for i in range(max_attachments):
 971                 assert "move-up" in lines.pop()
 972             assert "move-down" in lines.pop()
 973             assert "HTML" in lines.pop()
 974             assert "jump>1" in lines.pop()
 975             assert "jump>2" in lines.pop()
 976             assert "group-alternatives" in lines.pop()
 977             assert "Alternative" in lines.pop()
 978             assert "logo.png" in lines.pop()
 979             assert "toggle-unlink" in lines.pop()
 980             assert "content-id" in lines.pop()
 981             for i in range(max_attachments):
 982                 assert "move-up" in lines.pop()
 983             assert "move-down" in lines.pop()
 984             assert "Logo" in lines.pop()
 985             assert "jump>1" in lines.pop()
 986             assert "jump>4" in lines.pop()
 987             assert "group-related" in lines.pop()
 988             assert "Related" in lines.pop()
 989             assert "send-message" in lines.pop()
 990             assert len(lines) == 0
 991
 992         @pytest.fixture
 993         def fake_filewriter(self):
 994             class FileWriter:
 995                 def __init__(self):
 996                     self._writes = []
 997
 998                 def __call__(self, path, content, mode="w", **kwargs):
 999                     self._writes.append((path, content))
1000
1001                 def pop(self, index=-1):
1002                     return self._writes.pop(index)
1003
1004             return FileWriter()
1005
1006         @pytest.fixture
1007         def markdown_non_converter(self, const1, const2):
1008             return lambda s, text: f"{const1}{text}{const2}"
1009
1010         def test_converter_tree_basic(self, const1, const2, fake_filewriter):
1011             path = pathlib.Path(const2)
1012             tree = convert_markdown_to_html(
1013                 const1, path, filewriter_fn=fake_filewriter
1014             )
1015
1016             assert tree.subtype == "alternative"
1017             assert len(tree.children) == 2
1018             assert tree.children[0].subtype == "plain"
1019             assert tree.children[0].path == path
1020             assert tree.children[0].orig
1021             assert tree.children[1].subtype == "html"
1022             assert tree.children[1].path == path.with_suffix(".html")
1023
1024         def test_converter_writes(
1025             self,
1026             const1,
1027             const2,
1028             fake_filewriter,
1029             monkeypatch,
1030             markdown_non_converter,
1031         ):
1032             path = pathlib.Path(const2)
1033
1034             with monkeypatch.context() as m:
1035                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
1036                 convert_markdown_to_html(
1037                     const1, path, filewriter_fn=fake_filewriter
1038                 )
1039
1040             assert (path, const1) == fake_filewriter.pop(0)
1041             assert (
1042                 path.with_suffix(".html"),
1043                 markdown_non_converter(None, const1),
1044             ) == fake_filewriter.pop(0)
1045
1046         def test_markdown_inline_image_processor(self):
1047             imgpath1 = "file:/path/to/image.png"
1048             imgpath2 = "file:///path/to/image.png?url=params"
1049             imgpath3 = "/path/to/image.png"
1050             text = f"""![inline local image]({imgpath1})
1051                        ![image inlined
1052                          with newline]({imgpath2})
1053                        ![image local path]({imgpath3})"""
1054             text, html, images = markdown_with_inline_image_support(text)
1055
1056             # local paths have been normalised to URLs:
1057             imgpath3 = f"file://{imgpath3}"
1058
1059             assert 'src="cid:' in html
1060             assert "](cid:" in text
1061             assert len(images) == 3
1062             assert imgpath1 in images
1063             assert imgpath2 in images
1064             assert imgpath3 in images
1065             assert images[imgpath1].cid != images[imgpath2].cid
1066             assert images[imgpath1].cid != images[imgpath3].cid
1067             assert images[imgpath2].cid != images[imgpath3].cid
1068
1069         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1070             imgpath = "file:///path/to/image.png"
1071             text = f'![inline local image]({imgpath} "{const1}")'
1072             text, html, images = markdown_with_inline_image_support(text)
1073             assert images[imgpath].desc == const1
1074
1075         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1076             imgpath = "file:///path/to/image.png"
1077             text = f"![{const1}]({imgpath})"
1078             text, html, images = markdown_with_inline_image_support(text)
1079             assert images[imgpath].desc == const1
1080
1081         def test_markdown_inline_image_processor_title_over_alt_desc(
1082             self, const1, const2
1083         ):
1084             imgpath = "file:///path/to/image.png"
1085             text = f'![{const1}]({imgpath} "{const2}")'
1086             text, html, images = markdown_with_inline_image_support(text)
1087             assert images[imgpath].desc == const2
1088
1089         def test_markdown_inline_image_not_external(self):
1090             imgpath = "https://path/to/image.png"
1091             text = f"![inline image]({imgpath})"
1092             text, html, images = markdown_with_inline_image_support(text)
1093
1094             assert 'src="cid:' not in html
1095             assert "](cid:" not in text
1096             assert len(images) == 0
1097
1098         def test_markdown_inline_image_local_file(self):
1099             imgpath = "/path/to/image.png"
1100             text = f"![inline image]({imgpath})"
1101             text, html, images = markdown_with_inline_image_support(text)
1102
1103             for k, v in images.items():
1104                 assert k == f"file://{imgpath}"
1105                 break
1106
1107         @pytest.fixture
1108         def test_png(self):
1109             return (
1110                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1111                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1112             )
1113
1114         def test_markdown_inline_image_processor_base64(self, test_png):
1115             text = f"![1px white inlined]({test_png})"
1116             text, html, images = markdown_with_inline_image_support(text)
1117
1118             assert 'src="cid:' in html
1119             assert "](cid:" in text
1120             assert len(images) == 1
1121             assert test_png in images
1122
1123         def test_converter_tree_inline_image_base64(
1124             self, test_png, const1, fake_filewriter
1125         ):
1126             text = f"![inline base64 image]({test_png})"
1127             path = pathlib.Path(const1)
1128             tree = convert_markdown_to_html(
1129                 text,
1130                 path,
1131                 filewriter_fn=fake_filewriter,
1132                 related_to_html_only=False,
1133             )
1134             assert tree.subtype == "relative"
1135             assert tree.children[0].subtype == "alternative"
1136             assert tree.children[1].subtype == "png"
1137             written = fake_filewriter.pop()
1138             assert tree.children[1].path == written[0]
1139             assert written[1] == request.urlopen(test_png).read()
1140
1141         def test_converter_tree_inline_image_base64_related_to_html(
1142             self, test_png, const1, fake_filewriter
1143         ):
1144             text = f"![inline base64 image]({test_png})"
1145             path = pathlib.Path(const1)
1146             tree = convert_markdown_to_html(
1147                 text,
1148                 path,
1149                 filewriter_fn=fake_filewriter,
1150                 related_to_html_only=True,
1151             )
1152             assert tree.subtype == "alternative"
1153             assert tree.children[1].subtype == "relative"
1154             assert tree.children[1].children[1].subtype == "png"
1155             written = fake_filewriter.pop()
1156             assert tree.children[1].children[1].path == written[0]
1157             assert written[1] == request.urlopen(test_png).read()
1158
1159         def test_converter_tree_inline_image_cid(
1160             self, const1, fake_filewriter
1161         ):
1162             text = f"![inline base64 image](cid:{const1})"
1163             path = pathlib.Path(const1)
1164             tree = convert_markdown_to_html(
1165                 text,
1166                 path,
1167                 filewriter_fn=fake_filewriter,
1168                 related_to_html_only=False,
1169             )
1170             assert len(tree.children) == 2
1171             assert tree.children[0].cid != const1
1172             assert tree.children[0].type != "image"
1173             assert tree.children[1].cid != const1
1174             assert tree.children[1].type != "image"
1175
1176         def test_inline_image_collection(
1177             self, test_png, const1, const2, fake_filewriter
1178         ):
1179             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1180             relparts = collect_inline_images(
1181                 test_images, filewriter_fn=fake_filewriter
1182             )
1183
1184             written = fake_filewriter.pop()
1185             assert b"PNG" in written[1]
1186
1187             assert relparts[0].subtype == "png"
1188             assert relparts[0].path == written[0]
1189             assert relparts[0].cid == const1
1190             assert relparts[0].desc.endswith(const2)
1191
1192         def test_apply_stylesheet(self):
1193             if _PYNLINER:
1194                 html = "<p>Hello, world!</p>"
1195                 css = "p { color:red }"
1196                 out = apply_styling(html, css)
1197                 assert 'p style="color' in out
1198
1199         def test_apply_stylesheet_pygments(self):
1200             if _PYGMENTS_CSS:
1201                 html = (
1202                     f'<div class="{_CODEHILITE_CLASS}">'
1203                     "<pre>def foo():\n    return</pre></div>"
1204                 )
1205                 out = apply_styling(html, _PYGMENTS_CSS)
1206                 assert f'{_CODEHILITE_CLASS}" style="' in out
1207
1208         def test_mime_tree_relative_within_alternative(
1209             self, string_io, const1, capsys, mime_tree_related_to_html
1210         ):
1211             def converter(
1212                 drafttext,
1213                 draftpath,
1214                 cssfile,
1215                 related_to_html_only,
1216                 extensions,
1217                 tempdir,
1218             ):
1219                 return mime_tree_related_to_html
1220
1221             do_massage(
1222                 draft_f=string_io,
1223                 draftpath=const1,
1224                 cmd_f=sys.stdout,
1225                 converter=converter,
1226             )
1227
1228             captured = capsys.readouterr()
1229             lines = captured.out.splitlines()[4:-2]
1230             assert "first-entry" in lines.pop()
1231             assert "update-encoding" in lines.pop()
1232             assert "Plain" in lines.pop()
1233             assert "part.html" in lines.pop()
1234             assert "toggle-unlink" in lines.pop()
1235             assert "move-up" in lines.pop()
1236             while True:
1237                 top = lines.pop()
1238                 if "move-up" not in top:
1239                     break
1240             assert "move-down" in top
1241             assert "HTML" in lines.pop()
1242             assert "logo.png" in lines.pop()
1243             assert "toggle-unlink" in lines.pop()
1244             assert "content-id" in lines.pop()
1245             assert "move-up" in lines.pop()
1246             while True:
1247                 top = lines.pop()
1248                 if "move-up" not in top:
1249                     break
1250             assert "move-down" in top
1251             assert "move-down" in lines.pop()
1252             assert "Logo" in lines.pop()
1253             assert "jump>2" in lines.pop()
1254             assert "jump>3" in lines.pop()
1255             assert "group-related" in lines.pop()
1256             assert "Related" in lines.pop()
1257             assert "jump>1" in lines.pop()
1258             assert "jump>2" in lines.pop()
1259             assert "group-alternative" in lines.pop()
1260             assert "Alternative" in lines.pop()
1261             assert "send-message" in lines.pop()
1262             assert len(lines) == 0
1263
1264         def test_mime_tree_nested_trees_does_not_break_positioning(
1265             self, string_io, const1, capsys
1266         ):
1267             def converter(
1268                 drafttext,
1269                 draftpath,
1270                 cssfile,
1271                 related_to_html_only,
1272                 extensions,
1273                 tempdir,
1274             ):
1275                 return Multipart(
1276                     "relative",
1277                     children=[
1278                         Multipart(
1279                             "alternative",
1280                             children=[
1281                                 Part(
1282                                     "text",
1283                                     "plain",
1284                                     "part.txt",
1285                                     desc="Plain",
1286                                     orig=True,
1287                                 ),
1288                                 Multipart(
1289                                     "alternative",
1290                                     children=[
1291                                         Part(
1292                                             "text",
1293                                             "plain",
1294                                             "part.txt",
1295                                             desc="Nested plain",
1296                                         ),
1297                                         Part(
1298                                             "text",
1299                                             "html",
1300                                             "part.html",
1301                                             desc="Nested HTML",
1302                                         ),
1303                                     ],
1304                                     desc="Nested alternative",
1305                                 ),
1306                             ],
1307                             desc="Alternative",
1308                         ),
1309                         Part(
1310                             "text",
1311                             "png",
1312                             "logo.png",
1313                             cid="logo.png",
1314                             desc="Logo",
1315                         ),
1316                     ],
1317                     desc="Related",
1318                 )
1319
1320             do_massage(
1321                 draft_f=string_io,
1322                 draftpath=const1,
1323                 cmd_f=sys.stdout,
1324                 converter=converter,
1325             )
1326
1327             captured = capsys.readouterr()
1328             lines = captured.out.splitlines()
1329             while not "logo.png" in lines.pop():
1330                 pass
1331             lines.pop()
1332             assert "content-id" in lines.pop()
1333             assert "move-up" in lines.pop()
1334             while True:
1335                 top = lines.pop()
1336                 if "move-up" not in top:
1337                     break
1338             assert "move-down" in top
1339             # Due to the nested trees, the number of descendents of the sibling
1340             # actually needs to be considered, not just the nieces. So to move
1341             # from position 1 to position 6, it only needs one <move-down>
1342             # because that jumps over the entire sibling tree. Thus what
1343             # follows next must not be another <move-down>
1344             assert "Logo" in lines.pop()
1345
1346 except ImportError:
1347     pass