.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--related-to-html-only",
  81         action="store_true",
  82         help="Make related content be sibling to HTML parts only",
  83     )
  84
  85     def positive_integer(value):
  86         try:
  87             if int(value) > 0:
  88                 return int(value)
  89
  90         except ValueError:
  91             pass
  92
  93         raise ValueError(f"Must be a positive integer")
  94
  95     parser.add_argument(
  96         "--max-number-other-attachments",
  97         type=positive_integer,
  98         help="Make related content be sibling to HTML parts only",
  99     )
 100
 101     parser.add_argument(
 102         "--only-build",
 103         action="store_true",
 104         help="Only build, don't send the message",
 105     )
 106
 107     parser.add_argument(
 108         "--tempdir",
 109         default=None,
 110         help="Specify temporary directory to use for attachments",
 111     )
 112
 113     parser.add_argument(
 114         "--debug-commands",
 115         action="store_true",
 116         help="Turn on debug logging of commands generated to stderr",
 117     )
 118
 119     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
 120     massage_p = subp.add_parser(
 121         "massage", help="Massaging phase (internal use)"
 122     )
 123
 124     massage_p.add_argument(
 125         "--write-commands-to",
 126         "-o",
 127         metavar="PATH",
 128         dest="cmdpath",
 129         required=True,
 130         help="Temporary file path to write commands to",
 131     )
 132
 133     massage_p.add_argument(
 134         "--debug-walk",
 135         action="store_true",
 136         help="Turn on debugging to stderr of the MIME tree walk",
 137     )
 138
 139     massage_p.add_argument(
 140         "MAILDRAFT",
 141         nargs="?",
 142         help="If provided, the script is invoked as editor on the mail draft",
 143     )
 144
 145     return parser.parse_args(*args, **kwargs)
 146
 147
 148 # [ MARKDOWN WRAPPING ] #######################################################
 149
 150
 151 InlineImageInfo = namedtuple(
 152     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 153 )
 154
 155
 156 class InlineImageExtension(Extension):
 157     class RelatedImageInlineProcessor(ImageInlineProcessor):
 158         def __init__(self, re, md, ext):
 159             super().__init__(re, md)
 160             self._ext = ext
 161
 162         def handleMatch(self, m, data):
 163             el, start, end = super().handleMatch(m, data)
 164             if "src" in el.attrib:
 165                 src = el.attrib["src"]
 166                 if "://" not in src or src.startswith("file://"):
 167                     # We only inline local content
 168                     cid = self._ext.get_cid_for_image(el.attrib)
 169                     el.attrib["src"] = f"cid:{cid}"
 170             return el, start, end
 171
 172     def __init__(self):
 173         super().__init__()
 174         self._images = OrderedDict()
 175
 176     def extendMarkdown(self, md):
 177         md.registerExtension(self)
 178         inline_image_proc = self.RelatedImageInlineProcessor(
 179             IMAGE_LINK_RE, md, self
 180         )
 181         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 182
 183     def get_cid_for_image(self, attrib):
 184         msgid = make_msgid()[1:-1]
 185         path = attrib["src"]
 186         if path.startswith("/"):
 187             path = f"file://{path}"
 188         self._images[path] = InlineImageInfo(
 189             msgid, attrib.get("title", attrib.get("alt"))
 190         )
 191         return msgid
 192
 193     def get_images(self):
 194         return self._images
 195
 196
 197 def markdown_with_inline_image_support(
 198     text, *, extensions=None, extension_configs=None
 199 ):
 200     inline_image_handler = InlineImageExtension()
 201     extensions = extensions or []
 202     extensions.append(inline_image_handler)
 203     mdwn = markdown.Markdown(
 204         extensions=extensions, extension_configs=extension_configs
 205     )
 206     htmltext = mdwn.convert(text)
 207
 208     images = inline_image_handler.get_images()
 209
 210     def replace_image_with_cid(matchobj):
 211         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 212             if m in images:
 213                 return f"(cid:{images[m].cid}"
 214         return matchobj.group(0)
 215
 216     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 217     return text, htmltext, images
 218
 219
 220 # [ CSS STYLING ] #############################################################
 221
 222 try:
 223     import pynliner
 224
 225     _PYNLINER = True
 226
 227 except ImportError:
 228     _PYNLINER = False
 229
 230 try:
 231     from pygments.formatters import get_formatter_by_name
 232
 233     _CODEHILITE_CLASS = "codehilite"
 234
 235     _PYGMENTS_CSS = get_formatter_by_name(
 236         "html", style="default"
 237     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 238
 239 except ImportError:
 240     _PYGMENTS_CSS = None
 241
 242
 243 def apply_styling(html, css):
 244     return (
 245         pynliner.Pynliner()
 246         .from_string(html)
 247         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 248         .run()
 249     )
 250
 251
 252 # [ PARTS GENERATION ] ########################################################
 253
 254
 255 class Part(
 256     namedtuple(
 257         "Part",
 258         ["type", "subtype", "path", "desc", "cid", "orig"],
 259         defaults=[None, None, False],
 260     )
 261 ):
 262     def __str__(self):
 263         ret = f"<{self.type}/{self.subtype}>"
 264         if self.cid:
 265             ret = f"{ret} cid:{self.cid}"
 266         if self.orig:
 267             ret = f"{ret} ORIGINAL"
 268         return ret
 269
 270
 271 class Multipart(
 272     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 273 ):
 274     def __str__(self):
 275         return f"<multipart/{self.subtype}> children={len(self.children)}"
 276
 277     def __hash__(self):
 278         return hash(str(self.subtype) + "".join(str(self.children)))
 279
 280
 281 def filewriter_fn(path, content, mode="w", **kwargs):
 282     with open(path, mode, **kwargs) as out_f:
 283         out_f.write(content)
 284
 285
 286 def collect_inline_images(
 287     images, *, tempdir=None, filewriter_fn=filewriter_fn
 288 ):
 289     relparts = []
 290     for path, info in images.items():
 291         if path.startswith("cid:"):
 292             continue
 293
 294         data = request.urlopen(path)
 295
 296         mimetype = data.headers["Content-Type"]
 297         ext = mimetypes.guess_extension(mimetype)
 298         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 299         path = pathlib.Path(tempfilename[1])
 300
 301         filewriter_fn(path, data.read(), "w+b")
 302
 303         relparts.append(
 304             Part(
 305                 *mimetype.split("/"),
 306                 path,
 307                 cid=info.cid,
 308                 desc=f"Image: {info.desc}",
 309             )
 310         )
 311
 312     return relparts
 313
 314
 315 def convert_markdown_to_html(
 316     origtext,
 317     draftpath,
 318     *,
 319     related_to_html_only=False,
 320     cssfile=None,
 321     filewriter_fn=filewriter_fn,
 322     tempdir=None,
 323     extensions=None,
 324     extension_configs=None,
 325 ):
 326     # TODO extension_configs need to be handled differently
 327     extension_configs = extension_configs or {}
 328     extension_configs.setdefault("pymdownx.highlight", {})
 329     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 330
 331     origtext, htmltext, images = markdown_with_inline_image_support(
 332         origtext, extensions=extensions, extension_configs=extension_configs
 333     )
 334
 335     filewriter_fn(draftpath, origtext, encoding="utf-8")
 336     textpart = Part(
 337         "text", "plain", draftpath, "Plain-text version", orig=True
 338     )
 339
 340     htmltext = apply_styling(htmltext, cssfile)
 341
 342     htmlpath = draftpath.with_suffix(".html")
 343     filewriter_fn(
 344         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 345     )
 346     htmlpart = Part("text", "html", htmlpath, "HTML version")
 347
 348     imgparts = collect_inline_images(
 349         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 350     )
 351
 352     if related_to_html_only:
 353         # If there are inline image part, they will be contained within a
 354         # multipart/related part along with the HTML part only
 355         if imgparts:
 356             # replace htmlpart with a multipart/related container of the HTML
 357             # parts and the images
 358             htmlpart = Multipart(
 359                 "relative", [htmlpart] + imgparts, "Group of related content"
 360             )
 361
 362         return Multipart(
 363             "alternative", [textpart, htmlpart], "Group of alternative content"
 364         )
 365
 366     else:
 367         # If there are inline image part, they will be siblings to the
 368         # multipart/alternative tree within a multipart/related part
 369         altpart = Multipart(
 370             "alternative", [textpart, htmlpart], "Group of alternative content"
 371         )
 372         if imgparts:
 373             return Multipart(
 374                 "relative", [altpart] + imgparts, "Group of related content"
 375             )
 376         else:
 377             return altpart
 378
 379
 380 class MIMETreeDFWalker:
 381     def __init__(self, *, visitor_fn=None, debug=False):
 382         self._visitor_fn = visitor_fn or self._echovisit
 383         self._debug = debug
 384
 385     def _echovisit(self, node, ancestry, debugprint):
 386         debugprint(f"node={node} ancestry={ancestry}")
 387
 388     def walk(self, root, *, visitor_fn=None):
 389         """
 390         Recursive function to implement a depth-dirst walk of the MIME-tree
 391         rooted at `root`.
 392         """
 393         if isinstance(root, list):
 394             if len(root) > 1:
 395                 root = Multipart("mixed", children=root)
 396             else:
 397                 root = root[0]
 398
 399         self._walk(
 400             root,
 401             ancestry=[],
 402             descendents=[],
 403             visitor_fn=visitor_fn or self._visitor_fn,
 404         )
 405
 406     def _walk(self, node, *, ancestry, descendents, visitor_fn):
 407         # Let's start by enumerating the parts at the current level. At the
 408         # root level, ancestry will be the empty list, and we expect a
 409         # multipart/* container at this level. Later, e.g. within a
 410         # mutlipart/alternative container, the subtree will just be the
 411         # alternative parts, while the top of the ancestry will be the
 412         # multipart/alternative container, which we will process after the
 413         # following loop.
 414
 415         lead = f"{'│ '*len(ancestry)}"
 416         if isinstance(node, Multipart):
 417             self.debugprint(
 418                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
 419             )
 420
 421             # Depth-first, so push the current container onto the ancestry
 422             # stack, then descend …
 423             ancestry.append(node)
 424             self.debugprint(lead + "│ " * 2)
 425             for child in node.children:
 426                 self._walk(
 427                     child,
 428                     ancestry=ancestry,
 429                     descendents=descendents,
 430                     visitor_fn=visitor_fn,
 431                 )
 432             assert ancestry.pop() == node
 433             sibling_descendents = descendents
 434             descendents.extend(node.children)
 435
 436         else:
 437             self.debugprint(f"{lead}├{node}")
 438             sibling_descendents = descendents
 439
 440         if False and ancestry:
 441             self.debugprint(lead[:-1] + " │")
 442
 443         if visitor_fn:
 444             visitor_fn(
 445                 node, ancestry, sibling_descendents, debugprint=self.debugprint
 446             )
 447
 448     def debugprint(self, s, **kwargs):
 449         if self._debug:
 450             print(s, file=sys.stderr, **kwargs)
 451
 452
 453 # [ RUN MODES ] ###############################################################
 454
 455
 456 class MuttCommands:
 457     """
 458     Stupid class to interface writing out Mutt commands. This is quite a hack
 459     to deal with the fact that Mutt runs "push" commands in reverse order, so
 460     all of a sudden, things become very complicated when mixing with "real"
 461     commands.
 462
 463     Hence we keep two sets of commands, and one set of pushes. Commands are
 464     added to the first until a push is added, after which commands are added to
 465     the second set of commands.
 466
 467     On flush(), the first set is printed, followed by the pushes in reverse,
 468     and then the second set is printed. All 3 sets are then cleared.
 469     """
 470
 471     def __init__(self, out_f=sys.stdout, *, debug=False):
 472         self._cmd1, self._push, self._cmd2 = [], [], []
 473         self._out_f = out_f
 474         self._debug = debug
 475
 476     def cmd(self, s):
 477         self.debugprint(s)
 478         if self._push:
 479             self._cmd2.append(s)
 480         else:
 481             self._cmd1.append(s)
 482
 483     def push(self, s):
 484         s = s.replace('"', '"')
 485         s = f'push "{s}"'
 486         self.debugprint(s)
 487         self._push.insert(0, s)
 488
 489     def flush(self):
 490         print(
 491             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 492         )
 493         self._cmd1, self._push, self._cmd2 = [], [], []
 494
 495     def debugprint(self, s, **kwargs):
 496         if self._debug:
 497             print(s, file=sys.stderr, **kwargs)
 498
 499
 500 def do_setup(
 501     *,
 502     out_f=sys.stdout,
 503     temppath=None,
 504     tempdir=None,
 505     debug_commands=False,
 506 ):
 507     temppath = temppath or pathlib.Path(
 508         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 509     )
 510     cmds = MuttCommands(out_f, debug=debug_commands)
 511
 512     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 513
 514     cmds.cmd('set my_editor="$editor"')
 515     cmds.cmd('set my_edit_headers="$edit_headers"')
 516     cmds.cmd(f'set editor="{editor}"')
 517     cmds.cmd("unset edit_headers")
 518     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 519     cmds.push("<first-entry><edit-file>")
 520     cmds.flush()
 521
 522
 523 def do_massage(
 524     draft_f,
 525     draftpath,
 526     cmd_f,
 527     *,
 528     extensions=None,
 529     cssfile=None,
 530     converter=convert_markdown_to_html,
 531     related_to_html_only=True,
 532     only_build=False,
 533     max_other_attachments=20,
 534     tempdir=None,
 535     debug_commands=False,
 536     debug_walk=False,
 537 ):
 538     # Here's the big picture: we're being invoked as the editor on the email
 539     # draft, and whatever commands we write to the file given as cmdpath will
 540     # be run by the second source command in the macro definition.
 541
 542     # Let's start by cleaning up what the setup did (see above), i.e. we
 543     # restore the $editor and $edit_headers variables, and also unset the
 544     # variable used to identify the command file we're currently writing
 545     # to.
 546     cmds = MuttCommands(cmd_f, debug=debug_commands)
 547     cmds.cmd('set editor="$my_editor"')
 548     cmds.cmd('set edit_headers="$my_edit_headers"')
 549     cmds.cmd("unset my_editor")
 550     cmds.cmd("unset my_edit_headers")
 551
 552     # let's flush those commands, as there'll be a lot of pushes from now
 553     # on, which need to be run in reverse order
 554     cmds.flush()
 555
 556     extensions = extensions.split(",") if extensions else []
 557     tree = converter(
 558         draft_f.read(),
 559         draftpath,
 560         cssfile=cssfile,
 561         related_to_html_only=related_to_html_only,
 562         tempdir=tempdir,
 563         extensions=extensions,
 564     )
 565
 566     mimetree = MIMETreeDFWalker(debug=debug_walk)
 567
 568     state = dict(pos=1, tags={}, parts=1)
 569
 570     def visitor_fn(item, ancestry, descendents, *, debugprint=None):
 571         """
 572         Visitor function called for every node (part) of the MIME tree,
 573         depth-first, and responsible for telling NeoMutt how to assemble
 574         the tree.
 575         """
 576         KILL_LINE = r"\Ca\Ck"
 577
 578         if isinstance(item, Part):
 579             # We've hit a leaf-node, i.e. an alternative or a related part
 580             # with actual content.
 581
 582             # Let's add the part
 583             if item.orig:
 584                 # The original source already exists in the NeoMutt tree, but
 585                 # the underlying file may have been modified, so we need to
 586                 # update the encoding, but that's it:
 587                 cmds.push("<first-entry>")
 588                 cmds.push("<update-encoding>")
 589
 590                 # We really just need to be able to assume that at this point,
 591                 # NeoMutt is at position 1, and that we've processed only this
 592                 # part so far. Nevermind about actual attachments, we can
 593                 # safely ignore those as they stay at the end.
 594                 assert state["pos"] == 1
 595                 assert state["parts"] == 1
 596             else:
 597                 # … whereas all other parts need to be added, and they're all
 598                 # considered to be temporary and inline:
 599                 cmds.push(f"<attach-file>{item.path}<enter>")
 600                 cmds.push("<toggle-unlink><toggle-disposition>")
 601
 602                 # This added a part at the end of the list of parts, and that's
 603                 # just how many parts we've seen so far, so it's position in
 604                 # the NeoMutt compose list is the count of parts
 605                 state["parts"] += 1
 606                 state["pos"] = state["parts"]
 607
 608             # If the item (including the original) comes with additional
 609             # information, then we might just as well update the NeoMutt
 610             # tree now:
 611             if item.cid:
 612                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 613
 614             # Now for the biggest hack in this script, which is to handle
 615             # attachments, such as PDFs, that aren't related or alternatives.
 616             # The problem is that when we add an inline image, it always gets
 617             # appended to the list, i.e. inserted *after* other attachments.
 618             # Since we don't know the number of attachments, we also cannot
 619             # infer the postition of the new attachment. Therefore, we bubble
 620             # it all the way to the top, only to then move it down again:
 621             if state["pos"] > 1:  # skip for the first part
 622                 for i in range(max_other_attachments):
 623                     # could use any number here, but has to be larger than the
 624                     # number of possible attachments. The performance
 625                     # difference of using a high number is negligible.
 626                     # Bubble up the new part
 627                     cmds.push(f"<move-up>")
 628
 629                 # As we push the part to the right position in the list (i.e.
 630                 # the last of the subset of attachments this script added), we
 631                 # must handle the situation that subtrees are skipped by
 632                 # NeoMutt. Hence, the actual number of positions to move down
 633                 # is decremented by the number of descendents so far
 634                 # encountered.
 635                 for i in range(1, state["pos"] - len(descendents)):
 636                     cmds.push(f"<move-down>")
 637
 638         elif isinstance(item, Multipart):
 639             # This node has children, but we already visited them (see
 640             # above). The tags dictionary of State should contain a list of
 641             # their positions in the NeoMutt compose window, so iterate those
 642             # and tag the parts there:
 643             n_tags = len(state["tags"][item])
 644             for tag in state["tags"][item]:
 645                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
 646
 647             if item.subtype == "alternative":
 648                 cmds.push("<group-alternatives>")
 649             elif item.subtype in ("relative", "related"):
 650                 cmds.push("<group-related>")
 651             elif item.subtype == "multilingual":
 652                 cmds.push("<group-multilingual>")
 653             else:
 654                 raise NotImplementedError(
 655                     f"Handling of multipart/{item.subtype} is not implemented"
 656                 )
 657
 658             state["pos"] -= n_tags - 1
 659             state["parts"] += 1
 660
 661         else:
 662             # We should never get here
 663             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
 664
 665         # If the item has a description, we might just as well add it
 666         if item.desc:
 667             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 668
 669         if ancestry:
 670             # If there's an ancestry, record the current (assumed) position in
 671             # the NeoMutt compose window as needed-to-tag by our direct parent
 672             # (i.e. the last item of the ancestry)
 673             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
 674
 675             lead = "│ " * (len(ancestry) + 1) + "* "
 676             debugprint(
 677                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
 678                 f"{lead}descendents={[d.subtype for d in descendents]}\n"
 679                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
 680                 f"{lead}pos={state['pos']}, parts={state['parts']}"
 681             )
 682
 683     # -----------------
 684     # End of visitor_fn
 685
 686     # Let's walk the tree and visit every node with our fancy visitor
 687     # function
 688     mimetree.walk(tree, visitor_fn=visitor_fn)
 689
 690     if not only_build:
 691         cmds.push("<send-message>")
 692
 693     # Finally, cleanup. Since we're responsible for removing the temporary
 694     # file, how's this for a little hack?
 695     try:
 696         filename = cmd_f.name
 697     except AttributeError:
 698         filename = "pytest_internal_file"
 699     cmds.cmd(f"source 'rm -f {filename}|'")
 700     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 701     cmds.flush()
 702
 703
 704 # [ CLI ENTRY ] ###############################################################
 705
 706 if __name__ == "__main__":
 707     args = parse_cli_args()
 708
 709     if args.mode is None:
 710         do_setup(
 711             tempdir=args.tempdir,
 712             debug_commands=args.debug_commands,
 713         )
 714
 715     elif args.mode == "massage":
 716         with open(args.MAILDRAFT, "r") as draft_f, open(
 717             args.cmdpath, "w"
 718         ) as cmd_f:
 719             do_massage(
 720                 draft_f,
 721                 pathlib.Path(args.MAILDRAFT),
 722                 cmd_f,
 723                 extensions=args.extensions,
 724                 cssfile=args.css_file,
 725                 related_to_html_only=args.related_to_html_only,
 726                 max_other_attachments=args.max_number_other_attachments,
 727                 only_build=args.only_build,
 728                 tempdir=args.tempdir,
 729                 debug_commands=args.debug_commands,
 730                 debug_walk=args.debug_walk,
 731             )
 732
 733
 734 # [ TESTS ] ###################################################################
 735
 736 try:
 737     import pytest
 738     from io import StringIO
 739
 740     class Tests:
 741         @pytest.fixture
 742         def const1(self):
 743             return "CONSTANT STRING 1"
 744
 745         @pytest.fixture
 746         def const2(self):
 747             return "CONSTANT STRING 2"
 748
 749         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 750         # functions they call, else old stdout is used and not captured
 751
 752         def test_MuttCommands_cmd(self, const1, const2, capsys):
 753             "Assert order of commands"
 754             cmds = MuttCommands(out_f=sys.stdout)
 755             cmds.cmd(const1)
 756             cmds.cmd(const2)
 757             cmds.flush()
 758             captured = capsys.readouterr()
 759             assert captured.out == "\n".join((const1, const2, ""))
 760
 761         def test_MuttCommands_push(self, const1, const2, capsys):
 762             "Assert reverse order of pushes"
 763             cmds = MuttCommands(out_f=sys.stdout)
 764             cmds.push(const1)
 765             cmds.push(const2)
 766             cmds.flush()
 767             captured = capsys.readouterr()
 768             assert (
 769                 captured.out
 770                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 771             )
 772
 773         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 774             "Assert reverse order of pushes"
 775             cmds = MuttCommands(out_f=sys.stdout)
 776             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 777             for i in range(2):
 778                 cmds.cmd(lines[4 * i + 0])
 779                 cmds.cmd(lines[4 * i + 1])
 780                 cmds.push(lines[4 * i + 2])
 781                 cmds.push(lines[4 * i + 3])
 782             cmds.flush()
 783
 784             captured = capsys.readouterr()
 785             lines_out = captured.out.splitlines()
 786             assert lines[0] in lines_out[0]
 787             assert lines[1] in lines_out[1]
 788             assert lines[7] in lines_out[2]
 789             assert lines[6] in lines_out[3]
 790             assert lines[3] in lines_out[4]
 791             assert lines[2] in lines_out[5]
 792             assert lines[4] in lines_out[6]
 793             assert lines[5] in lines_out[7]
 794
 795         @pytest.fixture
 796         def mime_tree_related_to_alternative(self):
 797             return Multipart(
 798                 "relative",
 799                 children=[
 800                     Multipart(
 801                         "alternative",
 802                         children=[
 803                             Part(
 804                                 "text",
 805                                 "plain",
 806                                 "part.txt",
 807                                 desc="Plain",
 808                                 orig=True,
 809                             ),
 810                             Part("text", "html", "part.html", desc="HTML"),
 811                         ],
 812                         desc="Alternative",
 813                     ),
 814                     Part(
 815                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 816                     ),
 817                 ],
 818                 desc="Related",
 819             )
 820
 821         @pytest.fixture
 822         def mime_tree_related_to_html(self):
 823             return Multipart(
 824                 "alternative",
 825                 children=[
 826                     Part(
 827                         "text",
 828                         "plain",
 829                         "part.txt",
 830                         desc="Plain",
 831                         orig=True,
 832                     ),
 833                     Multipart(
 834                         "relative",
 835                         children=[
 836                             Part("text", "html", "part.html", desc="HTML"),
 837                             Part(
 838                                 "text",
 839                                 "png",
 840                                 "logo.png",
 841                                 cid="logo.png",
 842                                 desc="Logo",
 843                             ),
 844                         ],
 845                         desc="Related",
 846                     ),
 847                 ],
 848                 desc="Alternative",
 849             )
 850
 851         def test_MIMETreeDFWalker_depth_first_walk(
 852             self, mime_tree_related_to_alternative
 853         ):
 854             mimetree = MIMETreeDFWalker()
 855
 856             items = []
 857
 858             def visitor_fn(item, ancestry, descendents, debugprint):
 859                 items.append((item, len(ancestry), len(descendents)))
 860
 861             mimetree.walk(
 862                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
 863             )
 864             assert len(items) == 5
 865             assert items[0][0].subtype == "plain"
 866             assert items[0][1] == 2
 867             assert items[0][2] == 0
 868             assert items[1][0].subtype == "html"
 869             assert items[1][1] == 2
 870             assert items[1][2] == 0
 871             assert items[2][0].subtype == "alternative"
 872             assert items[2][1] == 1
 873             assert items[2][2] == 2
 874             assert items[3][0].subtype == "png"
 875             assert items[3][1] == 1
 876             assert items[3][2] == 2
 877             assert items[4][0].subtype == "relative"
 878             assert items[4][1] == 0
 879             assert items[4][2] == 4
 880
 881         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
 882             mimetree = MIMETreeDFWalker()
 883             items = []
 884
 885             def visitor_fn(item, ancestry, descendents, debugprint):
 886                 items.append(item)
 887
 888             p = Part("text", "plain", const1)
 889             mimetree.walk([p], visitor_fn=visitor_fn)
 890             assert items[-1].subtype == "plain"
 891             mimetree.walk([p, p], visitor_fn=visitor_fn)
 892             assert items[-1].subtype == "mixed"
 893
 894         def test_MIMETreeDFWalker_visitor_in_constructor(
 895             self, mime_tree_related_to_alternative
 896         ):
 897             items = []
 898
 899             def visitor_fn(item, ancestry, descendents, debugprint):
 900                 items.append(item)
 901
 902             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 903             mimetree.walk(mime_tree_related_to_alternative)
 904             assert len(items) == 5
 905
 906         @pytest.fixture
 907         def string_io(self, const1, text=None):
 908             return StringIO(text or const1)
 909
 910         def test_do_massage_basic(self, const1, string_io, capsys):
 911             def converter(
 912                 drafttext,
 913                 draftpath,
 914                 cssfile,
 915                 related_to_html_only,
 916                 extensions,
 917                 tempdir,
 918             ):
 919                 return Part("text", "plain", draftpath, orig=True)
 920
 921             do_massage(
 922                 draft_f=string_io,
 923                 draftpath=const1,
 924                 cmd_f=sys.stdout,
 925                 converter=converter,
 926             )
 927
 928             captured = capsys.readouterr()
 929             lines = captured.out.splitlines()
 930             assert '="$my_editor"' in lines.pop(0)
 931             assert '="$my_edit_headers"' in lines.pop(0)
 932             assert "unset my_editor" == lines.pop(0)
 933             assert "unset my_edit_headers" == lines.pop(0)
 934             assert "send-message" in lines.pop(0)
 935             assert "update-encoding" in lines.pop(0)
 936             assert "first-entry" in lines.pop(0)
 937             assert "source 'rm -f " in lines.pop(0)
 938             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 939
 940         def test_do_massage_fulltree(
 941             self, string_io, const1, mime_tree_related_to_alternative, capsys
 942         ):
 943             def converter(
 944                 drafttext,
 945                 draftpath,
 946                 cssfile,
 947                 related_to_html_only,
 948                 extensions,
 949                 tempdir,
 950             ):
 951                 return mime_tree_related_to_alternative
 952
 953             max_attachments = 5
 954             do_massage(
 955                 draft_f=string_io,
 956                 draftpath=const1,
 957                 cmd_f=sys.stdout,
 958                 max_other_attachments=max_attachments,
 959                 converter=converter,
 960             )
 961
 962             captured = capsys.readouterr()
 963             lines = captured.out.splitlines()[4:-2]
 964             assert "first-entry" in lines.pop()
 965             assert "update-encoding" in lines.pop()
 966             assert "Plain" in lines.pop()
 967             assert "part.html" in lines.pop()
 968             assert "toggle-unlink" in lines.pop()
 969             for i in range(max_attachments):
 970                 assert "move-up" in lines.pop()
 971             assert "move-down" in lines.pop()
 972             assert "HTML" in lines.pop()
 973             assert "jump>1" in lines.pop()
 974             assert "jump>2" in lines.pop()
 975             assert "group-alternatives" in lines.pop()
 976             assert "Alternative" in lines.pop()
 977             assert "logo.png" in lines.pop()
 978             assert "toggle-unlink" in lines.pop()
 979             assert "content-id" in lines.pop()
 980             for i in range(max_attachments):
 981                 assert "move-up" in lines.pop()
 982             assert "move-down" in lines.pop()
 983             assert "Logo" in lines.pop()
 984             assert "jump>1" in lines.pop()
 985             assert "jump>4" in lines.pop()
 986             assert "group-related" in lines.pop()
 987             assert "Related" in lines.pop()
 988             assert "send-message" in lines.pop()
 989             assert len(lines) == 0
 990
 991         @pytest.fixture
 992         def fake_filewriter(self):
 993             class FileWriter:
 994                 def __init__(self):
 995                     self._writes = []
 996
 997                 def __call__(self, path, content, mode="w", **kwargs):
 998                     self._writes.append((path, content))
 999
1000                 def pop(self, index=-1):
1001                     return self._writes.pop(index)
1002
1003             return FileWriter()
1004
1005         @pytest.fixture
1006         def markdown_non_converter(self, const1, const2):
1007             return lambda s, text: f"{const1}{text}{const2}"
1008
1009         def test_converter_tree_basic(self, const1, const2, fake_filewriter):
1010             path = pathlib.Path(const2)
1011             tree = convert_markdown_to_html(
1012                 const1, path, filewriter_fn=fake_filewriter
1013             )
1014
1015             assert tree.subtype == "alternative"
1016             assert len(tree.children) == 2
1017             assert tree.children[0].subtype == "plain"
1018             assert tree.children[0].path == path
1019             assert tree.children[0].orig
1020             assert tree.children[1].subtype == "html"
1021             assert tree.children[1].path == path.with_suffix(".html")
1022
1023         def test_converter_writes(
1024             self,
1025             const1,
1026             const2,
1027             fake_filewriter,
1028             monkeypatch,
1029             markdown_non_converter,
1030         ):
1031             path = pathlib.Path(const2)
1032
1033             with monkeypatch.context() as m:
1034                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
1035                 convert_markdown_to_html(
1036                     const1, path, filewriter_fn=fake_filewriter
1037                 )
1038
1039             assert (path, const1) == fake_filewriter.pop(0)
1040             assert (
1041                 path.with_suffix(".html"),
1042                 markdown_non_converter(None, const1),
1043             ) == fake_filewriter.pop(0)
1044
1045         def test_markdown_inline_image_processor(self):
1046             imgpath1 = "file:/path/to/image.png"
1047             imgpath2 = "file:///path/to/image.png?url=params"
1048             imgpath3 = "/path/to/image.png"
1049             text = f"""![inline local image]({imgpath1})
1050                        ![image inlined
1051                          with newline]({imgpath2})
1052                        ![image local path]({imgpath3})"""
1053             text, html, images = markdown_with_inline_image_support(text)
1054
1055             # local paths have been normalised to URLs:
1056             imgpath3 = f"file://{imgpath3}"
1057
1058             assert 'src="cid:' in html
1059             assert "](cid:" in text
1060             assert len(images) == 3
1061             assert imgpath1 in images
1062             assert imgpath2 in images
1063             assert imgpath3 in images
1064             assert images[imgpath1].cid != images[imgpath2].cid
1065             assert images[imgpath1].cid != images[imgpath3].cid
1066             assert images[imgpath2].cid != images[imgpath3].cid
1067
1068         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1069             imgpath = "file:///path/to/image.png"
1070             text = f'![inline local image]({imgpath} "{const1}")'
1071             text, html, images = markdown_with_inline_image_support(text)
1072             assert images[imgpath].desc == const1
1073
1074         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1075             imgpath = "file:///path/to/image.png"
1076             text = f"![{const1}]({imgpath})"
1077             text, html, images = markdown_with_inline_image_support(text)
1078             assert images[imgpath].desc == const1
1079
1080         def test_markdown_inline_image_processor_title_over_alt_desc(
1081             self, const1, const2
1082         ):
1083             imgpath = "file:///path/to/image.png"
1084             text = f'![{const1}]({imgpath} "{const2}")'
1085             text, html, images = markdown_with_inline_image_support(text)
1086             assert images[imgpath].desc == const2
1087
1088         def test_markdown_inline_image_not_external(self):
1089             imgpath = "https://path/to/image.png"
1090             text = f"![inline image]({imgpath})"
1091             text, html, images = markdown_with_inline_image_support(text)
1092
1093             assert 'src="cid:' not in html
1094             assert "](cid:" not in text
1095             assert len(images) == 0
1096
1097         def test_markdown_inline_image_local_file(self):
1098             imgpath = "/path/to/image.png"
1099             text = f"![inline image]({imgpath})"
1100             text, html, images = markdown_with_inline_image_support(text)
1101
1102             for k, v in images.items():
1103                 assert k == f"file://{imgpath}"
1104                 break
1105
1106         @pytest.fixture
1107         def test_png(self):
1108             return (
1109                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1110                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1111             )
1112
1113         def test_markdown_inline_image_processor_base64(self, test_png):
1114             text = f"![1px white inlined]({test_png})"
1115             text, html, images = markdown_with_inline_image_support(text)
1116
1117             assert 'src="cid:' in html
1118             assert "](cid:" in text
1119             assert len(images) == 1
1120             assert test_png in images
1121
1122         def test_converter_tree_inline_image_base64(
1123             self, test_png, const1, fake_filewriter
1124         ):
1125             text = f"![inline base64 image]({test_png})"
1126             path = pathlib.Path(const1)
1127             tree = convert_markdown_to_html(
1128                 text,
1129                 path,
1130                 filewriter_fn=fake_filewriter,
1131                 related_to_html_only=False,
1132             )
1133             assert tree.subtype == "relative"
1134             assert tree.children[0].subtype == "alternative"
1135             assert tree.children[1].subtype == "png"
1136             written = fake_filewriter.pop()
1137             assert tree.children[1].path == written[0]
1138             assert written[1] == request.urlopen(test_png).read()
1139
1140         def test_converter_tree_inline_image_base64_related_to_html(
1141             self, test_png, const1, fake_filewriter
1142         ):
1143             text = f"![inline base64 image]({test_png})"
1144             path = pathlib.Path(const1)
1145             tree = convert_markdown_to_html(
1146                 text,
1147                 path,
1148                 filewriter_fn=fake_filewriter,
1149                 related_to_html_only=True,
1150             )
1151             assert tree.subtype == "alternative"
1152             assert tree.children[1].subtype == "relative"
1153             assert tree.children[1].children[1].subtype == "png"
1154             written = fake_filewriter.pop()
1155             assert tree.children[1].children[1].path == written[0]
1156             assert written[1] == request.urlopen(test_png).read()
1157
1158         def test_converter_tree_inline_image_cid(
1159             self, const1, fake_filewriter
1160         ):
1161             text = f"![inline base64 image](cid:{const1})"
1162             path = pathlib.Path(const1)
1163             tree = convert_markdown_to_html(
1164                 text,
1165                 path,
1166                 filewriter_fn=fake_filewriter,
1167                 related_to_html_only=False,
1168             )
1169             assert len(tree.children) == 2
1170             assert tree.children[0].cid != const1
1171             assert tree.children[0].type != "image"
1172             assert tree.children[1].cid != const1
1173             assert tree.children[1].type != "image"
1174
1175         def test_inline_image_collection(
1176             self, test_png, const1, const2, fake_filewriter
1177         ):
1178             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1179             relparts = collect_inline_images(
1180                 test_images, filewriter_fn=fake_filewriter
1181             )
1182
1183             written = fake_filewriter.pop()
1184             assert b"PNG" in written[1]
1185
1186             assert relparts[0].subtype == "png"
1187             assert relparts[0].path == written[0]
1188             assert relparts[0].cid == const1
1189             assert relparts[0].desc.endswith(const2)
1190
1191         def test_apply_stylesheet(self):
1192             if _PYNLINER:
1193                 html = "<p>Hello, world!</p>"
1194                 css = "p { color:red }"
1195                 out = apply_styling(html, css)
1196                 assert 'p style="color' in out
1197
1198         def test_apply_stylesheet_pygments(self):
1199             if _PYGMENTS_CSS:
1200                 html = (
1201                     f'<div class="{_CODEHILITE_CLASS}">'
1202                     "<pre>def foo():\n    return</pre></div>"
1203                 )
1204                 out = apply_styling(html, _PYGMENTS_CSS)
1205                 assert f'{_CODEHILITE_CLASS}" style="' in out
1206
1207         def test_mime_tree_relative_within_alternative(
1208             self, string_io, const1, capsys, mime_tree_related_to_html
1209         ):
1210             def converter(
1211                 drafttext,
1212                 draftpath,
1213                 cssfile,
1214                 related_to_html_only,
1215                 extensions,
1216                 tempdir,
1217             ):
1218                 return mime_tree_related_to_html
1219
1220             do_massage(
1221                 draft_f=string_io,
1222                 draftpath=const1,
1223                 cmd_f=sys.stdout,
1224                 converter=converter,
1225             )
1226
1227             captured = capsys.readouterr()
1228             lines = captured.out.splitlines()[4:-2]
1229             assert "first-entry" in lines.pop()
1230             assert "update-encoding" in lines.pop()
1231             assert "Plain" in lines.pop()
1232             assert "part.html" in lines.pop()
1233             assert "toggle-unlink" in lines.pop()
1234             assert "move-up" in lines.pop()
1235             while True:
1236                 top = lines.pop()
1237                 if "move-up" not in top:
1238                     break
1239             assert "move-down" in top
1240             assert "HTML" in lines.pop()
1241             assert "logo.png" in lines.pop()
1242             assert "toggle-unlink" in lines.pop()
1243             assert "content-id" in lines.pop()
1244             assert "move-up" in lines.pop()
1245             while True:
1246                 top = lines.pop()
1247                 if "move-up" not in top:
1248                     break
1249             assert "move-down" in top
1250             assert "move-down" in lines.pop()
1251             assert "Logo" in lines.pop()
1252             assert "jump>2" in lines.pop()
1253             assert "jump>3" in lines.pop()
1254             assert "group-related" in lines.pop()
1255             assert "Related" in lines.pop()
1256             assert "jump>1" in lines.pop()
1257             assert "jump>2" in lines.pop()
1258             assert "group-alternative" in lines.pop()
1259             assert "Alternative" in lines.pop()
1260             assert "send-message" in lines.pop()
1261             assert len(lines) == 0
1262
1263         def test_mime_tree_nested_trees_does_not_break_positioning(
1264             self, string_io, const1, capsys
1265         ):
1266             def converter(
1267                 drafttext,
1268                 draftpath,
1269                 cssfile,
1270                 related_to_html_only,
1271                 extensions,
1272                 tempdir,
1273             ):
1274                 return Multipart(
1275                     "relative",
1276                     children=[
1277                         Multipart(
1278                             "alternative",
1279                             children=[
1280                                 Part(
1281                                     "text",
1282                                     "plain",
1283                                     "part.txt",
1284                                     desc="Plain",
1285                                     orig=True,
1286                                 ),
1287                                 Multipart(
1288                                     "alternative",
1289                                     children=[
1290                                         Part(
1291                                             "text",
1292                                             "plain",
1293                                             "part.txt",
1294                                             desc="Nested plain",
1295                                         ),
1296                                         Part(
1297                                             "text",
1298                                             "html",
1299                                             "part.html",
1300                                             desc="Nested HTML",
1301                                         ),
1302                                     ],
1303                                     desc="Nested alternative",
1304                                 ),
1305                             ],
1306                             desc="Alternative",
1307                         ),
1308                         Part(
1309                             "text",
1310                             "png",
1311                             "logo.png",
1312                             cid="logo.png",
1313                             desc="Logo",
1314                         ),
1315                     ],
1316                     desc="Related",
1317                 )
1318
1319             do_massage(
1320                 draft_f=string_io,
1321                 draftpath=const1,
1322                 cmd_f=sys.stdout,
1323                 converter=converter,
1324             )
1325
1326             captured = capsys.readouterr()
1327             lines = captured.out.splitlines()
1328             while not "logo.png" in lines.pop():
1329                 pass
1330             lines.pop()
1331             assert "content-id" in lines.pop()
1332             assert "move-up" in lines.pop()
1333             while True:
1334                 top = lines.pop()
1335                 if "move-up" not in top:
1336                     break
1337             assert "move-down" in top
1338             # Due to the nested trees, the number of descendents of the sibling
1339             # actually needs to be considered, not just the nieces. So to move
1340             # from position 1 to position 6, it only needs one <move-down>
1341             # because that jumps over the entire sibling tree. Thus what
1342             # follows next must not be another <move-down>
1343             assert "Logo" in lines.pop()
1344
1345 except ImportError:
1346     pass