.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--related-to-html-only",
  81         action="store_true",
  82         help="Make related content be sibling to HTML parts only"
  83     )
  84
  85     parser.add_argument(
  86         "--only-build",
  87         action="store_true",
  88         help="Only build, don't send the message",
  89     )
  90
  91     parser.add_argument(
  92         "--tempdir",
  93         default=None,
  94         help="Specify temporary directory to use for attachments",
  95     )
  96
  97     parser.add_argument(
  98         "--debug-commands",
  99         action="store_true",
 100         help="Turn on debug logging of commands generated to stderr",
 101     )
 102
 103     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
 104     massage_p = subp.add_parser(
 105         "massage", help="Massaging phase (internal use)"
 106     )
 107
 108     massage_p.add_argument(
 109         "--write-commands-to",
 110         "-o",
 111         metavar="PATH",
 112         dest="cmdpath",
 113         required=True,
 114         help="Temporary file path to write commands to",
 115     )
 116
 117     massage_p.add_argument(
 118         "--debug-walk",
 119         action="store_true",
 120         help="Turn on debugging to stderr of the MIME tree walk",
 121     )
 122
 123     massage_p.add_argument(
 124         "MAILDRAFT",
 125         nargs="?",
 126         help="If provided, the script is invoked as editor on the mail draft",
 127     )
 128
 129     return parser.parse_args(*args, **kwargs)
 130
 131
 132 # [ MARKDOWN WRAPPING ] #######################################################
 133
 134
 135 InlineImageInfo = namedtuple(
 136     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 137 )
 138
 139
 140 class InlineImageExtension(Extension):
 141     class RelatedImageInlineProcessor(ImageInlineProcessor):
 142         def __init__(self, re, md, ext):
 143             super().__init__(re, md)
 144             self._ext = ext
 145
 146         def handleMatch(self, m, data):
 147             el, start, end = super().handleMatch(m, data)
 148             if "src" in el.attrib:
 149                 src = el.attrib["src"]
 150                 if "://" not in src or src.startswith("file://"):
 151                     # We only inline local content
 152                     cid = self._ext.get_cid_for_image(el.attrib)
 153                     el.attrib["src"] = f"cid:{cid}"
 154             return el, start, end
 155
 156     def __init__(self):
 157         super().__init__()
 158         self._images = OrderedDict()
 159
 160     def extendMarkdown(self, md):
 161         md.registerExtension(self)
 162         inline_image_proc = self.RelatedImageInlineProcessor(
 163             IMAGE_LINK_RE, md, self
 164         )
 165         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 166
 167     def get_cid_for_image(self, attrib):
 168         msgid = make_msgid()[1:-1]
 169         path = attrib["src"]
 170         if path.startswith("/"):
 171             path = f"file://{path}"
 172         self._images[path] = InlineImageInfo(
 173             msgid, attrib.get("title", attrib.get("alt"))
 174         )
 175         return msgid
 176
 177     def get_images(self):
 178         return self._images
 179
 180
 181 def markdown_with_inline_image_support(
 182     text, *, extensions=None, extension_configs=None
 183 ):
 184     inline_image_handler = InlineImageExtension()
 185     extensions = extensions or []
 186     extensions.append(inline_image_handler)
 187     mdwn = markdown.Markdown(
 188         extensions=extensions, extension_configs=extension_configs
 189     )
 190     htmltext = mdwn.convert(text)
 191
 192     images = inline_image_handler.get_images()
 193
 194     def replace_image_with_cid(matchobj):
 195         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 196             if m in images:
 197                 return f"(cid:{images[m].cid}"
 198         return matchobj.group(0)
 199
 200     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 201     return text, htmltext, images
 202
 203
 204 # [ CSS STYLING ] #############################################################
 205
 206 try:
 207     import pynliner
 208
 209     _PYNLINER = True
 210
 211 except ImportError:
 212     _PYNLINER = False
 213
 214 try:
 215     from pygments.formatters import get_formatter_by_name
 216
 217     _CODEHILITE_CLASS = "codehilite"
 218
 219     _PYGMENTS_CSS = get_formatter_by_name(
 220         "html", style="default"
 221     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 222
 223 except ImportError:
 224     _PYGMENTS_CSS = None
 225
 226
 227 def apply_styling(html, css):
 228     return (
 229         pynliner.Pynliner()
 230         .from_string(html)
 231         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 232         .run()
 233     )
 234
 235
 236 # [ PARTS GENERATION ] ########################################################
 237
 238
 239 class Part(
 240     namedtuple(
 241         "Part",
 242         ["type", "subtype", "path", "desc", "cid", "orig"],
 243         defaults=[None, None, False],
 244     )
 245 ):
 246     def __str__(self):
 247         ret = f"<{self.type}/{self.subtype}>"
 248         if self.cid:
 249             ret = f"{ret} cid:{self.cid}"
 250         if self.orig:
 251             ret = f"{ret} ORIGINAL"
 252         return ret
 253
 254
 255 class Multipart(
 256     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 257 ):
 258     def __str__(self):
 259         return f"<multipart/{self.subtype}> children={len(self.children)}"
 260
 261     def __hash__(self):
 262         return hash(str(self.subtype) + "".join(str(self.children)))
 263
 264
 265 def filewriter_fn(path, content, mode="w", **kwargs):
 266     with open(path, mode, **kwargs) as out_f:
 267         out_f.write(content)
 268
 269
 270 def collect_inline_images(
 271     images, *, tempdir=None, filewriter_fn=filewriter_fn
 272 ):
 273     relparts = []
 274     for path, info in images.items():
 275         if path.startswith("cid:"):
 276             continue
 277
 278         data = request.urlopen(path)
 279
 280         mimetype = data.headers["Content-Type"]
 281         ext = mimetypes.guess_extension(mimetype)
 282         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 283         path = pathlib.Path(tempfilename[1])
 284
 285         filewriter_fn(path, data.read(), "w+b")
 286
 287         relparts.append(
 288             Part(
 289                 *mimetype.split("/"),
 290                 path,
 291                 cid=info.cid,
 292                 desc=f"Image: {info.desc}",
 293             )
 294         )
 295
 296     return relparts
 297
 298
 299 def convert_markdown_to_html(
 300     origtext,
 301     draftpath,
 302     *,
 303     related_to_html_only=False,
 304     cssfile=None,
 305     filewriter_fn=filewriter_fn,
 306     tempdir=None,
 307     extensions=None,
 308     extension_configs=None,
 309 ):
 310     # TODO extension_configs need to be handled differently
 311     extension_configs = extension_configs or {}
 312     extension_configs.setdefault("pymdownx.highlight", {})
 313     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 314
 315     origtext, htmltext, images = markdown_with_inline_image_support(
 316         origtext, extensions=extensions, extension_configs=extension_configs
 317     )
 318
 319     filewriter_fn(draftpath, origtext, encoding="utf-8")
 320     textpart = Part(
 321         "text", "plain", draftpath, "Plain-text version", orig=True
 322     )
 323
 324     htmltext = apply_styling(htmltext, cssfile)
 325
 326     htmlpath = draftpath.with_suffix(".html")
 327     filewriter_fn(
 328         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 329     )
 330     htmlpart = Part("text", "html", htmlpath, "HTML version")
 331
 332     imgparts = collect_inline_images(
 333         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 334     )
 335
 336     if related_to_html_only:
 337         # If there are inline image part, they will be contained within a
 338         # multipart/related part along with the HTML part only
 339         if imgparts:
 340             # replace htmlpart with a multipart/related container of the HTML
 341             # parts and the images
 342             htmlpart = Multipart(
 343                 "relative", [htmlpart] + imgparts, "Group of related content"
 344             )
 345
 346         return Multipart(
 347             "alternative", [textpart, htmlpart], "Group of alternative content"
 348         )
 349
 350     else:
 351         # If there are inline image part, they will be siblings to the
 352         # multipart/alternative tree within a multipart/related part
 353         altpart = Multipart(
 354             "alternative", [textpart, htmlpart], "Group of alternative content"
 355         )
 356         if imgparts:
 357             return Multipart(
 358                 "relative", [altpart] + imgparts, "Group of related content"
 359             )
 360         else:
 361             return altpart
 362
 363
 364 class MIMETreeDFWalker:
 365     def __init__(self, *, visitor_fn=None, debug=False):
 366         self._visitor_fn = visitor_fn or self._echovisit
 367         self._debug = debug
 368
 369     def _echovisit(self, node, ancestry, debugprint):
 370         debugprint(f"node={node} ancestry={ancestry}")
 371
 372     def walk(self, root, *, visitor_fn=None):
 373         """
 374         Recursive function to implement a depth-dirst walk of the MIME-tree
 375         rooted at `root`.
 376         """
 377         if isinstance(root, list):
 378             if len(root) > 1:
 379                 root = Multipart("mixed", children=root)
 380             else:
 381                 root = root[0]
 382
 383         self._walk(
 384             root,
 385             ancestry=[],
 386             visitor_fn=visitor_fn or self._visitor_fn,
 387         )
 388
 389     def _walk(self, node, *, ancestry, visitor_fn):
 390         # Let's start by enumerating the parts at the current level. At the
 391         # root level, ancestry will be the empty list, and we expect a
 392         # multipart/* container at this level. Later, e.g. within a
 393         # mutlipart/alternative container, the subtree will just be the
 394         # alternative parts, while the top of the ancestry will be the
 395         # multipart/alternative container, which we will process after the
 396         # following loop.
 397
 398         lead = f"{'│ '*len(ancestry)}"
 399         if isinstance(node, Multipart):
 400             self.debugprint(
 401                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
 402             )
 403
 404             # Depth-first, so push the current container onto the ancestry
 405             # stack, then descend …
 406             ancestry.append(node)
 407             self.debugprint(lead + "│ " * 2)
 408             for child in node.children:
 409                 self._walk(
 410                     child,
 411                     ancestry=ancestry,
 412                     visitor_fn=visitor_fn,
 413                 )
 414             assert ancestry.pop() == node
 415
 416         else:
 417             self.debugprint(f"{lead}├{node}")
 418
 419         if False and ancestry:
 420             self.debugprint(lead[:-1] + " │")
 421
 422         if visitor_fn:
 423             visitor_fn(node, ancestry, debugprint=self.debugprint)
 424
 425     def debugprint(self, s, **kwargs):
 426         if self._debug:
 427             print(s, file=sys.stderr, **kwargs)
 428
 429
 430 # [ RUN MODES ] ###############################################################
 431
 432
 433 class MuttCommands:
 434     """
 435     Stupid class to interface writing out Mutt commands. This is quite a hack
 436     to deal with the fact that Mutt runs "push" commands in reverse order, so
 437     all of a sudden, things become very complicated when mixing with "real"
 438     commands.
 439
 440     Hence we keep two sets of commands, and one set of pushes. Commands are
 441     added to the first until a push is added, after which commands are added to
 442     the second set of commands.
 443
 444     On flush(), the first set is printed, followed by the pushes in reverse,
 445     and then the second set is printed. All 3 sets are then cleared.
 446     """
 447
 448     def __init__(self, out_f=sys.stdout, *, debug=False):
 449         self._cmd1, self._push, self._cmd2 = [], [], []
 450         self._out_f = out_f
 451         self._debug = debug
 452
 453     def cmd(self, s):
 454         self.debugprint(s)
 455         if self._push:
 456             self._cmd2.append(s)
 457         else:
 458             self._cmd1.append(s)
 459
 460     def push(self, s):
 461         s = s.replace('"', '"')
 462         s = f'push "{s}"'
 463         self.debugprint(s)
 464         self._push.insert(0, s)
 465
 466     def flush(self):
 467         print(
 468             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 469         )
 470         self._cmd1, self._push, self._cmd2 = [], [], []
 471
 472     def debugprint(self, s, **kwargs):
 473         if self._debug:
 474             print(s, file=sys.stderr, **kwargs)
 475
 476
 477 def do_setup(
 478     *,
 479     out_f=sys.stdout,
 480     temppath=None,
 481     tempdir=None,
 482     debug_commands=False,
 483 ):
 484     temppath = temppath or pathlib.Path(
 485         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 486     )
 487     cmds = MuttCommands(out_f, debug=debug_commands)
 488
 489     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 490
 491     cmds.cmd('set my_editor="$editor"')
 492     cmds.cmd('set my_edit_headers="$edit_headers"')
 493     cmds.cmd(f'set editor="{editor}"')
 494     cmds.cmd("unset edit_headers")
 495     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 496     cmds.push("<first-entry><edit-file>")
 497     cmds.flush()
 498
 499
 500 def do_massage(
 501     draft_f,
 502     draftpath,
 503     cmd_f,
 504     *,
 505     extensions=None,
 506     cssfile=None,
 507     converter=convert_markdown_to_html,
 508     related_to_html_only=True,
 509     only_build=False,
 510     tempdir=None,
 511     debug_commands=False,
 512     debug_walk=False,
 513 ):
 514     # Here's the big picture: we're being invoked as the editor on the email
 515     # draft, and whatever commands we write to the file given as cmdpath will
 516     # be run by the second source command in the macro definition.
 517
 518     # Let's start by cleaning up what the setup did (see above), i.e. we
 519     # restore the $editor and $edit_headers variables, and also unset the
 520     # variable used to identify the command file we're currently writing
 521     # to.
 522     cmds = MuttCommands(cmd_f, debug=debug_commands)
 523     cmds.cmd('set editor="$my_editor"')
 524     cmds.cmd('set edit_headers="$my_edit_headers"')
 525     cmds.cmd("unset my_editor")
 526     cmds.cmd("unset my_edit_headers")
 527
 528     # let's flush those commands, as there'll be a lot of pushes from now
 529     # on, which need to be run in reverse order
 530     cmds.flush()
 531
 532     extensions = extensions.split(",") if extensions else []
 533     tree = converter(
 534         draft_f.read(),
 535         draftpath,
 536         cssfile=cssfile,
 537         related_to_html_only=related_to_html_only,
 538         tempdir=tempdir,
 539         extensions=extensions,
 540     )
 541
 542     mimetree = MIMETreeDFWalker(debug=debug_walk)
 543
 544     state = dict(pos=1, tags={}, parts=1)
 545
 546     def visitor_fn(item, ancestry, *, debugprint=None):
 547         """
 548         Visitor function called for every node (part) of the MIME tree,
 549         depth-first, and responsible for telling NeoMutt how to assemble
 550         the tree.
 551         """
 552         KILL_LINE = r"\Ca\Ck"
 553
 554         if isinstance(item, Part):
 555             # We've hit a leaf-node, i.e. an alternative or a related part
 556             # with actual content.
 557
 558             # Let's add the part
 559             if item.orig:
 560                 # The original source already exists in the NeoMutt tree, but
 561                 # the underlying file may have been modified, so we need to
 562                 # update the encoding, but that's it:
 563                 cmds.push("<first-entry>")
 564                 cmds.push("<update-encoding>")
 565
 566                 # We really just need to be able to assume that at this point,
 567                 # NeoMutt is at position 1, and that we've processed only this
 568                 # part so far. Nevermind about actual attachments, we can
 569                 # safely ignore those as they stay at the end.
 570                 assert state["pos"] == 1
 571                 assert state["parts"] == 1
 572             else:
 573                 # … whereas all other parts need to be added, and they're all
 574                 # considered to be temporary and inline:
 575                 cmds.push(f"<attach-file>{item.path}<enter>")
 576                 cmds.push("<toggle-unlink><toggle-disposition>")
 577
 578                 # This added a part at the end of the list of parts, and that's
 579                 # just how many parts we've seen so far, so it's position in
 580                 # the NeoMutt compose list is the count of parts
 581                 state["parts"] += 1
 582                 state["pos"] = state["parts"]
 583
 584             # If the item (including the original) comes with additional
 585             # information, then we might just as well update the NeoMutt
 586             # tree now:
 587             if item.cid:
 588                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 589
 590         elif isinstance(item, Multipart):
 591             # This node has children, but we already visited them (see
 592             # above). The tags dictionary of State should contain a list of
 593             # their positions in the NeoMutt compose window, so iterate those
 594             # and tag the parts there:
 595             for tag in state["tags"][item]:
 596                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
 597
 598             if item.subtype == "alternative":
 599                 cmds.push("<group-alternatives>")
 600             elif item.subtype in ("relative", "related"):
 601                 cmds.push("<group-related>")
 602             elif item.subtype == "multilingual":
 603                 cmds.push("<group-multilingual>")
 604             else:
 605                 raise NotImplementedError(
 606                     f"Handling of multipart/{item.subtype} is not implemented"
 607                 )
 608
 609             state["pos"] -= len(state["tags"][item]) - 1
 610             state["parts"] += 1
 611             del state["tags"][item]
 612
 613         else:
 614             # We should never get here
 615             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
 616
 617         # If the item has a description, we might just as well add it
 618         if item.desc:
 619             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 620
 621         if ancestry:
 622             # If there's an ancestry, record the current (assumed) position in
 623             # the NeoMutt compose window as needed-to-tag by our direct parent
 624             # (i.e. the last item of the ancestry)
 625             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
 626
 627             lead = "│ " * (len(ancestry) + 1) + "* "
 628             debugprint(
 629                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
 630                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
 631                 f"{lead}pos={state['pos']}, parts={state['parts']}"
 632             )
 633
 634     # -----------------
 635     # End of visitor_fn
 636
 637     # Let's walk the tree and visit every node with our fancy visitor
 638     # function
 639     mimetree.walk(tree, visitor_fn=visitor_fn)
 640
 641     if not only_build:
 642         cmds.push("<send-message>")
 643
 644     # Finally, cleanup. Since we're responsible for removing the temporary
 645     # file, how's this for a little hack?
 646     try:
 647         filename = cmd_f.name
 648     except AttributeError:
 649         filename = "pytest_internal_file"
 650     cmds.cmd(f"source 'rm -f {filename}|'")
 651     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 652     cmds.flush()
 653
 654
 655 # [ CLI ENTRY ] ###############################################################
 656
 657 if __name__ == "__main__":
 658     args = parse_cli_args()
 659
 660     if args.mode is None:
 661         do_setup(
 662             tempdir=args.tempdir,
 663             debug_commands=args.debug_commands,
 664         )
 665
 666     elif args.mode == "massage":
 667         with open(args.MAILDRAFT, "r") as draft_f, open(
 668             args.cmdpath, "w"
 669         ) as cmd_f:
 670             do_massage(
 671                 draft_f,
 672                 pathlib.Path(args.MAILDRAFT),
 673                 cmd_f,
 674                 extensions=args.extensions,
 675                 cssfile=args.css_file,
 676                 related_to_html_only=args.related_to_html_only,
 677                 only_build=args.only_build,
 678                 tempdir=args.tempdir,
 679                 debug_commands=args.debug_commands,
 680                 debug_walk=args.debug_walk,
 681             )
 682
 683
 684 # [ TESTS ] ###################################################################
 685
 686 try:
 687     import pytest
 688     from io import StringIO
 689
 690     class Tests:
 691         @pytest.fixture
 692         def const1(self):
 693             return "CONSTANT STRING 1"
 694
 695         @pytest.fixture
 696         def const2(self):
 697             return "CONSTANT STRING 2"
 698
 699         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 700         # functions they call, else old stdout is used and not captured
 701
 702         def test_MuttCommands_cmd(self, const1, const2, capsys):
 703             "Assert order of commands"
 704             cmds = MuttCommands(out_f=sys.stdout)
 705             cmds.cmd(const1)
 706             cmds.cmd(const2)
 707             cmds.flush()
 708             captured = capsys.readouterr()
 709             assert captured.out == "\n".join((const1, const2, ""))
 710
 711         def test_MuttCommands_push(self, const1, const2, capsys):
 712             "Assert reverse order of pushes"
 713             cmds = MuttCommands(out_f=sys.stdout)
 714             cmds.push(const1)
 715             cmds.push(const2)
 716             cmds.flush()
 717             captured = capsys.readouterr()
 718             assert (
 719                 captured.out
 720                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 721             )
 722
 723         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 724             "Assert reverse order of pushes"
 725             cmds = MuttCommands(out_f=sys.stdout)
 726             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 727             for i in range(2):
 728                 cmds.cmd(lines[4 * i + 0])
 729                 cmds.cmd(lines[4 * i + 1])
 730                 cmds.push(lines[4 * i + 2])
 731                 cmds.push(lines[4 * i + 3])
 732             cmds.flush()
 733
 734             captured = capsys.readouterr()
 735             lines_out = captured.out.splitlines()
 736             assert lines[0] in lines_out[0]
 737             assert lines[1] in lines_out[1]
 738             assert lines[7] in lines_out[2]
 739             assert lines[6] in lines_out[3]
 740             assert lines[3] in lines_out[4]
 741             assert lines[2] in lines_out[5]
 742             assert lines[4] in lines_out[6]
 743             assert lines[5] in lines_out[7]
 744
 745         @pytest.fixture
 746         def mime_tree_related_to_alternative(self):
 747             return Multipart(
 748                 "relative",
 749                 children=[
 750                     Multipart(
 751                         "alternative",
 752                         children=[
 753                             Part(
 754                                 "text",
 755                                 "plain",
 756                                 "part.txt",
 757                                 desc="Plain",
 758                                 orig=True,
 759                             ),
 760                             Part("text", "html", "part.html", desc="HTML"),
 761                         ],
 762                         desc="Alternative",
 763                     ),
 764                     Part(
 765                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 766                     ),
 767                 ],
 768                 desc="Related",
 769             )
 770
 771         @pytest.fixture
 772         def mime_tree_related_to_html(self):
 773             return Multipart(
 774                 "alternative",
 775                 children=[
 776                     Part(
 777                         "text",
 778                         "plain",
 779                         "part.txt",
 780                         desc="Plain",
 781                         orig=True,
 782                     ),
 783                     Multipart(
 784                         "relative",
 785                         children=[
 786                             Part("text", "html", "part.html", desc="HTML"),
 787                             Part(
 788                                 "text",
 789                                 "png",
 790                                 "logo.png",
 791                                 cid="logo.png",
 792                                 desc="Logo",
 793                             ),
 794                         ],
 795                         desc="Related",
 796                     ),
 797                 ],
 798                 desc="Alternative",
 799             )
 800
 801         def test_MIMETreeDFWalker_depth_first_walk(
 802             self, mime_tree_related_to_alternative
 803         ):
 804             mimetree = MIMETreeDFWalker()
 805
 806             items = []
 807
 808             def visitor_fn(item, ancestry, debugprint):
 809                 items.append((item, len(ancestry)))
 810
 811             mimetree.walk(
 812                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
 813             )
 814             assert len(items) == 5
 815             assert items[0][0].subtype == "plain"
 816             assert items[0][1] == 2
 817             assert items[1][0].subtype == "html"
 818             assert items[1][1] == 2
 819             assert items[2][0].subtype == "alternative"
 820             assert items[2][1] == 1
 821             assert items[3][0].subtype == "png"
 822             assert items[3][1] == 1
 823             assert items[4][0].subtype == "relative"
 824             assert items[4][1] == 0
 825
 826         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
 827             mimetree = MIMETreeDFWalker()
 828             items = []
 829
 830             def visitor_fn(item, ancestry, debugprint):
 831                 items.append(item)
 832
 833             p = Part("text", "plain", const1)
 834             mimetree.walk([p], visitor_fn=visitor_fn)
 835             assert items[-1].subtype == "plain"
 836             mimetree.walk([p, p], visitor_fn=visitor_fn)
 837             assert items[-1].subtype == "mixed"
 838
 839         def test_MIMETreeDFWalker_visitor_in_constructor(
 840             self, mime_tree_related_to_alternative
 841         ):
 842             items = []
 843
 844             def visitor_fn(item, ancestry, debugprint):
 845                 items.append(item)
 846
 847             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 848             mimetree.walk(mime_tree_related_to_alternative)
 849             assert len(items) == 5
 850
 851         @pytest.fixture
 852         def string_io(self, const1, text=None):
 853             return StringIO(text or const1)
 854
 855         def test_do_massage_basic(self, const1, string_io, capsys):
 856             def converter(
 857                 drafttext,
 858                 draftpath,
 859                 cssfile,
 860                 related_to_html_only,
 861                 extensions,
 862                 tempdir,
 863             ):
 864                 return Part("text", "plain", draftpath, orig=True)
 865
 866             do_massage(
 867                 draft_f=string_io,
 868                 draftpath=const1,
 869                 cmd_f=sys.stdout,
 870                 converter=converter,
 871             )
 872
 873             captured = capsys.readouterr()
 874             lines = captured.out.splitlines()
 875             assert '="$my_editor"' in lines.pop(0)
 876             assert '="$my_edit_headers"' in lines.pop(0)
 877             assert "unset my_editor" == lines.pop(0)
 878             assert "unset my_edit_headers" == lines.pop(0)
 879             assert "send-message" in lines.pop(0)
 880             assert "update-encoding" in lines.pop(0)
 881             assert "first-entry" in lines.pop(0)
 882             assert "source 'rm -f " in lines.pop(0)
 883             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 884
 885         def test_do_massage_fulltree(
 886             self, string_io, const1, mime_tree_related_to_alternative, capsys
 887         ):
 888             def converter(
 889                 drafttext,
 890                 draftpath,
 891                 cssfile,
 892                 related_to_html_only,
 893                 extensions,
 894                 tempdir,
 895             ):
 896                 return mime_tree_related_to_alternative
 897
 898             do_massage(
 899                 draft_f=string_io,
 900                 draftpath=const1,
 901                 cmd_f=sys.stdout,
 902                 converter=converter,
 903             )
 904
 905             captured = capsys.readouterr()
 906             lines = captured.out.splitlines()[4:-2]
 907             assert "first-entry" in lines.pop()
 908             assert "update-encoding" in lines.pop()
 909             assert "Plain" in lines.pop()
 910             assert "part.html" in lines.pop()
 911             assert "toggle-unlink" in lines.pop()
 912             assert "HTML" in lines.pop()
 913             assert "jump>1" in lines.pop()
 914             assert "jump>2" in lines.pop()
 915             assert "group-alternatives" in lines.pop()
 916             assert "Alternative" in lines.pop()
 917             assert "logo.png" in lines.pop()
 918             assert "toggle-unlink" in lines.pop()
 919             assert "content-id" in lines.pop()
 920             assert "Logo" in lines.pop()
 921             assert "jump>1" in lines.pop()
 922             assert "jump>4" in lines.pop()
 923             assert "group-related" in lines.pop()
 924             assert "Related" in lines.pop()
 925             assert "send-message" in lines.pop()
 926             assert len(lines) == 0
 927
 928         @pytest.fixture
 929         def fake_filewriter(self):
 930             class FileWriter:
 931                 def __init__(self):
 932                     self._writes = []
 933
 934                 def __call__(self, path, content, mode="w", **kwargs):
 935                     self._writes.append((path, content))
 936
 937                 def pop(self, index=-1):
 938                     return self._writes.pop(index)
 939
 940             return FileWriter()
 941
 942         @pytest.fixture
 943         def markdown_non_converter(self, const1, const2):
 944             return lambda s, text: f"{const1}{text}{const2}"
 945
 946         def test_converter_tree_basic(self, const1, const2, fake_filewriter):
 947             path = pathlib.Path(const2)
 948             tree = convert_markdown_to_html(
 949                 const1, path, filewriter_fn=fake_filewriter
 950             )
 951
 952             assert tree.subtype == "alternative"
 953             assert len(tree.children) == 2
 954             assert tree.children[0].subtype == "plain"
 955             assert tree.children[0].path == path
 956             assert tree.children[0].orig
 957             assert tree.children[1].subtype == "html"
 958             assert tree.children[1].path == path.with_suffix(".html")
 959
 960         def test_converter_writes(
 961             self,
 962             const1,
 963             const2,
 964             fake_filewriter,
 965             monkeypatch,
 966             markdown_non_converter,
 967         ):
 968             path = pathlib.Path(const2)
 969
 970             with monkeypatch.context() as m:
 971                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 972                 convert_markdown_to_html(
 973                     const1, path, filewriter_fn=fake_filewriter
 974                 )
 975
 976             assert (path, const1) == fake_filewriter.pop(0)
 977             assert (
 978                 path.with_suffix(".html"),
 979                 markdown_non_converter(None, const1),
 980             ) == fake_filewriter.pop(0)
 981
 982         def test_markdown_inline_image_processor(self):
 983             imgpath1 = "file:/path/to/image.png"
 984             imgpath2 = "file:///path/to/image.png?url=params"
 985             imgpath3 = "/path/to/image.png"
 986             text = f"""![inline local image]({imgpath1})
 987                        ![image inlined
 988                          with newline]({imgpath2})
 989                        ![image local path]({imgpath3})"""
 990             text, html, images = markdown_with_inline_image_support(text)
 991
 992             # local paths have been normalised to URLs:
 993             imgpath3 = f"file://{imgpath3}"
 994
 995             assert 'src="cid:' in html
 996             assert "](cid:" in text
 997             assert len(images) == 3
 998             assert imgpath1 in images
 999             assert imgpath2 in images
1000             assert imgpath3 in images
1001             assert images[imgpath1].cid != images[imgpath2].cid
1002             assert images[imgpath1].cid != images[imgpath3].cid
1003             assert images[imgpath2].cid != images[imgpath3].cid
1004
1005         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1006             imgpath = "file:///path/to/image.png"
1007             text = f'![inline local image]({imgpath} "{const1}")'
1008             text, html, images = markdown_with_inline_image_support(text)
1009             assert images[imgpath].desc == const1
1010
1011         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1012             imgpath = "file:///path/to/image.png"
1013             text = f"![{const1}]({imgpath})"
1014             text, html, images = markdown_with_inline_image_support(text)
1015             assert images[imgpath].desc == const1
1016
1017         def test_markdown_inline_image_processor_title_over_alt_desc(
1018             self, const1, const2
1019         ):
1020             imgpath = "file:///path/to/image.png"
1021             text = f'![{const1}]({imgpath} "{const2}")'
1022             text, html, images = markdown_with_inline_image_support(text)
1023             assert images[imgpath].desc == const2
1024
1025         def test_markdown_inline_image_not_external(self):
1026             imgpath = "https://path/to/image.png"
1027             text = f"![inline image]({imgpath})"
1028             text, html, images = markdown_with_inline_image_support(text)
1029
1030             assert 'src="cid:' not in html
1031             assert "](cid:" not in text
1032             assert len(images) == 0
1033
1034         def test_markdown_inline_image_local_file(self):
1035             imgpath = "/path/to/image.png"
1036             text = f"![inline image]({imgpath})"
1037             text, html, images = markdown_with_inline_image_support(text)
1038
1039             for k, v in images.items():
1040                 assert k == f"file://{imgpath}"
1041                 break
1042
1043         @pytest.fixture
1044         def test_png(self):
1045             return (
1046                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1047                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1048             )
1049
1050         def test_markdown_inline_image_processor_base64(self, test_png):
1051             text = f"![1px white inlined]({test_png})"
1052             text, html, images = markdown_with_inline_image_support(text)
1053
1054             assert 'src="cid:' in html
1055             assert "](cid:" in text
1056             assert len(images) == 1
1057             assert test_png in images
1058
1059         def test_converter_tree_inline_image_base64(
1060             self, test_png, const1, fake_filewriter
1061         ):
1062             text = f"![inline base64 image]({test_png})"
1063             path = pathlib.Path(const1)
1064             tree = convert_markdown_to_html(
1065                 text,
1066                 path,
1067                 filewriter_fn=fake_filewriter,
1068                 related_to_html_only=False,
1069             )
1070             assert tree.subtype == "relative"
1071             assert tree.children[0].subtype == "alternative"
1072             assert tree.children[1].subtype == "png"
1073             written = fake_filewriter.pop()
1074             assert tree.children[1].path == written[0]
1075             assert written[1] == request.urlopen(test_png).read()
1076
1077         def test_converter_tree_inline_image_base64_related_to_html(
1078             self, test_png, const1, fake_filewriter
1079         ):
1080             text = f"![inline base64 image]({test_png})"
1081             path = pathlib.Path(const1)
1082             tree = convert_markdown_to_html(
1083                 text,
1084                 path,
1085                 filewriter_fn=fake_filewriter,
1086                 related_to_html_only=True,
1087             )
1088             assert tree.subtype == "alternative"
1089             assert tree.children[1].subtype == "relative"
1090             assert tree.children[1].children[1].subtype == "png"
1091             written = fake_filewriter.pop()
1092             assert tree.children[1].children[1].path == written[0]
1093             assert written[1] == request.urlopen(test_png).read()
1094
1095         def test_converter_tree_inline_image_cid(
1096             self, const1, fake_filewriter
1097         ):
1098             text = f"![inline base64 image](cid:{const1})"
1099             path = pathlib.Path(const1)
1100             tree = convert_markdown_to_html(
1101                 text,
1102                 path,
1103                 filewriter_fn=fake_filewriter,
1104                 related_to_html_only=False,
1105             )
1106             assert len(tree.children) == 2
1107             assert tree.children[0].cid != const1
1108             assert tree.children[0].type != "image"
1109             assert tree.children[1].cid != const1
1110             assert tree.children[1].type != "image"
1111
1112         def test_inline_image_collection(
1113             self, test_png, const1, const2, fake_filewriter
1114         ):
1115             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1116             relparts = collect_inline_images(
1117                 test_images, filewriter_fn=fake_filewriter
1118             )
1119
1120             written = fake_filewriter.pop()
1121             assert b"PNG" in written[1]
1122
1123             assert relparts[0].subtype == "png"
1124             assert relparts[0].path == written[0]
1125             assert relparts[0].cid == const1
1126             assert relparts[0].desc.endswith(const2)
1127
1128         def test_apply_stylesheet(self):
1129             if _PYNLINER:
1130                 html = "<p>Hello, world!</p>"
1131                 css = "p { color:red }"
1132                 out = apply_styling(html, css)
1133                 assert 'p style="color' in out
1134
1135         def test_apply_stylesheet_pygments(self):
1136             if _PYGMENTS_CSS:
1137                 html = (
1138                     f'<div class="{_CODEHILITE_CLASS}">'
1139                     "<pre>def foo():\n    return</pre></div>"
1140                 )
1141                 out = apply_styling(html, _PYGMENTS_CSS)
1142                 assert f'{_CODEHILITE_CLASS}" style="' in out
1143
1144         def test_mime_tree_relative_within_alternative(
1145             self, string_io, const1, capsys, mime_tree_related_to_html
1146         ):
1147             def converter(
1148                 drafttext,
1149                 draftpath,
1150                 cssfile,
1151                 related_to_html_only,
1152                 extensions,
1153                 tempdir,
1154             ):
1155                 return mime_tree_related_to_html
1156
1157             do_massage(
1158                 draft_f=string_io,
1159                 draftpath=const1,
1160                 cmd_f=sys.stdout,
1161                 converter=converter,
1162             )
1163
1164             captured = capsys.readouterr()
1165             lines = captured.out.splitlines()[4:-2]
1166             assert "first-entry" in lines.pop()
1167             assert "update-encoding" in lines.pop()
1168             assert "Plain" in lines.pop()
1169             assert "part.html" in lines.pop()
1170             assert "toggle-unlink" in lines.pop()
1171             assert "HTML" in lines.pop()
1172             assert "logo.png" in lines.pop()
1173             assert "toggle-unlink" in lines.pop()
1174             assert "content-id" in lines.pop()
1175             assert "Logo" in lines.pop()
1176             assert "jump>2" in lines.pop()
1177             assert "jump>3" in lines.pop()
1178             assert "group-related" in lines.pop()
1179             assert "Related" in lines.pop()
1180             assert "jump>1" in lines.pop()
1181             assert "jump>2" in lines.pop()
1182             assert "group-alternative" in lines.pop()
1183             assert "Alternative" in lines.pop()
1184             assert "send-message" in lines.pop()
1185             assert len(lines) == 0
1186
1187 except ImportError:
1188     pass