.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--only-build",
  81         action="store_true",
  82         help="Only build, don't send the message",
  83     )
  84
  85     parser.add_argument(
  86         "--tempdir",
  87         default=None,
  88         help="Specify temporary directory to use for attachments",
  89     )
  90
  91     parser.add_argument(
  92         "--debug-commands",
  93         action="store_true",
  94         help="Turn on debug logging of commands generated to stderr",
  95     )
  96
  97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  98     massage_p = subp.add_parser(
  99         "massage", help="Massaging phase (internal use)"
 100     )
 101
 102     massage_p.add_argument(
 103         "--write-commands-to",
 104         "-o",
 105         metavar="PATH",
 106         dest="cmdpath",
 107         required=True,
 108         help="Temporary file path to write commands to",
 109     )
 110
 111     massage_p.add_argument(
 112         "--debug-walk",
 113         action="store_true",
 114         help="Turn on debugging to stderr of the MIME tree walk",
 115     )
 116
 117     massage_p.add_argument(
 118         "MAILDRAFT",
 119         nargs="?",
 120         help="If provided, the script is invoked as editor on the mail draft",
 121     )
 122
 123     return parser.parse_args(*args, **kwargs)
 124
 125
 126 # [ MARKDOWN WRAPPING ] #######################################################
 127
 128
 129 InlineImageInfo = namedtuple(
 130     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 131 )
 132
 133
 134 class InlineImageExtension(Extension):
 135     class RelatedImageInlineProcessor(ImageInlineProcessor):
 136         def __init__(self, re, md, ext):
 137             super().__init__(re, md)
 138             self._ext = ext
 139
 140         def handleMatch(self, m, data):
 141             el, start, end = super().handleMatch(m, data)
 142             if "src" in el.attrib:
 143                 src = el.attrib["src"]
 144                 if "://" not in src or src.startswith("file://"):
 145                     # We only inline local content
 146                     cid = self._ext.get_cid_for_image(el.attrib)
 147                     el.attrib["src"] = f"cid:{cid}"
 148             return el, start, end
 149
 150     def __init__(self):
 151         super().__init__()
 152         self._images = OrderedDict()
 153
 154     def extendMarkdown(self, md):
 155         md.registerExtension(self)
 156         inline_image_proc = self.RelatedImageInlineProcessor(
 157             IMAGE_LINK_RE, md, self
 158         )
 159         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 160
 161     def get_cid_for_image(self, attrib):
 162         msgid = make_msgid()[1:-1]
 163         path = attrib["src"]
 164         if path.startswith("/"):
 165             path = f"file://{path}"
 166         self._images[path] = InlineImageInfo(
 167             msgid, attrib.get("title", attrib.get("alt"))
 168         )
 169         return msgid
 170
 171     def get_images(self):
 172         return self._images
 173
 174
 175 def markdown_with_inline_image_support(
 176     text, *, extensions=None, extension_configs=None
 177 ):
 178     inline_image_handler = InlineImageExtension()
 179     extensions = extensions or []
 180     extensions.append(inline_image_handler)
 181     mdwn = markdown.Markdown(
 182         extensions=extensions, extension_configs=extension_configs
 183     )
 184     htmltext = mdwn.convert(text)
 185
 186     images = inline_image_handler.get_images()
 187
 188     def replace_image_with_cid(matchobj):
 189         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 190             if m in images:
 191                 return f"(cid:{images[m].cid}"
 192         return matchobj.group(0)
 193
 194     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 195     return text, htmltext, images
 196
 197
 198 # [ CSS STYLING ] #############################################################
 199
 200 try:
 201     import pynliner
 202
 203     _PYNLINER = True
 204
 205 except ImportError:
 206     _PYNLINER = False
 207
 208 try:
 209     from pygments.formatters import get_formatter_by_name
 210
 211     _CODEHILITE_CLASS = "codehilite"
 212
 213     _PYGMENTS_CSS = get_formatter_by_name(
 214         "html", style="default"
 215     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 216
 217 except ImportError:
 218     _PYGMENTS_CSS = None
 219
 220
 221 def apply_styling(html, css):
 222     return (
 223         pynliner.Pynliner()
 224         .from_string(html)
 225         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 226         .run()
 227     )
 228
 229
 230 # [ PARTS GENERATION ] ########################################################
 231
 232
 233 class Part(
 234     namedtuple(
 235         "Part",
 236         ["type", "subtype", "path", "desc", "cid", "orig"],
 237         defaults=[None, None, False],
 238     )
 239 ):
 240     def __str__(self):
 241         ret = f"<{self.type}/{self.subtype}>"
 242         if self.cid:
 243             ret = f"{ret} cid:{self.cid}"
 244         if self.orig:
 245             ret = f"{ret} ORIGINAL"
 246         return ret
 247
 248
 249 class Multipart(
 250     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 251 ):
 252     def __str__(self):
 253         return f"<multipart/{self.subtype}> children={len(self.children)}"
 254
 255     def __hash__(self):
 256         return hash(str(self.subtype) + "".join(str(self.children)))
 257
 258
 259 def filewriter_fn(path, content, mode="w", **kwargs):
 260     with open(path, mode, **kwargs) as out_f:
 261         out_f.write(content)
 262
 263
 264 def collect_inline_images(
 265     images, *, tempdir=None, filewriter_fn=filewriter_fn
 266 ):
 267     relparts = []
 268     for path, info in images.items():
 269         if path.startswith("cid:"):
 270             continue
 271
 272         data = request.urlopen(path)
 273
 274         mimetype = data.headers["Content-Type"]
 275         ext = mimetypes.guess_extension(mimetype)
 276         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 277         path = pathlib.Path(tempfilename[1])
 278
 279         filewriter_fn(path, data.read(), "w+b")
 280
 281         relparts.append(
 282             Part(
 283                 *mimetype.split("/"),
 284                 path,
 285                 cid=info.cid,
 286                 desc=f"Image: {info.desc}",
 287             )
 288         )
 289
 290     return relparts
 291
 292
 293 def convert_markdown_to_html(
 294     origtext,
 295     draftpath,
 296     *,
 297     cssfile=None,
 298     filewriter_fn=filewriter_fn,
 299     tempdir=None,
 300     extensions=None,
 301     extension_configs=None,
 302 ):
 303     # TODO extension_configs need to be handled differently
 304     extension_configs = extension_configs or {}
 305     extension_configs.setdefault("pymdownx.highlight", {})
 306     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 307
 308     origtext, htmltext, images = markdown_with_inline_image_support(
 309         origtext, extensions=extensions, extension_configs=extension_configs
 310     )
 311
 312     filewriter_fn(draftpath, origtext, encoding="utf-8")
 313     textpart = Part(
 314         "text", "plain", draftpath, "Plain-text version", orig=True
 315     )
 316
 317     htmltext = apply_styling(htmltext, cssfile)
 318
 319     htmlpath = draftpath.with_suffix(".html")
 320     filewriter_fn(
 321         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 322     )
 323     htmlpart = Part("text", "html", htmlpath, "HTML version")
 324
 325     altpart = Multipart(
 326         "alternative", [textpart, htmlpart], "Group of alternative content"
 327     )
 328
 329     imgparts = collect_inline_images(
 330         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 331     )
 332     if imgparts:
 333         return Multipart(
 334             "relative", [altpart] + imgparts, "Group of related content"
 335         )
 336     else:
 337         return altpart
 338
 339
 340 class MIMETreeDFWalker:
 341     def __init__(self, *, visitor_fn=None, debug=False):
 342         self._visitor_fn = visitor_fn
 343         self._debug = debug
 344
 345     def walk(self, root, *, visitor_fn=None):
 346         """
 347         Recursive function to implement a depth-dirst walk of the MIME-tree
 348         rooted at `root`.
 349         """
 350
 351         if isinstance(root, list):
 352             root = Multipart("mixed", children=root)
 353
 354         self._walk(
 355             root,
 356             stack=[],
 357             visitor_fn=visitor_fn or self._visitor_fn,
 358         )
 359
 360     def _walk(self, node, *, stack, visitor_fn):
 361         # Let's start by enumerating the parts at the current level. At the
 362         # root level, stack will be the empty list, and we expect a multipart/*
 363         # container at this level. Later, e.g. within a mutlipart/alternative
 364         # container, the subtree will just be the alternative parts, while the
 365         # top of the stack will be the multipart/alternative container, which
 366         # we will process after the following loop.
 367
 368         lead = f"{'| '*len(stack)}|-"
 369         if isinstance(node, Multipart):
 370             self.debugprint(
 371                 f"{lead}{node} parents={[s.subtype for s in stack]}"
 372             )
 373
 374             # Depth-first, so push the current container onto the stack,
 375             # then descend …
 376             stack.append(node)
 377             self.debugprint("| " * (len(stack) + 1))
 378             for child in node.children:
 379                 self._walk(
 380                     child,
 381                     stack=stack,
 382                     visitor_fn=visitor_fn,
 383                 )
 384             self.debugprint("| " * len(stack))
 385             assert stack.pop() == node
 386
 387         else:
 388             self.debugprint(f"{lead}{node}")
 389
 390         if visitor_fn:
 391             visitor_fn(node, stack, debugprint=self.debugprint)
 392
 393     def debugprint(self, s, **kwargs):
 394         if self._debug:
 395             print(s, file=sys.stderr, **kwargs)
 396
 397
 398 # [ RUN MODES ] ###############################################################
 399
 400
 401 class MuttCommands:
 402     """
 403     Stupid class to interface writing out Mutt commands. This is quite a hack
 404     to deal with the fact that Mutt runs "push" commands in reverse order, so
 405     all of a sudden, things become very complicated when mixing with "real"
 406     commands.
 407
 408     Hence we keep two sets of commands, and one set of pushes. Commands are
 409     added to the first until a push is added, after which commands are added to
 410     the second set of commands.
 411
 412     On flush(), the first set is printed, followed by the pushes in reverse,
 413     and then the second set is printed. All 3 sets are then cleared.
 414     """
 415
 416     def __init__(self, out_f=sys.stdout, *, debug=False):
 417         self._cmd1, self._push, self._cmd2 = [], [], []
 418         self._out_f = out_f
 419         self._debug = debug
 420
 421     def cmd(self, s):
 422         self.debugprint(s)
 423         if self._push:
 424             self._cmd2.append(s)
 425         else:
 426             self._cmd1.append(s)
 427
 428     def push(self, s):
 429         s = s.replace('"', '"')
 430         s = f'push "{s}"'
 431         self.debugprint(s)
 432         self._push.insert(0, s)
 433
 434     def flush(self):
 435         print(
 436             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 437         )
 438         self._cmd1, self._push, self._cmd2 = [], [], []
 439
 440     def debugprint(self, s, **kwargs):
 441         if self._debug:
 442             print(s, file=sys.stderr, **kwargs)
 443
 444
 445 def do_setup(
 446     *,
 447     out_f=sys.stdout,
 448     temppath=None,
 449     tempdir=None,
 450     debug_commands=False,
 451 ):
 452     temppath = temppath or pathlib.Path(
 453         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 454     )
 455     cmds = MuttCommands(out_f, debug=debug_commands)
 456
 457     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 458
 459     cmds.cmd('set my_editor="$editor"')
 460     cmds.cmd('set my_edit_headers="$edit_headers"')
 461     cmds.cmd(f'set editor="{editor}"')
 462     cmds.cmd("unset edit_headers")
 463     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 464     cmds.push("<first-entry><edit-file>")
 465     cmds.flush()
 466
 467
 468 def do_massage(
 469     draft_f,
 470     draftpath,
 471     cmd_f,
 472     *,
 473     extensions=None,
 474     cssfile=None,
 475     converter=convert_markdown_to_html,
 476     only_build=False,
 477     tempdir=None,
 478     debug_commands=False,
 479     debug_walk=False,
 480 ):
 481     # Here's the big picture: we're being invoked as the editor on the email
 482     # draft, and whatever commands we write to the file given as cmdpath will
 483     # be run by the second source command in the macro definition.
 484
 485     # Let's start by cleaning up what the setup did (see above), i.e. we
 486     # restore the $editor and $edit_headers variables, and also unset the
 487     # variable used to identify the command file we're currently writing
 488     # to.
 489     cmds = MuttCommands(cmd_f, debug=debug_commands)
 490     cmds.cmd('set editor="$my_editor"')
 491     cmds.cmd('set edit_headers="$my_edit_headers"')
 492     cmds.cmd("unset my_editor")
 493     cmds.cmd("unset my_edit_headers")
 494
 495     # let's flush those commands, as there'll be a lot of pushes from now
 496     # on, which need to be run in reverse order
 497     cmds.flush()
 498
 499     extensions = extensions.split(",") if extensions else []
 500     tree = converter(
 501         draft_f.read(),
 502         draftpath,
 503         cssfile=cssfile,
 504         tempdir=tempdir,
 505         extensions=extensions,
 506     )
 507
 508     mimetree = MIMETreeDFWalker(debug=debug_walk)
 509
 510     def visitor_fn(item, stack, *, debugprint=None):
 511         """
 512         Visitor function called for every node (part) of the MIME tree,
 513         depth-first, and responsible for telling NeoMutt how to assemble
 514         the tree.
 515         """
 516         KILL_LINE = r"\Ca\Ck"
 517
 518         if isinstance(item, Part):
 519             # We've hit a leaf-node, i.e. an alternative or a related part
 520             # with actual content.
 521
 522             # Let's add the part
 523             if item.orig:
 524                 # The original source already exists in the NeoMutt tree, but
 525                 # the underlying file may have been modified, so we need to
 526                 # update the encoding, but that's it:
 527                 cmds.push("<update-encoding>")
 528             else:
 529                 # … whereas all other parts need to be added, and they're all
 530                 # considered to be temporary and inline:
 531                 cmds.push(f"<attach-file>{item.path}<enter>")
 532                 cmds.push("<toggle-unlink><toggle-disposition>")
 533
 534             # If the item (including the original) comes with additional
 535             # information, then we might just as well update the NeoMutt
 536             # tree now:
 537             if item.cid:
 538                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 539
 540         elif isinstance(item, Multipart):
 541             # This node has children, but we already visited them (see
 542             # above), and so they have been tagged in NeoMutt's compose
 543             # window. Now it's just a matter of telling NeoMutt to do the
 544             # appropriate grouping:
 545             if item.subtype == "alternative":
 546                 cmds.push("<group-alternatives>")
 547             elif item.subtype in ("relative", "related"):
 548                 cmds.push("<group-related>")
 549             elif item.subtype == "multilingual":
 550                 cmds.push("<group-multilingual>")
 551
 552         else:
 553             # We should never get here
 554             assert not "is valid part"
 555
 556         # If the item has a description, we might just as well add it
 557         if item.desc:
 558             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 559
 560         # Finally, if we're at non-root level, tag the new container,
 561         # as it might itself be part of a container, to be processed
 562         # one level up:
 563         if stack:
 564             cmds.push("<tag-entry>")
 565
 566     # -----------------
 567     # End of visitor_fn
 568
 569     # Let's walk the tree and visit every node with our fancy visitor
 570     # function
 571     mimetree.walk(tree, visitor_fn=visitor_fn)
 572
 573     if not only_build:
 574         cmds.push("<send-message>")
 575
 576     # Finally, cleanup. Since we're responsible for removing the temporary
 577     # file, how's this for a little hack?
 578     try:
 579         filename = cmd_f.name
 580     except AttributeError:
 581         filename = "pytest_internal_file"
 582     cmds.cmd(f"source 'rm -f {filename}|'")
 583     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 584     cmds.flush()
 585
 586
 587 # [ CLI ENTRY ] ###############################################################
 588
 589 if __name__ == "__main__":
 590     args = parse_cli_args()
 591
 592     if args.mode is None:
 593         do_setup(
 594             tempdir=args.tempdir,
 595             debug_commands=args.debug_commands,
 596         )
 597
 598     elif args.mode == "massage":
 599         with open(args.MAILDRAFT, "r") as draft_f, open(
 600             args.cmdpath, "w"
 601         ) as cmd_f:
 602             do_massage(
 603                 draft_f,
 604                 pathlib.Path(args.MAILDRAFT),
 605                 cmd_f,
 606                 extensions=args.extensions,
 607                 cssfile=args.css_file,
 608                 only_build=args.only_build,
 609                 tempdir=args.tempdir,
 610                 debug_commands=args.debug_commands,
 611                 debug_walk=args.debug_walk,
 612             )
 613
 614
 615 # [ TESTS ] ###################################################################
 616
 617 try:
 618     import pytest
 619     from io import StringIO
 620
 621     class Tests:
 622         @pytest.fixture
 623         def const1(self):
 624             return "CONSTANT STRING 1"
 625
 626         @pytest.fixture
 627         def const2(self):
 628             return "CONSTANT STRING 2"
 629
 630         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 631         # functions they call, else old stdout is used and not captured
 632
 633         def test_MuttCommands_cmd(self, const1, const2, capsys):
 634             "Assert order of commands"
 635             cmds = MuttCommands(out_f=sys.stdout)
 636             cmds.cmd(const1)
 637             cmds.cmd(const2)
 638             cmds.flush()
 639             captured = capsys.readouterr()
 640             assert captured.out == "\n".join((const1, const2, ""))
 641
 642         def test_MuttCommands_push(self, const1, const2, capsys):
 643             "Assert reverse order of pushes"
 644             cmds = MuttCommands(out_f=sys.stdout)
 645             cmds.push(const1)
 646             cmds.push(const2)
 647             cmds.flush()
 648             captured = capsys.readouterr()
 649             assert (
 650                 captured.out
 651                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 652             )
 653
 654         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 655             "Assert reverse order of pushes"
 656             cmds = MuttCommands(out_f=sys.stdout)
 657             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 658             for i in range(2):
 659                 cmds.cmd(lines[4 * i + 0])
 660                 cmds.cmd(lines[4 * i + 1])
 661                 cmds.push(lines[4 * i + 2])
 662                 cmds.push(lines[4 * i + 3])
 663             cmds.flush()
 664
 665             captured = capsys.readouterr()
 666             lines_out = captured.out.splitlines()
 667             assert lines[0] in lines_out[0]
 668             assert lines[1] in lines_out[1]
 669             assert lines[7] in lines_out[2]
 670             assert lines[6] in lines_out[3]
 671             assert lines[3] in lines_out[4]
 672             assert lines[2] in lines_out[5]
 673             assert lines[4] in lines_out[6]
 674             assert lines[5] in lines_out[7]
 675
 676         @pytest.fixture
 677         def basic_mime_tree(self):
 678             return Multipart(
 679                 "relative",
 680                 children=[
 681                     Multipart(
 682                         "alternative",
 683                         children=[
 684                             Part(
 685                                 "text",
 686                                 "plain",
 687                                 "part.txt",
 688                                 desc="Plain",
 689                                 orig=True,
 690                             ),
 691                             Part("text", "html", "part.html", desc="HTML"),
 692                         ],
 693                         desc="Alternative",
 694                     ),
 695                     Part(
 696                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 697                     ),
 698                 ],
 699                 desc="Related",
 700             )
 701
 702         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 703             mimetree = MIMETreeDFWalker()
 704
 705             items = []
 706
 707             def visitor_fn(item, stack, debugprint):
 708                 items.append((item, len(stack)))
 709
 710             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 711             assert len(items) == 5
 712             assert items[0][0].subtype == "plain"
 713             assert items[0][1] == 2
 714             assert items[1][0].subtype == "html"
 715             assert items[1][1] == 2
 716             assert items[2][0].subtype == "alternative"
 717             assert items[2][1] == 1
 718             assert items[3][0].subtype == "png"
 719             assert items[3][1] == 1
 720             assert items[4][0].subtype == "relative"
 721             assert items[4][1] == 0
 722
 723         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
 724             mimetree = MIMETreeDFWalker()
 725             items = []
 726
 727             def visitor_fn(item, stack, debugprint):
 728                 items.append(item)
 729
 730             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
 731             assert items[-1].subtype == "mixed"
 732
 733         def test_MIMETreeDFWalker_visitor_in_constructor(
 734             self, basic_mime_tree
 735         ):
 736             items = []
 737
 738             def visitor_fn(item, stack, debugprint):
 739                 items.append(item)
 740
 741             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 742             mimetree.walk(basic_mime_tree)
 743             assert len(items) == 5
 744
 745         @pytest.fixture
 746         def string_io(self, const1, text=None):
 747             return StringIO(text or const1)
 748
 749         def test_do_massage_basic(self, const1, string_io, capsys):
 750             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 751                 return Part("text", "plain", draftpath, orig=True)
 752
 753             do_massage(
 754                 draft_f=string_io,
 755                 draftpath=const1,
 756                 cmd_f=sys.stdout,
 757                 converter=converter,
 758             )
 759
 760             captured = capsys.readouterr()
 761             lines = captured.out.splitlines()
 762             assert '="$my_editor"' in lines.pop(0)
 763             assert '="$my_edit_headers"' in lines.pop(0)
 764             assert "unset my_editor" == lines.pop(0)
 765             assert "unset my_edit_headers" == lines.pop(0)
 766             assert "send-message" in lines.pop(0)
 767             assert "update-encoding" in lines.pop(0)
 768             assert "source 'rm -f " in lines.pop(0)
 769             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 770
 771         def test_do_massage_fulltree(
 772             self, string_io, const1, basic_mime_tree, capsys
 773         ):
 774             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 775                 return basic_mime_tree
 776
 777             do_massage(
 778                 draft_f=string_io,
 779                 draftpath=const1,
 780                 cmd_f=sys.stdout,
 781                 converter=converter,
 782             )
 783
 784             captured = capsys.readouterr()
 785             lines = captured.out.splitlines()[4:]
 786             assert "send-message" in lines.pop(0)
 787             assert "Related" in lines.pop(0)
 788             assert "group-related" in lines.pop(0)
 789             assert "tag-entry" in lines.pop(0)
 790             assert "Logo" in lines.pop(0)
 791             assert "content-id" in lines.pop(0)
 792             assert "toggle-unlink" in lines.pop(0)
 793             assert "logo.png" in lines.pop(0)
 794             assert "tag-entry" in lines.pop(0)
 795             assert "Alternative" in lines.pop(0)
 796             assert "group-alternatives" in lines.pop(0)
 797             assert "tag-entry" in lines.pop(0)
 798             assert "HTML" in lines.pop(0)
 799             assert "toggle-unlink" in lines.pop(0)
 800             assert "part.html" in lines.pop(0)
 801             assert "tag-entry" in lines.pop(0)
 802             assert "Plain" in lines.pop(0)
 803             assert "update-encoding" in lines.pop(0)
 804             assert len(lines) == 2
 805
 806         @pytest.fixture
 807         def fake_filewriter(self):
 808             class FileWriter:
 809                 def __init__(self):
 810                     self._writes = []
 811
 812                 def __call__(self, path, content, mode="w", **kwargs):
 813                     self._writes.append((path, content))
 814
 815                 def pop(self, index=-1):
 816                     return self._writes.pop(index)
 817
 818             return FileWriter()
 819
 820         @pytest.fixture
 821         def markdown_non_converter(self, const1, const2):
 822             return lambda s, text: f"{const1}{text}{const2}"
 823
 824         def test_converter_tree_basic(
 825             self, const1, const2, fake_filewriter, markdown_non_converter
 826         ):
 827             path = pathlib.Path(const2)
 828             tree = convert_markdown_to_html(
 829                 const1, path, filewriter_fn=fake_filewriter
 830             )
 831
 832             assert tree.subtype == "alternative"
 833             assert len(tree.children) == 2
 834             assert tree.children[0].subtype == "plain"
 835             assert tree.children[0].path == path
 836             assert tree.children[0].orig
 837             assert tree.children[1].subtype == "html"
 838             assert tree.children[1].path == path.with_suffix(".html")
 839
 840         def test_converter_writes(
 841             self,
 842             const1,
 843             const2,
 844             fake_filewriter,
 845             monkeypatch,
 846             markdown_non_converter,
 847         ):
 848             path = pathlib.Path(const2)
 849
 850             with monkeypatch.context() as m:
 851                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 852                 convert_markdown_to_html(
 853                     const1, path, filewriter_fn=fake_filewriter
 854                 )
 855
 856             assert (path, const1) == fake_filewriter.pop(0)
 857             assert (
 858                 path.with_suffix(".html"),
 859                 markdown_non_converter(None, const1),
 860             ) == fake_filewriter.pop(0)
 861
 862         def test_markdown_inline_image_processor(self):
 863             imgpath1 = "file:/path/to/image.png"
 864             imgpath2 = "file:///path/to/image.png?url=params"
 865             imgpath3 = "/path/to/image.png"
 866             text = f"""![inline local image]({imgpath1})
 867                        ![image inlined
 868                          with newline]({imgpath2})
 869                        ![image local path]({imgpath3})"""
 870             text, html, images = markdown_with_inline_image_support(text)
 871
 872             # local paths have been normalised to URLs:
 873             imgpath3 = f"file://{imgpath3}"
 874
 875             assert 'src="cid:' in html
 876             assert "](cid:" in text
 877             assert len(images) == 3
 878             assert imgpath1 in images
 879             assert imgpath2 in images
 880             assert imgpath3 in images
 881             assert images[imgpath1].cid != images[imgpath2].cid
 882             assert images[imgpath1].cid != images[imgpath3].cid
 883             assert images[imgpath2].cid != images[imgpath3].cid
 884
 885         def test_markdown_inline_image_processor_title_to_desc(self, const1):
 886             imgpath = "file:///path/to/image.png"
 887             text = f'![inline local image]({imgpath} "{const1}")'
 888             text, html, images = markdown_with_inline_image_support(text)
 889             assert images[imgpath].desc == const1
 890
 891         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
 892             imgpath = "file:///path/to/image.png"
 893             text = f"![{const1}]({imgpath})"
 894             text, html, images = markdown_with_inline_image_support(text)
 895             assert images[imgpath].desc == const1
 896
 897         def test_markdown_inline_image_processor_title_over_alt_desc(
 898             self, const1, const2
 899         ):
 900             imgpath = "file:///path/to/image.png"
 901             text = f'![{const1}]({imgpath} "{const2}")'
 902             text, html, images = markdown_with_inline_image_support(text)
 903             assert images[imgpath].desc == const2
 904
 905         def test_markdown_inline_image_not_external(self):
 906             imgpath = "https://path/to/image.png"
 907             text = f"![inline image]({imgpath})"
 908             text, html, images = markdown_with_inline_image_support(text)
 909
 910             assert 'src="cid:' not in html
 911             assert "](cid:" not in text
 912             assert len(images) == 0
 913
 914         def test_markdown_inline_image_local_file(self):
 915             imgpath = "/path/to/image.png"
 916             text = f"![inline image]({imgpath})"
 917             text, html, images = markdown_with_inline_image_support(text)
 918
 919             for k, v in images.items():
 920                 assert k == f"file://{imgpath}"
 921                 break
 922
 923         @pytest.fixture
 924         def test_png(self):
 925             return (
 926                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
 927                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
 928             )
 929
 930         def test_markdown_inline_image_processor_base64(self, test_png):
 931             text = f"![1px white inlined]({test_png})"
 932             text, html, images = markdown_with_inline_image_support(text)
 933
 934             assert 'src="cid:' in html
 935             assert "](cid:" in text
 936             assert len(images) == 1
 937             assert test_png in images
 938
 939         def test_converter_tree_inline_image_base64(
 940             self, test_png, const1, fake_filewriter
 941         ):
 942             text = f"![inline base64 image]({test_png})"
 943             path = pathlib.Path(const1)
 944             tree = convert_markdown_to_html(
 945                 text, path, filewriter_fn=fake_filewriter
 946             )
 947
 948             assert tree.subtype == "relative"
 949             assert tree.children[1].subtype == "png"
 950             written = fake_filewriter.pop()
 951             assert tree.children[1].path == written[0]
 952             assert written[1] == request.urlopen(test_png).read()
 953
 954         def test_converter_tree_inline_image_cid(
 955             self, const1, fake_filewriter
 956         ):
 957             text = f"![inline base64 image](cid:{const1})"
 958             path = pathlib.Path(const1)
 959             tree = convert_markdown_to_html(
 960                 text,
 961                 path,
 962                 filewriter_fn=fake_filewriter,
 963                 related_to_html_only=False,
 964             )
 965             assert len(tree.children) == 2
 966             assert tree.children[0].cid != const1
 967             assert tree.children[0].type != "image"
 968             assert tree.children[1].cid != const1
 969             assert tree.children[1].type != "image"
 970
 971         def test_inline_image_collection(
 972             self, test_png, const1, const2, fake_filewriter
 973         ):
 974             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
 975             relparts = collect_inline_images(
 976                 test_images, filewriter_fn=fake_filewriter
 977             )
 978
 979             written = fake_filewriter.pop()
 980             assert b"PNG" in written[1]
 981
 982             assert relparts[0].subtype == "png"
 983             assert relparts[0].path == written[0]
 984             assert relparts[0].cid == const1
 985             assert relparts[0].desc.endswith(const2)
 986
 987         def test_apply_stylesheet(self):
 988             if _PYNLINER:
 989                 html = "<p>Hello, world!</p>"
 990                 css = "p { color:red }"
 991                 out = apply_styling(html, css)
 992                 assert 'p style="color' in out
 993
 994         def test_apply_stylesheet_pygments(self):
 995             if _PYGMENTS_CSS:
 996                 html = (
 997                     f'<div class="{_CODEHILITE_CLASS}">'
 998                     "<pre>def foo():\n    return</pre></div>"
 999                 )
1000                 out = apply_styling(html, _PYGMENTS_CSS)
1001                 assert f'{_CODEHILITE_CLASS}" style="' in out
1002
1003
1004 except ImportError:
1005     pass