.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--only-build",
  81         action="store_true",
  82         help="Only build, don't send the message",
  83     )
  84
  85     parser.add_argument(
  86         "--tempdir",
  87         default=None,
  88         help="Specify temporary directory to use for attachments",
  89     )
  90
  91     parser.add_argument(
  92         "--debug-commands",
  93         action="store_true",
  94         help="Turn on debug logging of commands generated to stderr",
  95     )
  96
  97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  98     massage_p = subp.add_parser(
  99         "massage", help="Massaging phase (internal use)"
 100     )
 101
 102     massage_p.add_argument(
 103         "--write-commands-to",
 104         "-o",
 105         metavar="PATH",
 106         dest="cmdpath",
 107         required=True,
 108         help="Temporary file path to write commands to",
 109     )
 110
 111     massage_p.add_argument(
 112         "--debug-walk",
 113         action="store_true",
 114         help="Turn on debugging to stderr of the MIME tree walk",
 115     )
 116
 117     massage_p.add_argument(
 118         "MAILDRAFT",
 119         nargs="?",
 120         help="If provided, the script is invoked as editor on the mail draft",
 121     )
 122
 123     return parser.parse_args(*args, **kwargs)
 124
 125
 126 # [ MARKDOWN WRAPPING ] #######################################################
 127
 128
 129 InlineImageInfo = namedtuple(
 130     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 131 )
 132
 133
 134 class InlineImageExtension(Extension):
 135     class RelatedImageInlineProcessor(ImageInlineProcessor):
 136         def __init__(self, re, md, ext):
 137             super().__init__(re, md)
 138             self._ext = ext
 139
 140         def handleMatch(self, m, data):
 141             el, start, end = super().handleMatch(m, data)
 142             if "src" in el.attrib:
 143                 src = el.attrib["src"]
 144                 if "://" not in src or src.startswith("file://"):
 145                     # We only inline local content
 146                     cid = self._ext.get_cid_for_image(el.attrib)
 147                     el.attrib["src"] = f"cid:{cid}"
 148             return el, start, end
 149
 150     def __init__(self):
 151         super().__init__()
 152         self._images = OrderedDict()
 153
 154     def extendMarkdown(self, md):
 155         md.registerExtension(self)
 156         inline_image_proc = self.RelatedImageInlineProcessor(
 157             IMAGE_LINK_RE, md, self
 158         )
 159         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 160
 161     def get_cid_for_image(self, attrib):
 162         msgid = make_msgid()[1:-1]
 163         path = attrib["src"]
 164         if path.startswith("/"):
 165             path = f"file://{path}"
 166         self._images[path] = InlineImageInfo(
 167             msgid, attrib.get("title", attrib.get("alt"))
 168         )
 169         return msgid
 170
 171     def get_images(self):
 172         return self._images
 173
 174
 175 def markdown_with_inline_image_support(
 176     text, *, extensions=None, extension_configs=None
 177 ):
 178     inline_image_handler = InlineImageExtension()
 179     extensions = extensions or []
 180     extensions.append(inline_image_handler)
 181     mdwn = markdown.Markdown(
 182         extensions=extensions, extension_configs=extension_configs
 183     )
 184     htmltext = mdwn.convert(text)
 185
 186     images = inline_image_handler.get_images()
 187
 188     def replace_image_with_cid(matchobj):
 189         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 190             if m in images:
 191                 return f"(cid:{images[m].cid}"
 192         return matchobj.group(0)
 193
 194     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 195     return text, htmltext, images
 196
 197
 198 # [ CSS STYLING ] #############################################################
 199
 200 try:
 201     import pynliner
 202
 203     _PYNLINER = True
 204
 205 except ImportError:
 206     _PYNLINER = False
 207
 208 try:
 209     from pygments.formatters import get_formatter_by_name
 210
 211     _CODEHILITE_CLASS = "codehilite"
 212
 213     _PYGMENTS_CSS = get_formatter_by_name(
 214         "html", style="default"
 215     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 216
 217 except ImportError:
 218     _PYGMENTS_CSS = None
 219
 220
 221 def apply_styling(html, css):
 222     return (
 223         pynliner.Pynliner()
 224         .from_string(html)
 225         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 226         .run()
 227     )
 228
 229
 230 # [ PARTS GENERATION ] ########################################################
 231
 232
 233 class Part(
 234     namedtuple(
 235         "Part",
 236         ["type", "subtype", "path", "desc", "cid", "orig"],
 237         defaults=[None, None, False],
 238     )
 239 ):
 240     def __str__(self):
 241         ret = f"<{self.type}/{self.subtype}>"
 242         if self.cid:
 243             ret = f"{ret} cid:{self.cid}"
 244         if self.orig:
 245             ret = f"{ret} ORIGINAL"
 246         return ret
 247
 248
 249 class Multipart(
 250     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 251 ):
 252     def __str__(self):
 253         return f"<multipart/{self.subtype}> children={len(self.children)}"
 254
 255
 256 def filewriter_fn(path, content, mode="w", **kwargs):
 257     with open(path, mode, **kwargs) as out_f:
 258         out_f.write(content)
 259
 260
 261 def collect_inline_images(
 262     images, *, tempdir=None, filewriter_fn=filewriter_fn
 263 ):
 264     relparts = []
 265     for path, info in images.items():
 266         data = request.urlopen(path)
 267
 268         mimetype = data.headers["Content-Type"]
 269         ext = mimetypes.guess_extension(mimetype)
 270         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 271         path = pathlib.Path(tempfilename[1])
 272
 273         filewriter_fn(path, data.read(), "w+b")
 274
 275         relparts.append(
 276             Part(
 277                 *mimetype.split("/"),
 278                 path,
 279                 cid=info.cid,
 280                 desc=f"Image: {info.desc}",
 281             )
 282         )
 283
 284     return relparts
 285
 286
 287 def convert_markdown_to_html(
 288     origtext,
 289     draftpath,
 290     *,
 291     cssfile=None,
 292     filewriter_fn=filewriter_fn,
 293     tempdir=None,
 294     extensions=None,
 295     extension_configs=None,
 296 ):
 297     # TODO extension_configs need to be handled differently
 298     extension_configs = extension_configs or {}
 299     extension_configs.setdefault("pymdownx.highlight", {})
 300     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 301
 302     origtext, htmltext, images = markdown_with_inline_image_support(
 303         origtext, extensions=extensions, extension_configs=extension_configs
 304     )
 305
 306     filewriter_fn(draftpath, origtext, encoding="utf-8")
 307     textpart = Part(
 308         "text", "plain", draftpath, "Plain-text version", orig=True
 309     )
 310
 311     htmltext = apply_styling(htmltext, cssfile)
 312
 313     htmlpath = draftpath.with_suffix(".html")
 314     filewriter_fn(
 315         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 316     )
 317     htmlpart = Part("text", "html", htmlpath, "HTML version")
 318
 319     altpart = Multipart(
 320         "alternative", [textpart, htmlpart], "Group of alternative content"
 321     )
 322
 323     imgparts = collect_inline_images(
 324         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 325     )
 326     if imgparts:
 327         return Multipart(
 328             "relative", [altpart] + imgparts, "Group of related content"
 329         )
 330     else:
 331         return altpart
 332
 333
 334 class MIMETreeDFWalker:
 335     def __init__(self, *, visitor_fn=None, debug=False):
 336         self._visitor_fn = visitor_fn
 337         self._debug = debug
 338
 339     def walk(self, root, *, visitor_fn=None):
 340         """
 341         Recursive function to implement a depth-dirst walk of the MIME-tree
 342         rooted at `root`.
 343         """
 344
 345         if isinstance(root, list):
 346             root = Multipart("mixed", children=root)
 347
 348         self._walk(
 349             root,
 350             stack=[],
 351             visitor_fn=visitor_fn or self._visitor_fn,
 352         )
 353
 354     def _walk(self, node, *, stack, visitor_fn):
 355         # Let's start by enumerating the parts at the current level. At the
 356         # root level, stack will be the empty list, and we expect a multipart/*
 357         # container at this level. Later, e.g. within a mutlipart/alternative
 358         # container, the subtree will just be the alternative parts, while the
 359         # top of the stack will be the multipart/alternative container, which
 360         # we will process after the following loop.
 361
 362         lead = f"{'| '*len(stack)}|-"
 363         if isinstance(node, Multipart):
 364             self.debugprint(
 365                 f"{lead}{node} parents={[s.subtype for s in stack]}"
 366             )
 367
 368             # Depth-first, so push the current container onto the stack,
 369             # then descend …
 370             stack.append(node)
 371             self.debugprint("| " * (len(stack) + 1))
 372             for child in node.children:
 373                 self._walk(
 374                     child,
 375                     stack=stack,
 376                     visitor_fn=visitor_fn,
 377                 )
 378             self.debugprint("| " * len(stack))
 379             assert stack.pop() == node
 380
 381         else:
 382             self.debugprint(f"{lead}{node}")
 383
 384         if visitor_fn:
 385             visitor_fn(node, stack, debugprint=self.debugprint)
 386
 387     def debugprint(self, s, **kwargs):
 388         if self._debug:
 389             print(s, file=sys.stderr, **kwargs)
 390
 391
 392 # [ RUN MODES ] ###############################################################
 393
 394
 395 class MuttCommands:
 396     """
 397     Stupid class to interface writing out Mutt commands. This is quite a hack
 398     to deal with the fact that Mutt runs "push" commands in reverse order, so
 399     all of a sudden, things become very complicated when mixing with "real"
 400     commands.
 401
 402     Hence we keep two sets of commands, and one set of pushes. Commands are
 403     added to the first until a push is added, after which commands are added to
 404     the second set of commands.
 405
 406     On flush(), the first set is printed, followed by the pushes in reverse,
 407     and then the second set is printed. All 3 sets are then cleared.
 408     """
 409
 410     def __init__(self, out_f=sys.stdout, *, debug=False):
 411         self._cmd1, self._push, self._cmd2 = [], [], []
 412         self._out_f = out_f
 413         self._debug = debug
 414
 415     def cmd(self, s):
 416         self.debugprint(s)
 417         if self._push:
 418             self._cmd2.append(s)
 419         else:
 420             self._cmd1.append(s)
 421
 422     def push(self, s):
 423         s = s.replace('"', '"')
 424         s = f'push "{s}"'
 425         self.debugprint(s)
 426         self._push.insert(0, s)
 427
 428     def flush(self):
 429         print(
 430             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 431         )
 432         self._cmd1, self._push, self._cmd2 = [], [], []
 433
 434     def debugprint(self, s, **kwargs):
 435         if self._debug:
 436             print(s, file=sys.stderr, **kwargs)
 437
 438
 439 def do_setup(
 440     *,
 441     out_f=sys.stdout,
 442     temppath=None,
 443     tempdir=None,
 444     debug_commands=False,
 445 ):
 446     temppath = temppath or pathlib.Path(
 447         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 448     )
 449     cmds = MuttCommands(out_f, debug=debug_commands)
 450
 451     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 452
 453     cmds.cmd('set my_editor="$editor"')
 454     cmds.cmd('set my_edit_headers="$edit_headers"')
 455     cmds.cmd(f'set editor="{editor}"')
 456     cmds.cmd("unset edit_headers")
 457     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 458     cmds.push("<first-entry><edit-file>")
 459     cmds.flush()
 460
 461
 462 def do_massage(
 463     draft_f,
 464     draftpath,
 465     cmd_f,
 466     *,
 467     extensions=None,
 468     cssfile=None,
 469     converter=convert_markdown_to_html,
 470     only_build=False,
 471     tempdir=None,
 472     debug_commands=False,
 473     debug_walk=False,
 474 ):
 475     # Here's the big picture: we're being invoked as the editor on the email
 476     # draft, and whatever commands we write to the file given as cmdpath will
 477     # be run by the second source command in the macro definition.
 478
 479     # Let's start by cleaning up what the setup did (see above), i.e. we
 480     # restore the $editor and $edit_headers variables, and also unset the
 481     # variable used to identify the command file we're currently writing
 482     # to.
 483     cmds = MuttCommands(cmd_f, debug=debug_commands)
 484     cmds.cmd('set editor="$my_editor"')
 485     cmds.cmd('set edit_headers="$my_edit_headers"')
 486     cmds.cmd("unset my_editor")
 487     cmds.cmd("unset my_edit_headers")
 488
 489     # let's flush those commands, as there'll be a lot of pushes from now
 490     # on, which need to be run in reverse order
 491     cmds.flush()
 492
 493     extensions = extensions.split(",") if extensions else []
 494     tree = converter(
 495         draft_f.read(),
 496         draftpath,
 497         cssfile=cssfile,
 498         tempdir=tempdir,
 499         extensions=extensions,
 500     )
 501
 502     mimetree = MIMETreeDFWalker(debug=debug_walk)
 503
 504     def visitor_fn(item, stack, *, debugprint=None):
 505         """
 506         Visitor function called for every node (part) of the MIME tree,
 507         depth-first, and responsible for telling NeoMutt how to assemble
 508         the tree.
 509         """
 510         KILL_LINE = r"\Ca\Ck"
 511
 512         if isinstance(item, Part):
 513             # We've hit a leaf-node, i.e. an alternative or a related part
 514             # with actual content.
 515
 516             # Let's add the part
 517             if item.orig:
 518                 # The original source already exists in the NeoMutt tree, but
 519                 # the underlying file may have been modified, so we need to
 520                 # update the encoding, but that's it:
 521                 cmds.push("<update-encoding>")
 522             else:
 523                 # … whereas all other parts need to be added, and they're all
 524                 # considered to be temporary and inline:
 525                 cmds.push(f"<attach-file>{item.path}<enter>")
 526                 cmds.push("<toggle-unlink><toggle-disposition>")
 527
 528             # If the item (including the original) comes with additional
 529             # information, then we might just as well update the NeoMutt
 530             # tree now:
 531             if item.cid:
 532                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 533
 534         elif isinstance(item, Multipart):
 535             # This node has children, but we already visited them (see
 536             # above), and so they have been tagged in NeoMutt's compose
 537             # window. Now it's just a matter of telling NeoMutt to do the
 538             # appropriate grouping:
 539             if item.subtype == "alternative":
 540                 cmds.push("<group-alternatives>")
 541             elif item.subtype in ("relative", "related"):
 542                 cmds.push("<group-related>")
 543             elif item.subtype == "multilingual":
 544                 cmds.push("<group-multilingual>")
 545
 546         else:
 547             # We should never get here
 548             assert not "is valid part"
 549
 550         # If the item has a description, we might just as well add it
 551         if item.desc:
 552             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 553
 554         # Finally, if we're at non-root level, tag the new container,
 555         # as it might itself be part of a container, to be processed
 556         # one level up:
 557         if stack:
 558             cmds.push("<tag-entry>")
 559
 560     # -----------------
 561     # End of visitor_fn
 562
 563     # Let's walk the tree and visit every node with our fancy visitor
 564     # function
 565     mimetree.walk(tree, visitor_fn=visitor_fn)
 566
 567     if not only_build:
 568         cmds.push("<send-message>")
 569
 570     # Finally, cleanup. Since we're responsible for removing the temporary
 571     # file, how's this for a little hack?
 572     try:
 573         filename = cmd_f.name
 574     except AttributeError:
 575         filename = "pytest_internal_file"
 576     cmds.cmd(f"source 'rm -f {filename}|'")
 577     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 578     cmds.flush()
 579
 580
 581 # [ CLI ENTRY ] ###############################################################
 582
 583 if __name__ == "__main__":
 584     args = parse_cli_args()
 585
 586     if args.mode is None:
 587         do_setup(
 588             tempdir=args.tempdir,
 589             debug_commands=args.debug_commands,
 590         )
 591
 592     elif args.mode == "massage":
 593         with open(args.MAILDRAFT, "r") as draft_f, open(
 594             args.cmdpath, "w"
 595         ) as cmd_f:
 596             do_massage(
 597                 draft_f,
 598                 pathlib.Path(args.MAILDRAFT),
 599                 cmd_f,
 600                 extensions=args.extensions,
 601                 cssfile=args.css_file,
 602                 only_build=args.only_build,
 603                 tempdir=args.tempdir,
 604                 debug_commands=args.debug_commands,
 605                 debug_walk=args.debug_walk,
 606             )
 607
 608
 609 # [ TESTS ] ###################################################################
 610
 611 try:
 612     import pytest
 613     from io import StringIO
 614
 615     class Tests:
 616         @pytest.fixture
 617         def const1(self):
 618             return "CONSTANT STRING 1"
 619
 620         @pytest.fixture
 621         def const2(self):
 622             return "CONSTANT STRING 2"
 623
 624         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 625         # functions they call, else old stdout is used and not captured
 626
 627         def test_MuttCommands_cmd(self, const1, const2, capsys):
 628             "Assert order of commands"
 629             cmds = MuttCommands(out_f=sys.stdout)
 630             cmds.cmd(const1)
 631             cmds.cmd(const2)
 632             cmds.flush()
 633             captured = capsys.readouterr()
 634             assert captured.out == "\n".join((const1, const2, ""))
 635
 636         def test_MuttCommands_push(self, const1, const2, capsys):
 637             "Assert reverse order of pushes"
 638             cmds = MuttCommands(out_f=sys.stdout)
 639             cmds.push(const1)
 640             cmds.push(const2)
 641             cmds.flush()
 642             captured = capsys.readouterr()
 643             assert (
 644                 captured.out
 645                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 646             )
 647
 648         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 649             "Assert reverse order of pushes"
 650             cmds = MuttCommands(out_f=sys.stdout)
 651             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 652             for i in range(2):
 653                 cmds.cmd(lines[4 * i + 0])
 654                 cmds.cmd(lines[4 * i + 1])
 655                 cmds.push(lines[4 * i + 2])
 656                 cmds.push(lines[4 * i + 3])
 657             cmds.flush()
 658
 659             captured = capsys.readouterr()
 660             lines_out = captured.out.splitlines()
 661             assert lines[0] in lines_out[0]
 662             assert lines[1] in lines_out[1]
 663             assert lines[7] in lines_out[2]
 664             assert lines[6] in lines_out[3]
 665             assert lines[3] in lines_out[4]
 666             assert lines[2] in lines_out[5]
 667             assert lines[4] in lines_out[6]
 668             assert lines[5] in lines_out[7]
 669
 670         @pytest.fixture
 671         def basic_mime_tree(self):
 672             return Multipart(
 673                 "relative",
 674                 children=[
 675                     Multipart(
 676                         "alternative",
 677                         children=[
 678                             Part(
 679                                 "text",
 680                                 "plain",
 681                                 "part.txt",
 682                                 desc="Plain",
 683                                 orig=True,
 684                             ),
 685                             Part("text", "html", "part.html", desc="HTML"),
 686                         ],
 687                         desc="Alternative",
 688                     ),
 689                     Part(
 690                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 691                     ),
 692                 ],
 693                 desc="Related",
 694             )
 695
 696         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 697             mimetree = MIMETreeDFWalker()
 698
 699             items = []
 700
 701             def visitor_fn(item, stack, debugprint):
 702                 items.append((item, len(stack)))
 703
 704             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 705             assert len(items) == 5
 706             assert items[0][0].subtype == "plain"
 707             assert items[0][1] == 2
 708             assert items[1][0].subtype == "html"
 709             assert items[1][1] == 2
 710             assert items[2][0].subtype == "alternative"
 711             assert items[2][1] == 1
 712             assert items[3][0].subtype == "png"
 713             assert items[3][1] == 1
 714             assert items[4][0].subtype == "relative"
 715             assert items[4][1] == 0
 716
 717         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
 718             mimetree = MIMETreeDFWalker()
 719             items = []
 720
 721             def visitor_fn(item, stack, debugprint):
 722                 items.append(item)
 723
 724             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
 725             assert items[-1].subtype == "mixed"
 726
 727         def test_MIMETreeDFWalker_visitor_in_constructor(
 728             self, basic_mime_tree
 729         ):
 730             items = []
 731
 732             def visitor_fn(item, stack, debugprint):
 733                 items.append(item)
 734
 735             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 736             mimetree.walk(basic_mime_tree)
 737             assert len(items) == 5
 738
 739         @pytest.fixture
 740         def string_io(self, const1, text=None):
 741             return StringIO(text or const1)
 742
 743         def test_do_massage_basic(self, const1, string_io, capsys):
 744             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 745                 return Part("text", "plain", draftpath, orig=True)
 746
 747             do_massage(
 748                 draft_f=string_io,
 749                 draftpath=const1,
 750                 cmd_f=sys.stdout,
 751                 converter=converter,
 752             )
 753
 754             captured = capsys.readouterr()
 755             lines = captured.out.splitlines()
 756             assert '="$my_editor"' in lines.pop(0)
 757             assert '="$my_edit_headers"' in lines.pop(0)
 758             assert "unset my_editor" == lines.pop(0)
 759             assert "unset my_edit_headers" == lines.pop(0)
 760             assert "send-message" in lines.pop(0)
 761             assert "update-encoding" in lines.pop(0)
 762             assert "source 'rm -f " in lines.pop(0)
 763             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 764
 765         def test_do_massage_fulltree(
 766             self, string_io, const1, basic_mime_tree, capsys
 767         ):
 768             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 769                 return basic_mime_tree
 770
 771             do_massage(
 772                 draft_f=string_io,
 773                 draftpath=const1,
 774                 cmd_f=sys.stdout,
 775                 converter=converter,
 776             )
 777
 778             captured = capsys.readouterr()
 779             lines = captured.out.splitlines()[4:]
 780             assert "send-message" in lines.pop(0)
 781             assert "Related" in lines.pop(0)
 782             assert "group-related" in lines.pop(0)
 783             assert "tag-entry" in lines.pop(0)
 784             assert "Logo" in lines.pop(0)
 785             assert "content-id" in lines.pop(0)
 786             assert "toggle-unlink" in lines.pop(0)
 787             assert "logo.png" in lines.pop(0)
 788             assert "tag-entry" in lines.pop(0)
 789             assert "Alternative" in lines.pop(0)
 790             assert "group-alternatives" in lines.pop(0)
 791             assert "tag-entry" in lines.pop(0)
 792             assert "HTML" in lines.pop(0)
 793             assert "toggle-unlink" in lines.pop(0)
 794             assert "part.html" in lines.pop(0)
 795             assert "tag-entry" in lines.pop(0)
 796             assert "Plain" in lines.pop(0)
 797             assert "update-encoding" in lines.pop(0)
 798             assert len(lines) == 2
 799
 800         @pytest.fixture
 801         def fake_filewriter(self):
 802             class FileWriter:
 803                 def __init__(self):
 804                     self._writes = []
 805
 806                 def __call__(self, path, content, mode="w", **kwargs):
 807                     self._writes.append((path, content))
 808
 809                 def pop(self, index=-1):
 810                     return self._writes.pop(index)
 811
 812             return FileWriter()
 813
 814         @pytest.fixture
 815         def markdown_non_converter(self, const1, const2):
 816             return lambda s, text: f"{const1}{text}{const2}"
 817
 818         def test_converter_tree_basic(
 819             self, const1, const2, fake_filewriter, markdown_non_converter
 820         ):
 821             path = pathlib.Path(const2)
 822             tree = convert_markdown_to_html(
 823                 const1, path, filewriter_fn=fake_filewriter
 824             )
 825
 826             assert tree.subtype == "alternative"
 827             assert len(tree.children) == 2
 828             assert tree.children[0].subtype == "plain"
 829             assert tree.children[0].path == path
 830             assert tree.children[0].orig
 831             assert tree.children[1].subtype == "html"
 832             assert tree.children[1].path == path.with_suffix(".html")
 833
 834         def test_converter_writes(
 835             self,
 836             const1,
 837             const2,
 838             fake_filewriter,
 839             monkeypatch,
 840             markdown_non_converter,
 841         ):
 842             path = pathlib.Path(const2)
 843
 844             with monkeypatch.context() as m:
 845                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 846                 convert_markdown_to_html(
 847                     const1, path, filewriter_fn=fake_filewriter
 848                 )
 849
 850             assert (path, const1) == fake_filewriter.pop(0)
 851             assert (
 852                 path.with_suffix(".html"),
 853                 markdown_non_converter(None, const1),
 854             ) == fake_filewriter.pop(0)
 855
 856         def test_markdown_inline_image_processor(self):
 857             imgpath1 = "file:/path/to/image.png"
 858             imgpath2 = "file:///path/to/image.png?url=params"
 859             imgpath3 = "/path/to/image.png"
 860             text = f"""![inline local image]({imgpath1})
 861                        ![image inlined
 862                          with newline]({imgpath2})
 863                        ![image local path]({imgpath3})"""
 864             text, html, images = markdown_with_inline_image_support(text)
 865
 866             # local paths have been normalised to URLs:
 867             imgpath3 = f"file://{imgpath3}"
 868
 869             assert 'src="cid:' in html
 870             assert "](cid:" in text
 871             assert len(images) == 3
 872             assert imgpath1 in images
 873             assert imgpath2 in images
 874             assert imgpath3 in images
 875             assert images[imgpath1].cid != images[imgpath2].cid
 876             assert images[imgpath1].cid != images[imgpath3].cid
 877             assert images[imgpath2].cid != images[imgpath3].cid
 878
 879         def test_markdown_inline_image_processor_title_to_desc(self, const1):
 880             imgpath = "file:///path/to/image.png"
 881             text = f'![inline local image]({imgpath} "{const1}")'
 882             text, html, images = markdown_with_inline_image_support(text)
 883             assert images[imgpath].desc == const1
 884
 885         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
 886             imgpath = "file:///path/to/image.png"
 887             text = f"![{const1}]({imgpath})"
 888             text, html, images = markdown_with_inline_image_support(text)
 889             assert images[imgpath].desc == const1
 890
 891         def test_markdown_inline_image_processor_title_over_alt_desc(
 892             self, const1, const2
 893         ):
 894             imgpath = "file:///path/to/image.png"
 895             text = f'![{const1}]({imgpath} "{const2}")'
 896             text, html, images = markdown_with_inline_image_support(text)
 897             assert images[imgpath].desc == const2
 898
 899         def test_markdown_inline_image_not_external(self):
 900             imgpath = "https://path/to/image.png"
 901             text = f"![inline image]({imgpath})"
 902             text, html, images = markdown_with_inline_image_support(text)
 903
 904             assert 'src="cid:' not in html
 905             assert "](cid:" not in text
 906             assert len(images) == 0
 907
 908         def test_markdown_inline_image_local_file(self):
 909             imgpath = "/path/to/image.png"
 910             text = f"![inline image]({imgpath})"
 911             text, html, images = markdown_with_inline_image_support(text)
 912
 913             for k, v in images.items():
 914                 assert k == f"file://{imgpath}"
 915                 break
 916
 917         @pytest.fixture
 918         def test_png(self):
 919             return (
 920                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
 921                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
 922             )
 923
 924         def test_markdown_inline_image_processor_base64(self, test_png):
 925             text = f"![1px white inlined]({test_png})"
 926             text, html, images = markdown_with_inline_image_support(text)
 927
 928             assert 'src="cid:' in html
 929             assert "](cid:" in text
 930             assert len(images) == 1
 931             assert test_png in images
 932
 933         def test_converter_tree_inline_image_base64(
 934             self, test_png, const1, fake_filewriter
 935         ):
 936             text = f"![inline base64 image]({test_png})"
 937             path = pathlib.Path(const1)
 938             tree = convert_markdown_to_html(
 939                 text, path, filewriter_fn=fake_filewriter
 940             )
 941
 942             assert tree.subtype == "relative"
 943             assert tree.children[1].subtype == "png"
 944             written = fake_filewriter.pop()
 945             assert tree.children[1].path == written[0]
 946             assert written[1] == request.urlopen(test_png).read()
 947
 948         def test_inline_image_collection(
 949             self, test_png, const1, const2, fake_filewriter
 950         ):
 951             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
 952             relparts = collect_inline_images(
 953                 test_images, filewriter_fn=fake_filewriter
 954             )
 955
 956             written = fake_filewriter.pop()
 957             assert b"PNG" in written[1]
 958
 959             assert relparts[0].subtype == "png"
 960             assert relparts[0].path == written[0]
 961             assert relparts[0].cid == const1
 962             assert relparts[0].desc.endswith(const2)
 963
 964         def test_apply_stylesheet(self):
 965             if _PYNLINER:
 966                 html = "<p>Hello, world!</p>"
 967                 css = "p { color:red }"
 968                 out = apply_styling(html, css)
 969                 assert 'p style="color' in out
 970
 971         def test_apply_stylesheet_pygments(self):
 972             if _PYGMENTS_CSS:
 973                 html = (
 974                     f'<div class="{_CODEHILITE_CLASS}">'
 975                     "<pre>def foo():\n    return</pre></div>"
 976                 )
 977                 out = apply_styling(html, _PYGMENTS_CSS)
 978                 assert f'{_CODEHILITE_CLASS}" style="' in out
 979
 980
 981 except ImportError:
 982     pass