.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--only-build",
  81         action="store_true",
  82         help="Only build, don't send the message",
  83     )
  84
  85     parser.add_argument(
  86         "--tempdir",
  87         default=None,
  88         help="Specify temporary directory to use for attachments",
  89     )
  90
  91     parser.add_argument(
  92         "--debug-commands",
  93         action="store_true",
  94         help="Turn on debug logging of commands generated to stderr",
  95     )
  96
  97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  98     massage_p = subp.add_parser(
  99         "massage", help="Massaging phase (internal use)"
 100     )
 101
 102     massage_p.add_argument(
 103         "--write-commands-to",
 104         "-o",
 105         metavar="PATH",
 106         dest="cmdpath",
 107         required=True,
 108         help="Temporary file path to write commands to",
 109     )
 110
 111     massage_p.add_argument(
 112         "--debug-walk",
 113         action="store_true",
 114         help="Turn on debugging to stderr of the MIME tree walk",
 115     )
 116
 117     massage_p.add_argument(
 118         "MAILDRAFT",
 119         nargs="?",
 120         help="If provided, the script is invoked as editor on the mail draft",
 121     )
 122
 123     return parser.parse_args(*args, **kwargs)
 124
 125
 126 # [ MARKDOWN WRAPPING ] #######################################################
 127
 128
 129 InlineImageInfo = namedtuple(
 130     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 131 )
 132
 133
 134 class InlineImageExtension(Extension):
 135     class RelatedImageInlineProcessor(ImageInlineProcessor):
 136         def __init__(self, re, md, ext):
 137             super().__init__(re, md)
 138             self._ext = ext
 139
 140         def handleMatch(self, m, data):
 141             el, start, end = super().handleMatch(m, data)
 142             if "src" in el.attrib:
 143                 src = el.attrib["src"]
 144                 if "://" not in src or src.startswith("file://"):
 145                     # We only inline local content
 146                     cid = self._ext.get_cid_for_image(el.attrib)
 147                     el.attrib["src"] = f"cid:{cid}"
 148             return el, start, end
 149
 150     def __init__(self):
 151         super().__init__()
 152         self._images = OrderedDict()
 153
 154     def extendMarkdown(self, md):
 155         md.registerExtension(self)
 156         inline_image_proc = self.RelatedImageInlineProcessor(
 157             IMAGE_LINK_RE, md, self
 158         )
 159         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 160
 161     def get_cid_for_image(self, attrib):
 162         msgid = make_msgid()[1:-1]
 163         path = attrib["src"]
 164         if path.startswith("/"):
 165             path = f"file://{path}"
 166         self._images[path] = InlineImageInfo(
 167             msgid, attrib.get("title", attrib.get("alt"))
 168         )
 169         return msgid
 170
 171     def get_images(self):
 172         return self._images
 173
 174
 175 def markdown_with_inline_image_support(
 176     text, *, extensions=None, extension_configs=None
 177 ):
 178     inline_image_handler = InlineImageExtension()
 179     extensions = extensions or []
 180     extensions.append(inline_image_handler)
 181     mdwn = markdown.Markdown(
 182         extensions=extensions, extension_configs=extension_configs
 183     )
 184     htmltext = mdwn.convert(text)
 185
 186     images = inline_image_handler.get_images()
 187
 188     def replace_image_with_cid(matchobj):
 189         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 190             if m in images:
 191                 return f"(cid:{images[m].cid}"
 192         return matchobj.group(0)
 193
 194     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 195     return text, htmltext, images
 196
 197
 198 # [ CSS STYLING ] #############################################################
 199
 200 try:
 201     import pynliner
 202
 203     _PYNLINER = True
 204
 205 except ImportError:
 206     _PYNLINER = False
 207
 208 try:
 209     from pygments.formatters import get_formatter_by_name
 210
 211     _CODEHILITE_CLASS = "codehilite"
 212
 213     _PYGMENTS_CSS = get_formatter_by_name(
 214         "html", style="default"
 215     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 216
 217 except ImportError:
 218     _PYGMENTS_CSS = None
 219
 220
 221 def apply_styling(html, css):
 222     return (
 223         pynliner.Pynliner()
 224         .from_string(html)
 225         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 226         .run()
 227     )
 228
 229
 230 # [ PARTS GENERATION ] ########################################################
 231
 232
 233 class Part(
 234     namedtuple(
 235         "Part",
 236         ["type", "subtype", "path", "desc", "cid", "orig"],
 237         defaults=[None, None, False],
 238     )
 239 ):
 240     def __str__(self):
 241         ret = f"<{self.type}/{self.subtype}>"
 242         if self.cid:
 243             ret = f"{ret} cid:{self.cid}"
 244         if self.orig:
 245             ret = f"{ret} ORIGINAL"
 246         return ret
 247
 248
 249 class Multipart(
 250     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 251 ):
 252     def __str__(self):
 253         return f"<multipart/{self.subtype}> children={len(self.children)}"
 254
 255     def __hash__(self):
 256         return hash(str(self.subtype) + "".join(str(self.children)))
 257
 258
 259 def filewriter_fn(path, content, mode="w", **kwargs):
 260     with open(path, mode, **kwargs) as out_f:
 261         out_f.write(content)
 262
 263
 264 def collect_inline_images(
 265     images, *, tempdir=None, filewriter_fn=filewriter_fn
 266 ):
 267     relparts = []
 268     for path, info in images.items():
 269         if path.startswith("cid:"):
 270             continue
 271
 272         data = request.urlopen(path)
 273
 274         mimetype = data.headers["Content-Type"]
 275         ext = mimetypes.guess_extension(mimetype)
 276         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 277         path = pathlib.Path(tempfilename[1])
 278
 279         filewriter_fn(path, data.read(), "w+b")
 280
 281         relparts.append(
 282             Part(
 283                 *mimetype.split("/"),
 284                 path,
 285                 cid=info.cid,
 286                 desc=f"Image: {info.desc}",
 287             )
 288         )
 289
 290     return relparts
 291
 292
 293 def convert_markdown_to_html(
 294     origtext,
 295     draftpath,
 296     *,
 297     cssfile=None,
 298     filewriter_fn=filewriter_fn,
 299     tempdir=None,
 300     extensions=None,
 301     extension_configs=None,
 302 ):
 303     # TODO extension_configs need to be handled differently
 304     extension_configs = extension_configs or {}
 305     extension_configs.setdefault("pymdownx.highlight", {})
 306     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 307
 308     origtext, htmltext, images = markdown_with_inline_image_support(
 309         origtext, extensions=extensions, extension_configs=extension_configs
 310     )
 311
 312     filewriter_fn(draftpath, origtext, encoding="utf-8")
 313     textpart = Part(
 314         "text", "plain", draftpath, "Plain-text version", orig=True
 315     )
 316
 317     htmltext = apply_styling(htmltext, cssfile)
 318
 319     htmlpath = draftpath.with_suffix(".html")
 320     filewriter_fn(
 321         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 322     )
 323     htmlpart = Part("text", "html", htmlpath, "HTML version")
 324
 325     altpart = Multipart(
 326         "alternative", [textpart, htmlpart], "Group of alternative content"
 327     )
 328
 329     imgparts = collect_inline_images(
 330         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 331     )
 332     if imgparts:
 333         return Multipart(
 334             "relative", [altpart] + imgparts, "Group of related content"
 335         )
 336     else:
 337         return altpart
 338
 339
 340 class MIMETreeDFWalker:
 341     def __init__(self, *, visitor_fn=None, debug=False):
 342         self._visitor_fn = visitor_fn or self._echovisit
 343         self._debug = debug
 344
 345     def _echovisit(self, node, ancestry, debugprint):
 346         debugprint(f"node={node} ancestry={ancestry}")
 347
 348     def walk(self, root, *, visitor_fn=None):
 349         """
 350         Recursive function to implement a depth-dirst walk of the MIME-tree
 351         rooted at `root`.
 352         """
 353
 354         if isinstance(root, list):
 355             root = Multipart("mixed", children=root)
 356
 357         self._walk(
 358             root,
 359             stack=[],
 360             visitor_fn=visitor_fn or self._visitor_fn,
 361         )
 362
 363     def _walk(self, node, *, stack, visitor_fn):
 364         # Let's start by enumerating the parts at the current level. At the
 365         # root level, stack will be the empty list, and we expect a multipart/*
 366         # container at this level. Later, e.g. within a mutlipart/alternative
 367         # container, the subtree will just be the alternative parts, while the
 368         # top of the stack will be the multipart/alternative container, which
 369         # we will process after the following loop.
 370
 371         lead = f"{'| '*len(stack)}|-"
 372         if isinstance(node, Multipart):
 373             self.debugprint(
 374                 f"{lead}{node} parents={[s.subtype for s in stack]}"
 375             )
 376
 377             # Depth-first, so push the current container onto the stack,
 378             # then descend …
 379             stack.append(node)
 380             self.debugprint("| " * (len(stack) + 1))
 381             for child in node.children:
 382                 self._walk(
 383                     child,
 384                     stack=stack,
 385                     visitor_fn=visitor_fn,
 386                 )
 387             self.debugprint("| " * len(stack))
 388             assert stack.pop() == node
 389
 390         else:
 391             self.debugprint(f"{lead}{node}")
 392
 393         if visitor_fn:
 394             visitor_fn(node, stack, debugprint=self.debugprint)
 395
 396     def debugprint(self, s, **kwargs):
 397         if self._debug:
 398             print(s, file=sys.stderr, **kwargs)
 399
 400
 401 # [ RUN MODES ] ###############################################################
 402
 403
 404 class MuttCommands:
 405     """
 406     Stupid class to interface writing out Mutt commands. This is quite a hack
 407     to deal with the fact that Mutt runs "push" commands in reverse order, so
 408     all of a sudden, things become very complicated when mixing with "real"
 409     commands.
 410
 411     Hence we keep two sets of commands, and one set of pushes. Commands are
 412     added to the first until a push is added, after which commands are added to
 413     the second set of commands.
 414
 415     On flush(), the first set is printed, followed by the pushes in reverse,
 416     and then the second set is printed. All 3 sets are then cleared.
 417     """
 418
 419     def __init__(self, out_f=sys.stdout, *, debug=False):
 420         self._cmd1, self._push, self._cmd2 = [], [], []
 421         self._out_f = out_f
 422         self._debug = debug
 423
 424     def cmd(self, s):
 425         self.debugprint(s)
 426         if self._push:
 427             self._cmd2.append(s)
 428         else:
 429             self._cmd1.append(s)
 430
 431     def push(self, s):
 432         s = s.replace('"', '"')
 433         s = f'push "{s}"'
 434         self.debugprint(s)
 435         self._push.insert(0, s)
 436
 437     def flush(self):
 438         print(
 439             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 440         )
 441         self._cmd1, self._push, self._cmd2 = [], [], []
 442
 443     def debugprint(self, s, **kwargs):
 444         if self._debug:
 445             print(s, file=sys.stderr, **kwargs)
 446
 447
 448 def do_setup(
 449     *,
 450     out_f=sys.stdout,
 451     temppath=None,
 452     tempdir=None,
 453     debug_commands=False,
 454 ):
 455     temppath = temppath or pathlib.Path(
 456         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 457     )
 458     cmds = MuttCommands(out_f, debug=debug_commands)
 459
 460     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 461
 462     cmds.cmd('set my_editor="$editor"')
 463     cmds.cmd('set my_edit_headers="$edit_headers"')
 464     cmds.cmd(f'set editor="{editor}"')
 465     cmds.cmd("unset edit_headers")
 466     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 467     cmds.push("<first-entry><edit-file>")
 468     cmds.flush()
 469
 470
 471 def do_massage(
 472     draft_f,
 473     draftpath,
 474     cmd_f,
 475     *,
 476     extensions=None,
 477     cssfile=None,
 478     converter=convert_markdown_to_html,
 479     only_build=False,
 480     tempdir=None,
 481     debug_commands=False,
 482     debug_walk=False,
 483 ):
 484     # Here's the big picture: we're being invoked as the editor on the email
 485     # draft, and whatever commands we write to the file given as cmdpath will
 486     # be run by the second source command in the macro definition.
 487
 488     # Let's start by cleaning up what the setup did (see above), i.e. we
 489     # restore the $editor and $edit_headers variables, and also unset the
 490     # variable used to identify the command file we're currently writing
 491     # to.
 492     cmds = MuttCommands(cmd_f, debug=debug_commands)
 493     cmds.cmd('set editor="$my_editor"')
 494     cmds.cmd('set edit_headers="$my_edit_headers"')
 495     cmds.cmd("unset my_editor")
 496     cmds.cmd("unset my_edit_headers")
 497
 498     # let's flush those commands, as there'll be a lot of pushes from now
 499     # on, which need to be run in reverse order
 500     cmds.flush()
 501
 502     extensions = extensions.split(",") if extensions else []
 503     tree = converter(
 504         draft_f.read(),
 505         draftpath,
 506         cssfile=cssfile,
 507         tempdir=tempdir,
 508         extensions=extensions,
 509     )
 510
 511     mimetree = MIMETreeDFWalker(debug=debug_walk)
 512
 513     def visitor_fn(item, stack, *, debugprint=None):
 514         """
 515         Visitor function called for every node (part) of the MIME tree,
 516         depth-first, and responsible for telling NeoMutt how to assemble
 517         the tree.
 518         """
 519         KILL_LINE = r"\Ca\Ck"
 520
 521         if isinstance(item, Part):
 522             # We've hit a leaf-node, i.e. an alternative or a related part
 523             # with actual content.
 524
 525             # Let's add the part
 526             if item.orig:
 527                 # The original source already exists in the NeoMutt tree, but
 528                 # the underlying file may have been modified, so we need to
 529                 # update the encoding, but that's it:
 530                 cmds.push("<update-encoding>")
 531             else:
 532                 # … whereas all other parts need to be added, and they're all
 533                 # considered to be temporary and inline:
 534                 cmds.push(f"<attach-file>{item.path}<enter>")
 535                 cmds.push("<toggle-unlink><toggle-disposition>")
 536
 537             # If the item (including the original) comes with additional
 538             # information, then we might just as well update the NeoMutt
 539             # tree now:
 540             if item.cid:
 541                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 542
 543         elif isinstance(item, Multipart):
 544             # This node has children, but we already visited them (see
 545             # above), and so they have been tagged in NeoMutt's compose
 546             # window. Now it's just a matter of telling NeoMutt to do the
 547             # appropriate grouping:
 548             if item.subtype == "alternative":
 549                 cmds.push("<group-alternatives>")
 550             elif item.subtype in ("relative", "related"):
 551                 cmds.push("<group-related>")
 552             elif item.subtype == "multilingual":
 553                 cmds.push("<group-multilingual>")
 554
 555         else:
 556             # We should never get here
 557             assert not "is valid part"
 558
 559         # If the item has a description, we might just as well add it
 560         if item.desc:
 561             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 562
 563         # Finally, if we're at non-root level, tag the new container,
 564         # as it might itself be part of a container, to be processed
 565         # one level up:
 566         if stack:
 567             cmds.push("<tag-entry>")
 568
 569     # -----------------
 570     # End of visitor_fn
 571
 572     # Let's walk the tree and visit every node with our fancy visitor
 573     # function
 574     mimetree.walk(tree, visitor_fn=visitor_fn)
 575
 576     if not only_build:
 577         cmds.push("<send-message>")
 578
 579     # Finally, cleanup. Since we're responsible for removing the temporary
 580     # file, how's this for a little hack?
 581     try:
 582         filename = cmd_f.name
 583     except AttributeError:
 584         filename = "pytest_internal_file"
 585     cmds.cmd(f"source 'rm -f {filename}|'")
 586     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 587     cmds.flush()
 588
 589
 590 # [ CLI ENTRY ] ###############################################################
 591
 592 if __name__ == "__main__":
 593     args = parse_cli_args()
 594
 595     if args.mode is None:
 596         do_setup(
 597             tempdir=args.tempdir,
 598             debug_commands=args.debug_commands,
 599         )
 600
 601     elif args.mode == "massage":
 602         with open(args.MAILDRAFT, "r") as draft_f, open(
 603             args.cmdpath, "w"
 604         ) as cmd_f:
 605             do_massage(
 606                 draft_f,
 607                 pathlib.Path(args.MAILDRAFT),
 608                 cmd_f,
 609                 extensions=args.extensions,
 610                 cssfile=args.css_file,
 611                 only_build=args.only_build,
 612                 tempdir=args.tempdir,
 613                 debug_commands=args.debug_commands,
 614                 debug_walk=args.debug_walk,
 615             )
 616
 617
 618 # [ TESTS ] ###################################################################
 619
 620 try:
 621     import pytest
 622     from io import StringIO
 623
 624     class Tests:
 625         @pytest.fixture
 626         def const1(self):
 627             return "CONSTANT STRING 1"
 628
 629         @pytest.fixture
 630         def const2(self):
 631             return "CONSTANT STRING 2"
 632
 633         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 634         # functions they call, else old stdout is used and not captured
 635
 636         def test_MuttCommands_cmd(self, const1, const2, capsys):
 637             "Assert order of commands"
 638             cmds = MuttCommands(out_f=sys.stdout)
 639             cmds.cmd(const1)
 640             cmds.cmd(const2)
 641             cmds.flush()
 642             captured = capsys.readouterr()
 643             assert captured.out == "\n".join((const1, const2, ""))
 644
 645         def test_MuttCommands_push(self, const1, const2, capsys):
 646             "Assert reverse order of pushes"
 647             cmds = MuttCommands(out_f=sys.stdout)
 648             cmds.push(const1)
 649             cmds.push(const2)
 650             cmds.flush()
 651             captured = capsys.readouterr()
 652             assert (
 653                 captured.out
 654                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 655             )
 656
 657         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 658             "Assert reverse order of pushes"
 659             cmds = MuttCommands(out_f=sys.stdout)
 660             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 661             for i in range(2):
 662                 cmds.cmd(lines[4 * i + 0])
 663                 cmds.cmd(lines[4 * i + 1])
 664                 cmds.push(lines[4 * i + 2])
 665                 cmds.push(lines[4 * i + 3])
 666             cmds.flush()
 667
 668             captured = capsys.readouterr()
 669             lines_out = captured.out.splitlines()
 670             assert lines[0] in lines_out[0]
 671             assert lines[1] in lines_out[1]
 672             assert lines[7] in lines_out[2]
 673             assert lines[6] in lines_out[3]
 674             assert lines[3] in lines_out[4]
 675             assert lines[2] in lines_out[5]
 676             assert lines[4] in lines_out[6]
 677             assert lines[5] in lines_out[7]
 678
 679         @pytest.fixture
 680         def basic_mime_tree(self):
 681             return Multipart(
 682                 "relative",
 683                 children=[
 684                     Multipart(
 685                         "alternative",
 686                         children=[
 687                             Part(
 688                                 "text",
 689                                 "plain",
 690                                 "part.txt",
 691                                 desc="Plain",
 692                                 orig=True,
 693                             ),
 694                             Part("text", "html", "part.html", desc="HTML"),
 695                         ],
 696                         desc="Alternative",
 697                     ),
 698                     Part(
 699                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 700                     ),
 701                 ],
 702                 desc="Related",
 703             )
 704
 705         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 706             mimetree = MIMETreeDFWalker()
 707
 708             items = []
 709
 710             def visitor_fn(item, stack, debugprint):
 711                 items.append((item, len(stack)))
 712
 713             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 714             assert len(items) == 5
 715             assert items[0][0].subtype == "plain"
 716             assert items[0][1] == 2
 717             assert items[1][0].subtype == "html"
 718             assert items[1][1] == 2
 719             assert items[2][0].subtype == "alternative"
 720             assert items[2][1] == 1
 721             assert items[3][0].subtype == "png"
 722             assert items[3][1] == 1
 723             assert items[4][0].subtype == "relative"
 724             assert items[4][1] == 0
 725
 726         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
 727             mimetree = MIMETreeDFWalker()
 728             items = []
 729
 730             def visitor_fn(item, stack, debugprint):
 731                 items.append(item)
 732
 733             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
 734             assert items[-1].subtype == "mixed"
 735
 736         def test_MIMETreeDFWalker_visitor_in_constructor(
 737             self, basic_mime_tree
 738         ):
 739             items = []
 740
 741             def visitor_fn(item, stack, debugprint):
 742                 items.append(item)
 743
 744             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 745             mimetree.walk(basic_mime_tree)
 746             assert len(items) == 5
 747
 748         @pytest.fixture
 749         def string_io(self, const1, text=None):
 750             return StringIO(text or const1)
 751
 752         def test_do_massage_basic(self, const1, string_io, capsys):
 753             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 754                 return Part("text", "plain", draftpath, orig=True)
 755
 756             do_massage(
 757                 draft_f=string_io,
 758                 draftpath=const1,
 759                 cmd_f=sys.stdout,
 760                 converter=converter,
 761             )
 762
 763             captured = capsys.readouterr()
 764             lines = captured.out.splitlines()
 765             assert '="$my_editor"' in lines.pop(0)
 766             assert '="$my_edit_headers"' in lines.pop(0)
 767             assert "unset my_editor" == lines.pop(0)
 768             assert "unset my_edit_headers" == lines.pop(0)
 769             assert "send-message" in lines.pop(0)
 770             assert "update-encoding" in lines.pop(0)
 771             assert "source 'rm -f " in lines.pop(0)
 772             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 773
 774         def test_do_massage_fulltree(
 775             self, string_io, const1, basic_mime_tree, capsys
 776         ):
 777             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 778                 return basic_mime_tree
 779
 780             do_massage(
 781                 draft_f=string_io,
 782                 draftpath=const1,
 783                 cmd_f=sys.stdout,
 784                 converter=converter,
 785             )
 786
 787             captured = capsys.readouterr()
 788             lines = captured.out.splitlines()[4:]
 789             assert "send-message" in lines.pop(0)
 790             assert "Related" in lines.pop(0)
 791             assert "group-related" in lines.pop(0)
 792             assert "tag-entry" in lines.pop(0)
 793             assert "Logo" in lines.pop(0)
 794             assert "content-id" in lines.pop(0)
 795             assert "toggle-unlink" in lines.pop(0)
 796             assert "logo.png" in lines.pop(0)
 797             assert "tag-entry" in lines.pop(0)
 798             assert "Alternative" in lines.pop(0)
 799             assert "group-alternatives" in lines.pop(0)
 800             assert "tag-entry" in lines.pop(0)
 801             assert "HTML" in lines.pop(0)
 802             assert "toggle-unlink" in lines.pop(0)
 803             assert "part.html" in lines.pop(0)
 804             assert "tag-entry" in lines.pop(0)
 805             assert "Plain" in lines.pop(0)
 806             assert "update-encoding" in lines.pop(0)
 807             assert len(lines) == 2
 808
 809         @pytest.fixture
 810         def fake_filewriter(self):
 811             class FileWriter:
 812                 def __init__(self):
 813                     self._writes = []
 814
 815                 def __call__(self, path, content, mode="w", **kwargs):
 816                     self._writes.append((path, content))
 817
 818                 def pop(self, index=-1):
 819                     return self._writes.pop(index)
 820
 821             return FileWriter()
 822
 823         @pytest.fixture
 824         def markdown_non_converter(self, const1, const2):
 825             return lambda s, text: f"{const1}{text}{const2}"
 826
 827         def test_converter_tree_basic(
 828             self, const1, const2, fake_filewriter, markdown_non_converter
 829         ):
 830             path = pathlib.Path(const2)
 831             tree = convert_markdown_to_html(
 832                 const1, path, filewriter_fn=fake_filewriter
 833             )
 834
 835             assert tree.subtype == "alternative"
 836             assert len(tree.children) == 2
 837             assert tree.children[0].subtype == "plain"
 838             assert tree.children[0].path == path
 839             assert tree.children[0].orig
 840             assert tree.children[1].subtype == "html"
 841             assert tree.children[1].path == path.with_suffix(".html")
 842
 843         def test_converter_writes(
 844             self,
 845             const1,
 846             const2,
 847             fake_filewriter,
 848             monkeypatch,
 849             markdown_non_converter,
 850         ):
 851             path = pathlib.Path(const2)
 852
 853             with monkeypatch.context() as m:
 854                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 855                 convert_markdown_to_html(
 856                     const1, path, filewriter_fn=fake_filewriter
 857                 )
 858
 859             assert (path, const1) == fake_filewriter.pop(0)
 860             assert (
 861                 path.with_suffix(".html"),
 862                 markdown_non_converter(None, const1),
 863             ) == fake_filewriter.pop(0)
 864
 865         def test_markdown_inline_image_processor(self):
 866             imgpath1 = "file:/path/to/image.png"
 867             imgpath2 = "file:///path/to/image.png?url=params"
 868             imgpath3 = "/path/to/image.png"
 869             text = f"""![inline local image]({imgpath1})
 870                        ![image inlined
 871                          with newline]({imgpath2})
 872                        ![image local path]({imgpath3})"""
 873             text, html, images = markdown_with_inline_image_support(text)
 874
 875             # local paths have been normalised to URLs:
 876             imgpath3 = f"file://{imgpath3}"
 877
 878             assert 'src="cid:' in html
 879             assert "](cid:" in text
 880             assert len(images) == 3
 881             assert imgpath1 in images
 882             assert imgpath2 in images
 883             assert imgpath3 in images
 884             assert images[imgpath1].cid != images[imgpath2].cid
 885             assert images[imgpath1].cid != images[imgpath3].cid
 886             assert images[imgpath2].cid != images[imgpath3].cid
 887
 888         def test_markdown_inline_image_processor_title_to_desc(self, const1):
 889             imgpath = "file:///path/to/image.png"
 890             text = f'![inline local image]({imgpath} "{const1}")'
 891             text, html, images = markdown_with_inline_image_support(text)
 892             assert images[imgpath].desc == const1
 893
 894         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
 895             imgpath = "file:///path/to/image.png"
 896             text = f"![{const1}]({imgpath})"
 897             text, html, images = markdown_with_inline_image_support(text)
 898             assert images[imgpath].desc == const1
 899
 900         def test_markdown_inline_image_processor_title_over_alt_desc(
 901             self, const1, const2
 902         ):
 903             imgpath = "file:///path/to/image.png"
 904             text = f'![{const1}]({imgpath} "{const2}")'
 905             text, html, images = markdown_with_inline_image_support(text)
 906             assert images[imgpath].desc == const2
 907
 908         def test_markdown_inline_image_not_external(self):
 909             imgpath = "https://path/to/image.png"
 910             text = f"![inline image]({imgpath})"
 911             text, html, images = markdown_with_inline_image_support(text)
 912
 913             assert 'src="cid:' not in html
 914             assert "](cid:" not in text
 915             assert len(images) == 0
 916
 917         def test_markdown_inline_image_local_file(self):
 918             imgpath = "/path/to/image.png"
 919             text = f"![inline image]({imgpath})"
 920             text, html, images = markdown_with_inline_image_support(text)
 921
 922             for k, v in images.items():
 923                 assert k == f"file://{imgpath}"
 924                 break
 925
 926         @pytest.fixture
 927         def test_png(self):
 928             return (
 929                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
 930                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
 931             )
 932
 933         def test_markdown_inline_image_processor_base64(self, test_png):
 934             text = f"![1px white inlined]({test_png})"
 935             text, html, images = markdown_with_inline_image_support(text)
 936
 937             assert 'src="cid:' in html
 938             assert "](cid:" in text
 939             assert len(images) == 1
 940             assert test_png in images
 941
 942         def test_converter_tree_inline_image_base64(
 943             self, test_png, const1, fake_filewriter
 944         ):
 945             text = f"![inline base64 image]({test_png})"
 946             path = pathlib.Path(const1)
 947             tree = convert_markdown_to_html(
 948                 text, path, filewriter_fn=fake_filewriter
 949             )
 950
 951             assert tree.subtype == "relative"
 952             assert tree.children[1].subtype == "png"
 953             written = fake_filewriter.pop()
 954             assert tree.children[1].path == written[0]
 955             assert written[1] == request.urlopen(test_png).read()
 956
 957         def test_converter_tree_inline_image_cid(
 958             self, const1, fake_filewriter
 959         ):
 960             text = f"![inline base64 image](cid:{const1})"
 961             path = pathlib.Path(const1)
 962             tree = convert_markdown_to_html(
 963                 text,
 964                 path,
 965                 filewriter_fn=fake_filewriter,
 966                 related_to_html_only=False,
 967             )
 968             assert len(tree.children) == 2
 969             assert tree.children[0].cid != const1
 970             assert tree.children[0].type != "image"
 971             assert tree.children[1].cid != const1
 972             assert tree.children[1].type != "image"
 973
 974         def test_inline_image_collection(
 975             self, test_png, const1, const2, fake_filewriter
 976         ):
 977             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
 978             relparts = collect_inline_images(
 979                 test_images, filewriter_fn=fake_filewriter
 980             )
 981
 982             written = fake_filewriter.pop()
 983             assert b"PNG" in written[1]
 984
 985             assert relparts[0].subtype == "png"
 986             assert relparts[0].path == written[0]
 987             assert relparts[0].cid == const1
 988             assert relparts[0].desc.endswith(const2)
 989
 990         def test_apply_stylesheet(self):
 991             if _PYNLINER:
 992                 html = "<p>Hello, world!</p>"
 993                 css = "p { color:red }"
 994                 out = apply_styling(html, css)
 995                 assert 'p style="color' in out
 996
 997         def test_apply_stylesheet_pygments(self):
 998             if _PYGMENTS_CSS:
 999                 html = (
1000                     f'<div class="{_CODEHILITE_CLASS}">'
1001                     "<pre>def foo():\n    return</pre></div>"
1002                 )
1003                 out = apply_styling(html, _PYGMENTS_CSS)
1004                 assert f'{_CODEHILITE_CLASS}" style="' in out
1005
1006
1007 except ImportError:
1008     pass