.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--only-build",
  81         action="store_true",
  82         help="Only build, don't send the message",
  83     )
  84
  85     parser.add_argument(
  86         "--tempdir",
  87         default=None,
  88         help="Specify temporary directory to use for attachments",
  89     )
  90
  91     parser.add_argument(
  92         "--debug-commands",
  93         action="store_true",
  94         help="Turn on debug logging of commands generated to stderr",
  95     )
  96
  97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  98     massage_p = subp.add_parser(
  99         "massage", help="Massaging phase (internal use)"
 100     )
 101
 102     massage_p.add_argument(
 103         "--write-commands-to",
 104         "-o",
 105         metavar="PATH",
 106         dest="cmdpath",
 107         required=True,
 108         help="Temporary file path to write commands to",
 109     )
 110
 111     massage_p.add_argument(
 112         "--debug-walk",
 113         action="store_true",
 114         help="Turn on debugging to stderr of the MIME tree walk",
 115     )
 116
 117     massage_p.add_argument(
 118         "MAILDRAFT",
 119         nargs="?",
 120         help="If provided, the script is invoked as editor on the mail draft",
 121     )
 122
 123     return parser.parse_args(*args, **kwargs)
 124
 125
 126 # [ MARKDOWN WRAPPING ] #######################################################
 127
 128
 129 InlineImageInfo = namedtuple(
 130     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 131 )
 132
 133
 134 class InlineImageExtension(Extension):
 135     class RelatedImageInlineProcessor(ImageInlineProcessor):
 136         def __init__(self, re, md, ext):
 137             super().__init__(re, md)
 138             self._ext = ext
 139
 140         def handleMatch(self, m, data):
 141             el, start, end = super().handleMatch(m, data)
 142             if "src" in el.attrib:
 143                 src = el.attrib["src"]
 144                 if "://" not in src or src.startswith("file://"):
 145                     # We only inline local content
 146                     cid = self._ext.get_cid_for_image(el.attrib)
 147                     el.attrib["src"] = f"cid:{cid}"
 148             return el, start, end
 149
 150     def __init__(self):
 151         super().__init__()
 152         self._images = OrderedDict()
 153
 154     def extendMarkdown(self, md):
 155         md.registerExtension(self)
 156         inline_image_proc = self.RelatedImageInlineProcessor(
 157             IMAGE_LINK_RE, md, self
 158         )
 159         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 160
 161     def get_cid_for_image(self, attrib):
 162         msgid = make_msgid()[1:-1]
 163         path = attrib["src"]
 164         if path.startswith("/"):
 165             path = f"file://{path}"
 166         self._images[path] = InlineImageInfo(
 167             msgid, attrib.get("title", attrib.get("alt"))
 168         )
 169         return msgid
 170
 171     def get_images(self):
 172         return self._images
 173
 174
 175 def markdown_with_inline_image_support(
 176     text, *, extensions=None, extension_configs=None
 177 ):
 178     inline_image_handler = InlineImageExtension()
 179     extensions = extensions or []
 180     extensions.append(inline_image_handler)
 181     mdwn = markdown.Markdown(
 182         extensions=extensions, extension_configs=extension_configs
 183     )
 184     htmltext = mdwn.convert(text)
 185
 186     images = inline_image_handler.get_images()
 187
 188     def replace_image_with_cid(matchobj):
 189         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 190             if m in images:
 191                 return f"(cid:{images[m].cid}"
 192         return matchobj.group(0)
 193
 194     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 195     return text, htmltext, images
 196
 197
 198 # [ CSS STYLING ] #############################################################
 199
 200 try:
 201     import pynliner
 202
 203     _PYNLINER = True
 204
 205 except ImportError:
 206     _PYNLINER = False
 207
 208 try:
 209     from pygments.formatters import get_formatter_by_name
 210
 211     _CODEHILITE_CLASS = "codehilite"
 212
 213     _PYGMENTS_CSS = get_formatter_by_name(
 214         "html", style="default"
 215     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 216
 217 except ImportError:
 218     _PYGMENTS_CSS = None
 219
 220
 221 def apply_styling(html, css):
 222     return (
 223         pynliner.Pynliner()
 224         .from_string(html)
 225         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 226         .run()
 227     )
 228
 229
 230 # [ PARTS GENERATION ] ########################################################
 231
 232
 233 class Part(
 234     namedtuple(
 235         "Part",
 236         ["type", "subtype", "path", "desc", "cid", "orig"],
 237         defaults=[None, None, False],
 238     )
 239 ):
 240     def __str__(self):
 241         ret = f"<{self.type}/{self.subtype}>"
 242         if self.cid:
 243             ret = f"{ret} cid:{self.cid}"
 244         if self.orig:
 245             ret = f"{ret} ORIGINAL"
 246         return ret
 247
 248
 249 class Multipart(
 250     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 251 ):
 252     def __str__(self):
 253         return f"<multipart/{self.subtype}> children={len(self.children)}"
 254
 255     def __hash__(self):
 256         return hash(str(self.subtype) + "".join(str(self.children)))
 257
 258
 259 def filewriter_fn(path, content, mode="w", **kwargs):
 260     with open(path, mode, **kwargs) as out_f:
 261         out_f.write(content)
 262
 263
 264 def collect_inline_images(
 265     images, *, tempdir=None, filewriter_fn=filewriter_fn
 266 ):
 267     relparts = []
 268     for path, info in images.items():
 269         if path.startswith("cid:"):
 270             continue
 271
 272         data = request.urlopen(path)
 273
 274         mimetype = data.headers["Content-Type"]
 275         ext = mimetypes.guess_extension(mimetype)
 276         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 277         path = pathlib.Path(tempfilename[1])
 278
 279         filewriter_fn(path, data.read(), "w+b")
 280
 281         relparts.append(
 282             Part(
 283                 *mimetype.split("/"),
 284                 path,
 285                 cid=info.cid,
 286                 desc=f"Image: {info.desc}",
 287             )
 288         )
 289
 290     return relparts
 291
 292
 293 def convert_markdown_to_html(
 294     origtext,
 295     draftpath,
 296     *,
 297     cssfile=None,
 298     filewriter_fn=filewriter_fn,
 299     tempdir=None,
 300     extensions=None,
 301     extension_configs=None,
 302 ):
 303     # TODO extension_configs need to be handled differently
 304     extension_configs = extension_configs or {}
 305     extension_configs.setdefault("pymdownx.highlight", {})
 306     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 307
 308     origtext, htmltext, images = markdown_with_inline_image_support(
 309         origtext, extensions=extensions, extension_configs=extension_configs
 310     )
 311
 312     filewriter_fn(draftpath, origtext, encoding="utf-8")
 313     textpart = Part(
 314         "text", "plain", draftpath, "Plain-text version", orig=True
 315     )
 316
 317     htmltext = apply_styling(htmltext, cssfile)
 318
 319     htmlpath = draftpath.with_suffix(".html")
 320     filewriter_fn(
 321         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 322     )
 323     htmlpart = Part("text", "html", htmlpath, "HTML version")
 324
 325     altpart = Multipart(
 326         "alternative", [textpart, htmlpart], "Group of alternative content"
 327     )
 328
 329     imgparts = collect_inline_images(
 330         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 331     )
 332     if imgparts:
 333         return Multipart(
 334             "relative", [altpart] + imgparts, "Group of related content"
 335         )
 336     else:
 337         return altpart
 338
 339
 340 class MIMETreeDFWalker:
 341     def __init__(self, *, visitor_fn=None, debug=False):
 342         self._visitor_fn = visitor_fn or self._echovisit
 343         self._debug = debug
 344
 345     def _echovisit(self, node, ancestry, debugprint):
 346         debugprint(f"node={node} ancestry={ancestry}")
 347
 348     def walk(self, root, *, visitor_fn=None):
 349         """
 350         Recursive function to implement a depth-dirst walk of the MIME-tree
 351         rooted at `root`.
 352         """
 353         if isinstance(root, list):
 354             if len(root) > 1:
 355                 root = Multipart("mixed", children=root)
 356             else:
 357                 root = root[0]
 358
 359         self._walk(
 360             root,
 361             stack=[],
 362             visitor_fn=visitor_fn or self._visitor_fn,
 363         )
 364
 365     def _walk(self, node, *, stack, visitor_fn):
 366         # Let's start by enumerating the parts at the current level. At the
 367         # root level, stack will be the empty list, and we expect a multipart/*
 368         # container at this level. Later, e.g. within a mutlipart/alternative
 369         # container, the subtree will just be the alternative parts, while the
 370         # top of the stack will be the multipart/alternative container, which
 371         # we will process after the following loop.
 372
 373         lead = f"{'| '*len(stack)}|-"
 374         if isinstance(node, Multipart):
 375             self.debugprint(
 376                 f"{lead}{node} parents={[s.subtype for s in stack]}"
 377             )
 378
 379             # Depth-first, so push the current container onto the stack,
 380             # then descend …
 381             stack.append(node)
 382             self.debugprint("| " * (len(stack) + 1))
 383             for child in node.children:
 384                 self._walk(
 385                     child,
 386                     stack=stack,
 387                     visitor_fn=visitor_fn,
 388                 )
 389             self.debugprint("| " * len(stack))
 390             assert stack.pop() == node
 391
 392         else:
 393             self.debugprint(f"{lead}{node}")
 394
 395         if visitor_fn:
 396             visitor_fn(node, stack, debugprint=self.debugprint)
 397
 398     def debugprint(self, s, **kwargs):
 399         if self._debug:
 400             print(s, file=sys.stderr, **kwargs)
 401
 402
 403 # [ RUN MODES ] ###############################################################
 404
 405
 406 class MuttCommands:
 407     """
 408     Stupid class to interface writing out Mutt commands. This is quite a hack
 409     to deal with the fact that Mutt runs "push" commands in reverse order, so
 410     all of a sudden, things become very complicated when mixing with "real"
 411     commands.
 412
 413     Hence we keep two sets of commands, and one set of pushes. Commands are
 414     added to the first until a push is added, after which commands are added to
 415     the second set of commands.
 416
 417     On flush(), the first set is printed, followed by the pushes in reverse,
 418     and then the second set is printed. All 3 sets are then cleared.
 419     """
 420
 421     def __init__(self, out_f=sys.stdout, *, debug=False):
 422         self._cmd1, self._push, self._cmd2 = [], [], []
 423         self._out_f = out_f
 424         self._debug = debug
 425
 426     def cmd(self, s):
 427         self.debugprint(s)
 428         if self._push:
 429             self._cmd2.append(s)
 430         else:
 431             self._cmd1.append(s)
 432
 433     def push(self, s):
 434         s = s.replace('"', '"')
 435         s = f'push "{s}"'
 436         self.debugprint(s)
 437         self._push.insert(0, s)
 438
 439     def flush(self):
 440         print(
 441             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 442         )
 443         self._cmd1, self._push, self._cmd2 = [], [], []
 444
 445     def debugprint(self, s, **kwargs):
 446         if self._debug:
 447             print(s, file=sys.stderr, **kwargs)
 448
 449
 450 def do_setup(
 451     *,
 452     out_f=sys.stdout,
 453     temppath=None,
 454     tempdir=None,
 455     debug_commands=False,
 456 ):
 457     temppath = temppath or pathlib.Path(
 458         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 459     )
 460     cmds = MuttCommands(out_f, debug=debug_commands)
 461
 462     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 463
 464     cmds.cmd('set my_editor="$editor"')
 465     cmds.cmd('set my_edit_headers="$edit_headers"')
 466     cmds.cmd(f'set editor="{editor}"')
 467     cmds.cmd("unset edit_headers")
 468     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 469     cmds.push("<first-entry><edit-file>")
 470     cmds.flush()
 471
 472
 473 def do_massage(
 474     draft_f,
 475     draftpath,
 476     cmd_f,
 477     *,
 478     extensions=None,
 479     cssfile=None,
 480     converter=convert_markdown_to_html,
 481     only_build=False,
 482     tempdir=None,
 483     debug_commands=False,
 484     debug_walk=False,
 485 ):
 486     # Here's the big picture: we're being invoked as the editor on the email
 487     # draft, and whatever commands we write to the file given as cmdpath will
 488     # be run by the second source command in the macro definition.
 489
 490     # Let's start by cleaning up what the setup did (see above), i.e. we
 491     # restore the $editor and $edit_headers variables, and also unset the
 492     # variable used to identify the command file we're currently writing
 493     # to.
 494     cmds = MuttCommands(cmd_f, debug=debug_commands)
 495     cmds.cmd('set editor="$my_editor"')
 496     cmds.cmd('set edit_headers="$my_edit_headers"')
 497     cmds.cmd("unset my_editor")
 498     cmds.cmd("unset my_edit_headers")
 499
 500     # let's flush those commands, as there'll be a lot of pushes from now
 501     # on, which need to be run in reverse order
 502     cmds.flush()
 503
 504     extensions = extensions.split(",") if extensions else []
 505     tree = converter(
 506         draft_f.read(),
 507         draftpath,
 508         cssfile=cssfile,
 509         tempdir=tempdir,
 510         extensions=extensions,
 511     )
 512
 513     mimetree = MIMETreeDFWalker(debug=debug_walk)
 514
 515     def visitor_fn(item, stack, *, debugprint=None):
 516         """
 517         Visitor function called for every node (part) of the MIME tree,
 518         depth-first, and responsible for telling NeoMutt how to assemble
 519         the tree.
 520         """
 521         KILL_LINE = r"\Ca\Ck"
 522
 523         if isinstance(item, Part):
 524             # We've hit a leaf-node, i.e. an alternative or a related part
 525             # with actual content.
 526
 527             # Let's add the part
 528             if item.orig:
 529                 # The original source already exists in the NeoMutt tree, but
 530                 # the underlying file may have been modified, so we need to
 531                 # update the encoding, but that's it:
 532                 cmds.push("<update-encoding>")
 533             else:
 534                 # … whereas all other parts need to be added, and they're all
 535                 # considered to be temporary and inline:
 536                 cmds.push(f"<attach-file>{item.path}<enter>")
 537                 cmds.push("<toggle-unlink><toggle-disposition>")
 538
 539             # If the item (including the original) comes with additional
 540             # information, then we might just as well update the NeoMutt
 541             # tree now:
 542             if item.cid:
 543                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 544
 545         elif isinstance(item, Multipart):
 546             # This node has children, but we already visited them (see
 547             # above), and so they have been tagged in NeoMutt's compose
 548             # window. Now it's just a matter of telling NeoMutt to do the
 549             # appropriate grouping:
 550             if item.subtype == "alternative":
 551                 cmds.push("<group-alternatives>")
 552             elif item.subtype in ("relative", "related"):
 553                 cmds.push("<group-related>")
 554             elif item.subtype == "multilingual":
 555                 cmds.push("<group-multilingual>")
 556
 557         else:
 558             # We should never get here
 559             assert not "is valid part"
 560
 561         # If the item has a description, we might just as well add it
 562         if item.desc:
 563             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 564
 565         # Finally, if we're at non-root level, tag the new container,
 566         # as it might itself be part of a container, to be processed
 567         # one level up:
 568         if stack:
 569             cmds.push("<tag-entry>")
 570
 571     # -----------------
 572     # End of visitor_fn
 573
 574     # Let's walk the tree and visit every node with our fancy visitor
 575     # function
 576     mimetree.walk(tree, visitor_fn=visitor_fn)
 577
 578     if not only_build:
 579         cmds.push("<send-message>")
 580
 581     # Finally, cleanup. Since we're responsible for removing the temporary
 582     # file, how's this for a little hack?
 583     try:
 584         filename = cmd_f.name
 585     except AttributeError:
 586         filename = "pytest_internal_file"
 587     cmds.cmd(f"source 'rm -f {filename}|'")
 588     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 589     cmds.flush()
 590
 591
 592 # [ CLI ENTRY ] ###############################################################
 593
 594 if __name__ == "__main__":
 595     args = parse_cli_args()
 596
 597     if args.mode is None:
 598         do_setup(
 599             tempdir=args.tempdir,
 600             debug_commands=args.debug_commands,
 601         )
 602
 603     elif args.mode == "massage":
 604         with open(args.MAILDRAFT, "r") as draft_f, open(
 605             args.cmdpath, "w"
 606         ) as cmd_f:
 607             do_massage(
 608                 draft_f,
 609                 pathlib.Path(args.MAILDRAFT),
 610                 cmd_f,
 611                 extensions=args.extensions,
 612                 cssfile=args.css_file,
 613                 only_build=args.only_build,
 614                 tempdir=args.tempdir,
 615                 debug_commands=args.debug_commands,
 616                 debug_walk=args.debug_walk,
 617             )
 618
 619
 620 # [ TESTS ] ###################################################################
 621
 622 try:
 623     import pytest
 624     from io import StringIO
 625
 626     class Tests:
 627         @pytest.fixture
 628         def const1(self):
 629             return "CONSTANT STRING 1"
 630
 631         @pytest.fixture
 632         def const2(self):
 633             return "CONSTANT STRING 2"
 634
 635         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 636         # functions they call, else old stdout is used and not captured
 637
 638         def test_MuttCommands_cmd(self, const1, const2, capsys):
 639             "Assert order of commands"
 640             cmds = MuttCommands(out_f=sys.stdout)
 641             cmds.cmd(const1)
 642             cmds.cmd(const2)
 643             cmds.flush()
 644             captured = capsys.readouterr()
 645             assert captured.out == "\n".join((const1, const2, ""))
 646
 647         def test_MuttCommands_push(self, const1, const2, capsys):
 648             "Assert reverse order of pushes"
 649             cmds = MuttCommands(out_f=sys.stdout)
 650             cmds.push(const1)
 651             cmds.push(const2)
 652             cmds.flush()
 653             captured = capsys.readouterr()
 654             assert (
 655                 captured.out
 656                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 657             )
 658
 659         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 660             "Assert reverse order of pushes"
 661             cmds = MuttCommands(out_f=sys.stdout)
 662             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 663             for i in range(2):
 664                 cmds.cmd(lines[4 * i + 0])
 665                 cmds.cmd(lines[4 * i + 1])
 666                 cmds.push(lines[4 * i + 2])
 667                 cmds.push(lines[4 * i + 3])
 668             cmds.flush()
 669
 670             captured = capsys.readouterr()
 671             lines_out = captured.out.splitlines()
 672             assert lines[0] in lines_out[0]
 673             assert lines[1] in lines_out[1]
 674             assert lines[7] in lines_out[2]
 675             assert lines[6] in lines_out[3]
 676             assert lines[3] in lines_out[4]
 677             assert lines[2] in lines_out[5]
 678             assert lines[4] in lines_out[6]
 679             assert lines[5] in lines_out[7]
 680
 681         @pytest.fixture
 682         def basic_mime_tree(self):
 683             return Multipart(
 684                 "relative",
 685                 children=[
 686                     Multipart(
 687                         "alternative",
 688                         children=[
 689                             Part(
 690                                 "text",
 691                                 "plain",
 692                                 "part.txt",
 693                                 desc="Plain",
 694                                 orig=True,
 695                             ),
 696                             Part("text", "html", "part.html", desc="HTML"),
 697                         ],
 698                         desc="Alternative",
 699                     ),
 700                     Part(
 701                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 702                     ),
 703                 ],
 704                 desc="Related",
 705             )
 706
 707         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 708             mimetree = MIMETreeDFWalker()
 709
 710             items = []
 711
 712             def visitor_fn(item, stack, debugprint):
 713                 items.append((item, len(stack)))
 714
 715             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 716             assert len(items) == 5
 717             assert items[0][0].subtype == "plain"
 718             assert items[0][1] == 2
 719             assert items[1][0].subtype == "html"
 720             assert items[1][1] == 2
 721             assert items[2][0].subtype == "alternative"
 722             assert items[2][1] == 1
 723             assert items[3][0].subtype == "png"
 724             assert items[3][1] == 1
 725             assert items[4][0].subtype == "relative"
 726             assert items[4][1] == 0
 727
 728         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
 729             mimetree = MIMETreeDFWalker()
 730             items = []
 731
 732             def visitor_fn(item, stack, debugprint):
 733                 items.append(item)
 734
 735             p = Part("text", "plain", const1)
 736             mimetree.walk([p], visitor_fn=visitor_fn)
 737             assert items[-1].subtype == "plain"
 738             mimetree.walk([p, p], visitor_fn=visitor_fn)
 739             assert items[-1].subtype == "mixed"
 740
 741         def test_MIMETreeDFWalker_visitor_in_constructor(
 742             self, basic_mime_tree
 743         ):
 744             items = []
 745
 746             def visitor_fn(item, stack, debugprint):
 747                 items.append(item)
 748
 749             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 750             mimetree.walk(basic_mime_tree)
 751             assert len(items) == 5
 752
 753         @pytest.fixture
 754         def string_io(self, const1, text=None):
 755             return StringIO(text or const1)
 756
 757         def test_do_massage_basic(self, const1, string_io, capsys):
 758             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 759                 return Part("text", "plain", draftpath, orig=True)
 760
 761             do_massage(
 762                 draft_f=string_io,
 763                 draftpath=const1,
 764                 cmd_f=sys.stdout,
 765                 converter=converter,
 766             )
 767
 768             captured = capsys.readouterr()
 769             lines = captured.out.splitlines()
 770             assert '="$my_editor"' in lines.pop(0)
 771             assert '="$my_edit_headers"' in lines.pop(0)
 772             assert "unset my_editor" == lines.pop(0)
 773             assert "unset my_edit_headers" == lines.pop(0)
 774             assert "send-message" in lines.pop(0)
 775             assert "update-encoding" in lines.pop(0)
 776             assert "source 'rm -f " in lines.pop(0)
 777             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 778
 779         def test_do_massage_fulltree(
 780             self, string_io, const1, basic_mime_tree, capsys
 781         ):
 782             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 783                 return basic_mime_tree
 784
 785             do_massage(
 786                 draft_f=string_io,
 787                 draftpath=const1,
 788                 cmd_f=sys.stdout,
 789                 converter=converter,
 790             )
 791
 792             captured = capsys.readouterr()
 793             lines = captured.out.splitlines()[4:]
 794             assert "send-message" in lines.pop(0)
 795             assert "Related" in lines.pop(0)
 796             assert "group-related" in lines.pop(0)
 797             assert "tag-entry" in lines.pop(0)
 798             assert "Logo" in lines.pop(0)
 799             assert "content-id" in lines.pop(0)
 800             assert "toggle-unlink" in lines.pop(0)
 801             assert "logo.png" in lines.pop(0)
 802             assert "tag-entry" in lines.pop(0)
 803             assert "Alternative" in lines.pop(0)
 804             assert "group-alternatives" in lines.pop(0)
 805             assert "tag-entry" in lines.pop(0)
 806             assert "HTML" in lines.pop(0)
 807             assert "toggle-unlink" in lines.pop(0)
 808             assert "part.html" in lines.pop(0)
 809             assert "tag-entry" in lines.pop(0)
 810             assert "Plain" in lines.pop(0)
 811             assert "update-encoding" in lines.pop(0)
 812             assert len(lines) == 2
 813
 814         @pytest.fixture
 815         def fake_filewriter(self):
 816             class FileWriter:
 817                 def __init__(self):
 818                     self._writes = []
 819
 820                 def __call__(self, path, content, mode="w", **kwargs):
 821                     self._writes.append((path, content))
 822
 823                 def pop(self, index=-1):
 824                     return self._writes.pop(index)
 825
 826             return FileWriter()
 827
 828         @pytest.fixture
 829         def markdown_non_converter(self, const1, const2):
 830             return lambda s, text: f"{const1}{text}{const2}"
 831
 832         def test_converter_tree_basic(
 833             self, const1, const2, fake_filewriter, markdown_non_converter
 834         ):
 835             path = pathlib.Path(const2)
 836             tree = convert_markdown_to_html(
 837                 const1, path, filewriter_fn=fake_filewriter
 838             )
 839
 840             assert tree.subtype == "alternative"
 841             assert len(tree.children) == 2
 842             assert tree.children[0].subtype == "plain"
 843             assert tree.children[0].path == path
 844             assert tree.children[0].orig
 845             assert tree.children[1].subtype == "html"
 846             assert tree.children[1].path == path.with_suffix(".html")
 847
 848         def test_converter_writes(
 849             self,
 850             const1,
 851             const2,
 852             fake_filewriter,
 853             monkeypatch,
 854             markdown_non_converter,
 855         ):
 856             path = pathlib.Path(const2)
 857
 858             with monkeypatch.context() as m:
 859                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 860                 convert_markdown_to_html(
 861                     const1, path, filewriter_fn=fake_filewriter
 862                 )
 863
 864             assert (path, const1) == fake_filewriter.pop(0)
 865             assert (
 866                 path.with_suffix(".html"),
 867                 markdown_non_converter(None, const1),
 868             ) == fake_filewriter.pop(0)
 869
 870         def test_markdown_inline_image_processor(self):
 871             imgpath1 = "file:/path/to/image.png"
 872             imgpath2 = "file:///path/to/image.png?url=params"
 873             imgpath3 = "/path/to/image.png"
 874             text = f"""![inline local image]({imgpath1})
 875                        ![image inlined
 876                          with newline]({imgpath2})
 877                        ![image local path]({imgpath3})"""
 878             text, html, images = markdown_with_inline_image_support(text)
 879
 880             # local paths have been normalised to URLs:
 881             imgpath3 = f"file://{imgpath3}"
 882
 883             assert 'src="cid:' in html
 884             assert "](cid:" in text
 885             assert len(images) == 3
 886             assert imgpath1 in images
 887             assert imgpath2 in images
 888             assert imgpath3 in images
 889             assert images[imgpath1].cid != images[imgpath2].cid
 890             assert images[imgpath1].cid != images[imgpath3].cid
 891             assert images[imgpath2].cid != images[imgpath3].cid
 892
 893         def test_markdown_inline_image_processor_title_to_desc(self, const1):
 894             imgpath = "file:///path/to/image.png"
 895             text = f'![inline local image]({imgpath} "{const1}")'
 896             text, html, images = markdown_with_inline_image_support(text)
 897             assert images[imgpath].desc == const1
 898
 899         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
 900             imgpath = "file:///path/to/image.png"
 901             text = f"![{const1}]({imgpath})"
 902             text, html, images = markdown_with_inline_image_support(text)
 903             assert images[imgpath].desc == const1
 904
 905         def test_markdown_inline_image_processor_title_over_alt_desc(
 906             self, const1, const2
 907         ):
 908             imgpath = "file:///path/to/image.png"
 909             text = f'![{const1}]({imgpath} "{const2}")'
 910             text, html, images = markdown_with_inline_image_support(text)
 911             assert images[imgpath].desc == const2
 912
 913         def test_markdown_inline_image_not_external(self):
 914             imgpath = "https://path/to/image.png"
 915             text = f"![inline image]({imgpath})"
 916             text, html, images = markdown_with_inline_image_support(text)
 917
 918             assert 'src="cid:' not in html
 919             assert "](cid:" not in text
 920             assert len(images) == 0
 921
 922         def test_markdown_inline_image_local_file(self):
 923             imgpath = "/path/to/image.png"
 924             text = f"![inline image]({imgpath})"
 925             text, html, images = markdown_with_inline_image_support(text)
 926
 927             for k, v in images.items():
 928                 assert k == f"file://{imgpath}"
 929                 break
 930
 931         @pytest.fixture
 932         def test_png(self):
 933             return (
 934                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
 935                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
 936             )
 937
 938         def test_markdown_inline_image_processor_base64(self, test_png):
 939             text = f"![1px white inlined]({test_png})"
 940             text, html, images = markdown_with_inline_image_support(text)
 941
 942             assert 'src="cid:' in html
 943             assert "](cid:" in text
 944             assert len(images) == 1
 945             assert test_png in images
 946
 947         def test_converter_tree_inline_image_base64(
 948             self, test_png, const1, fake_filewriter
 949         ):
 950             text = f"![inline base64 image]({test_png})"
 951             path = pathlib.Path(const1)
 952             tree = convert_markdown_to_html(
 953                 text, path, filewriter_fn=fake_filewriter
 954             )
 955
 956             assert tree.subtype == "relative"
 957             assert tree.children[1].subtype == "png"
 958             written = fake_filewriter.pop()
 959             assert tree.children[1].path == written[0]
 960             assert written[1] == request.urlopen(test_png).read()
 961
 962         def test_converter_tree_inline_image_cid(
 963             self, const1, fake_filewriter
 964         ):
 965             text = f"![inline base64 image](cid:{const1})"
 966             path = pathlib.Path(const1)
 967             tree = convert_markdown_to_html(
 968                 text,
 969                 path,
 970                 filewriter_fn=fake_filewriter,
 971                 related_to_html_only=False,
 972             )
 973             assert len(tree.children) == 2
 974             assert tree.children[0].cid != const1
 975             assert tree.children[0].type != "image"
 976             assert tree.children[1].cid != const1
 977             assert tree.children[1].type != "image"
 978
 979         def test_inline_image_collection(
 980             self, test_png, const1, const2, fake_filewriter
 981         ):
 982             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
 983             relparts = collect_inline_images(
 984                 test_images, filewriter_fn=fake_filewriter
 985             )
 986
 987             written = fake_filewriter.pop()
 988             assert b"PNG" in written[1]
 989
 990             assert relparts[0].subtype == "png"
 991             assert relparts[0].path == written[0]
 992             assert relparts[0].cid == const1
 993             assert relparts[0].desc.endswith(const2)
 994
 995         def test_apply_stylesheet(self):
 996             if _PYNLINER:
 997                 html = "<p>Hello, world!</p>"
 998                 css = "p { color:red }"
 999                 out = apply_styling(html, css)
1000                 assert 'p style="color' in out
1001
1002         def test_apply_stylesheet_pygments(self):
1003             if _PYGMENTS_CSS:
1004                 html = (
1005                     f'<div class="{_CODEHILITE_CLASS}">'
1006                     "<pre>def foo():\n    return</pre></div>"
1007                 )
1008                 out = apply_styling(html, _PYGMENTS_CSS)
1009                 assert f'{_CODEHILITE_CLASS}" style="' in out
1010
1011
1012 except ImportError:
1013     pass