.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--only-build",
  81         action="store_true",
  82         help="Only build, don't send the message",
  83     )
  84
  85     parser.add_argument(
  86         "--tempdir",
  87         default=None,
  88         help="Specify temporary directory to use for attachments",
  89     )
  90
  91     parser.add_argument(
  92         "--debug-commands",
  93         action="store_true",
  94         help="Turn on debug logging of commands generated to stderr",
  95     )
  96
  97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  98     massage_p = subp.add_parser(
  99         "massage", help="Massaging phase (internal use)"
 100     )
 101
 102     massage_p.add_argument(
 103         "--write-commands-to",
 104         metavar="PATH",
 105         dest="cmdpath",
 106         help="Temporary file path to write commands to",
 107     )
 108
 109     massage_p.add_argument(
 110         "--debug-walk",
 111         action="store_true",
 112         help="Turn on debugging to stderr of the MIME tree walk",
 113     )
 114
 115     massage_p.add_argument(
 116         "MAILDRAFT",
 117         nargs="?",
 118         help="If provided, the script is invoked as editor on the mail draft",
 119     )
 120
 121     return parser.parse_args(*args, **kwargs)
 122
 123
 124 # [ MARKDOWN WRAPPING ] #######################################################
 125
 126
 127 InlineImageInfo = namedtuple(
 128     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 129 )
 130
 131
 132 class InlineImageExtension(Extension):
 133     class RelatedImageInlineProcessor(ImageInlineProcessor):
 134         def __init__(self, re, md, ext):
 135             super().__init__(re, md)
 136             self._ext = ext
 137
 138         def handleMatch(self, m, data):
 139             el, start, end = super().handleMatch(m, data)
 140             if "src" in el.attrib:
 141                 src = el.attrib["src"]
 142                 if "://" not in src or src.startswith("file://"):
 143                     # We only inline local content
 144                     cid = self._ext.get_cid_for_image(el.attrib)
 145                     el.attrib["src"] = f"cid:{cid}"
 146             return el, start, end
 147
 148     def __init__(self):
 149         super().__init__()
 150         self._images = OrderedDict()
 151
 152     def extendMarkdown(self, md):
 153         md.registerExtension(self)
 154         inline_image_proc = self.RelatedImageInlineProcessor(
 155             IMAGE_LINK_RE, md, self
 156         )
 157         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 158
 159     def get_cid_for_image(self, attrib):
 160         msgid = make_msgid()[1:-1]
 161         path = attrib["src"]
 162         if path.startswith("/"):
 163             path = f"file://{path}"
 164         self._images[path] = InlineImageInfo(
 165             msgid, attrib.get("title", attrib.get("alt"))
 166         )
 167         return msgid
 168
 169     def get_images(self):
 170         return self._images
 171
 172
 173 def markdown_with_inline_image_support(
 174     text, *, extensions=None, extension_configs=None
 175 ):
 176     inline_image_handler = InlineImageExtension()
 177     extensions = extensions or []
 178     extensions.append(inline_image_handler)
 179     mdwn = markdown.Markdown(
 180         extensions=extensions, extension_configs=extension_configs
 181     )
 182     htmltext = mdwn.convert(text)
 183
 184     images = inline_image_handler.get_images()
 185
 186     def replace_image_with_cid(matchobj):
 187         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 188             if m in images:
 189                 return f"(cid:{images[m].cid}"
 190         return matchobj.group(0)
 191
 192     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 193     return text, htmltext, images
 194
 195
 196 # [ CSS STYLING ] #############################################################
 197
 198 try:
 199     import pynliner
 200
 201     _PYNLINER = True
 202
 203 except ImportError:
 204     _PYNLINER = False
 205
 206 try:
 207     from pygments.formatters import get_formatter_by_name
 208
 209     _CODEHILITE_CLASS = "codehilite"
 210
 211     _PYGMENTS_CSS = get_formatter_by_name(
 212         "html", style="default"
 213     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 214
 215 except ImportError:
 216     _PYGMENTS_CSS = None
 217
 218
 219 def apply_styling(html, css):
 220     return (
 221         pynliner.Pynliner()
 222         .from_string(html)
 223         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 224         .run()
 225     )
 226
 227
 228 # [ PARTS GENERATION ] ########################################################
 229
 230
 231 class Part(
 232     namedtuple(
 233         "Part",
 234         ["type", "subtype", "path", "desc", "cid", "orig"],
 235         defaults=[None, None, False],
 236     )
 237 ):
 238     def __str__(self):
 239         ret = f"<{self.type}/{self.subtype}>"
 240         if self.cid:
 241             ret = f"{ret} cid:{self.cid}"
 242         if self.orig:
 243             ret = f"{ret} ORIGINAL"
 244         return ret
 245
 246
 247 class Multipart(
 248     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 249 ):
 250     def __str__(self):
 251         return f"<multipart/{self.subtype}> children={len(self.children)}"
 252
 253
 254 def filewriter_fn(path, content, mode="w", **kwargs):
 255     with open(path, mode, **kwargs) as out_f:
 256         out_f.write(content)
 257
 258
 259 def collect_inline_images(
 260     images, *, tempdir=None, filewriter_fn=filewriter_fn
 261 ):
 262     relparts = []
 263     for path, info in images.items():
 264         data = request.urlopen(path)
 265
 266         mimetype = data.headers["Content-Type"]
 267         ext = mimetypes.guess_extension(mimetype)
 268         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 269         path = pathlib.Path(tempfilename[1])
 270
 271         filewriter_fn(path, data.read(), "w+b")
 272
 273         relparts.append(
 274             Part(
 275                 *mimetype.split("/"),
 276                 path,
 277                 cid=info.cid,
 278                 desc=f"Image: {info.desc}",
 279             )
 280         )
 281
 282     return relparts
 283
 284
 285 def convert_markdown_to_html(
 286     origtext,
 287     draftpath,
 288     *,
 289     cssfile=None,
 290     filewriter_fn=filewriter_fn,
 291     tempdir=None,
 292     extensions=None,
 293     extension_configs=None,
 294 ):
 295     # TODO extension_configs need to be handled differently
 296     extension_configs = extension_configs or {}
 297     extension_configs.setdefault("pymdownx.highlight", {})
 298     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 299
 300     origtext, htmltext, images = markdown_with_inline_image_support(
 301         origtext, extensions=extensions, extension_configs=extension_configs
 302     )
 303
 304     filewriter_fn(draftpath, origtext, encoding="utf-8")
 305     textpart = Part(
 306         "text", "plain", draftpath, "Plain-text version", orig=True
 307     )
 308
 309     htmltext = apply_styling(htmltext, cssfile)
 310
 311     htmlpath = draftpath.with_suffix(".html")
 312     filewriter_fn(
 313         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 314     )
 315     htmlpart = Part("text", "html", htmlpath, "HTML version")
 316
 317     altpart = Multipart(
 318         "alternative", [textpart, htmlpart], "Group of alternative content"
 319     )
 320
 321     imgparts = collect_inline_images(
 322         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 323     )
 324     if imgparts:
 325         return Multipart(
 326             "relative", [altpart] + imgparts, "Group of related content"
 327         )
 328     else:
 329         return altpart
 330
 331
 332 class MIMETreeDFWalker:
 333     def __init__(self, *, visitor_fn=None, debug=False):
 334         self._visitor_fn = visitor_fn
 335         self._debug = debug
 336
 337     def walk(self, root, *, visitor_fn=None):
 338         """
 339         Recursive function to implement a depth-dirst walk of the MIME-tree
 340         rooted at `root`.
 341         """
 342
 343         if isinstance(root, list):
 344             root = Multipart("mixed", children=root)
 345
 346         self._walk(
 347             root,
 348             stack=[],
 349             visitor_fn=visitor_fn or self._visitor_fn,
 350         )
 351
 352     def _walk(self, node, *, stack, visitor_fn):
 353         # Let's start by enumerating the parts at the current level. At the
 354         # root level, stack will be the empty list, and we expect a multipart/*
 355         # container at this level. Later, e.g. within a mutlipart/alternative
 356         # container, the subtree will just be the alternative parts, while the
 357         # top of the stack will be the multipart/alternative container, which
 358         # we will process after the following loop.
 359
 360         lead = f"{'| '*len(stack)}|-"
 361         if isinstance(node, Multipart):
 362             self.debugprint(
 363                 f"{lead}{node} parents={[s.subtype for s in stack]}"
 364             )
 365
 366             # Depth-first, so push the current container onto the stack,
 367             # then descend …
 368             stack.append(node)
 369             self.debugprint("| " * (len(stack) + 1))
 370             for child in node.children:
 371                 self._walk(
 372                     child,
 373                     stack=stack,
 374                     visitor_fn=visitor_fn,
 375                 )
 376             self.debugprint("| " * len(stack))
 377             assert stack.pop() == node
 378
 379         else:
 380             self.debugprint(f"{lead}{node}")
 381
 382         if visitor_fn:
 383             visitor_fn(node, stack, debugprint=self.debugprint)
 384
 385     def debugprint(self, s, **kwargs):
 386         if self._debug:
 387             print(s, file=sys.stderr, **kwargs)
 388
 389
 390 # [ RUN MODES ] ###############################################################
 391
 392
 393 class MuttCommands:
 394     """
 395     Stupid class to interface writing out Mutt commands. This is quite a hack
 396     to deal with the fact that Mutt runs "push" commands in reverse order, so
 397     all of a sudden, things become very complicated when mixing with "real"
 398     commands.
 399
 400     Hence we keep two sets of commands, and one set of pushes. Commands are
 401     added to the first until a push is added, after which commands are added to
 402     the second set of commands.
 403
 404     On flush(), the first set is printed, followed by the pushes in reverse,
 405     and then the second set is printed. All 3 sets are then cleared.
 406     """
 407
 408     def __init__(self, out_f=sys.stdout, *, debug=False):
 409         self._cmd1, self._push, self._cmd2 = [], [], []
 410         self._out_f = out_f
 411         self._debug = debug
 412
 413     def cmd(self, s):
 414         self.debugprint(s)
 415         if self._push:
 416             self._cmd2.append(s)
 417         else:
 418             self._cmd1.append(s)
 419
 420     def push(self, s):
 421         s = s.replace('"', '"')
 422         s = f'push "{s}"'
 423         self.debugprint(s)
 424         self._push.insert(0, s)
 425
 426     def flush(self):
 427         print(
 428             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 429         )
 430         self._cmd1, self._push, self._cmd2 = [], [], []
 431
 432     def debugprint(self, s, **kwargs):
 433         if self._debug:
 434             print(s, file=sys.stderr, **kwargs)
 435
 436
 437 def do_setup(
 438     *,
 439     out_f=sys.stdout,
 440     temppath=None,
 441     tempdir=None,
 442     debug_commands=False,
 443 ):
 444     temppath = temppath or pathlib.Path(
 445         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 446     )
 447     cmds = MuttCommands(out_f, debug=debug_commands)
 448
 449     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 450
 451     cmds.cmd('set my_editor="$editor"')
 452     cmds.cmd('set my_edit_headers="$edit_headers"')
 453     cmds.cmd(f'set editor="{editor}"')
 454     cmds.cmd("unset edit_headers")
 455     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 456     cmds.push("<first-entry><edit-file>")
 457     cmds.flush()
 458
 459
 460 def do_massage(
 461     draft_f,
 462     draftpath,
 463     cmd_f,
 464     *,
 465     extensions=None,
 466     cssfile=None,
 467     converter=convert_markdown_to_html,
 468     only_build=False,
 469     tempdir=None,
 470     debug_commands=False,
 471     debug_walk=False,
 472 ):
 473     # Here's the big picture: we're being invoked as the editor on the email
 474     # draft, and whatever commands we write to the file given as cmdpath will
 475     # be run by the second source command in the macro definition.
 476
 477     # Let's start by cleaning up what the setup did (see above), i.e. we
 478     # restore the $editor and $edit_headers variables, and also unset the
 479     # variable used to identify the command file we're currently writing
 480     # to.
 481     cmds = MuttCommands(cmd_f, debug=debug_commands)
 482     cmds.cmd('set editor="$my_editor"')
 483     cmds.cmd('set edit_headers="$my_edit_headers"')
 484     cmds.cmd("unset my_editor")
 485     cmds.cmd("unset my_edit_headers")
 486
 487     # let's flush those commands, as there'll be a lot of pushes from now
 488     # on, which need to be run in reverse order
 489     cmds.flush()
 490
 491     extensions = extensions.split(",") if extensions else []
 492     tree = converter(
 493         draft_f.read(),
 494         draftpath,
 495         cssfile=cssfile,
 496         tempdir=tempdir,
 497         extensions=extensions,
 498     )
 499
 500     mimetree = MIMETreeDFWalker(debug=debug_walk)
 501
 502     def visitor_fn(item, stack, *, debugprint=None):
 503         """
 504         Visitor function called for every node (part) of the MIME tree,
 505         depth-first, and responsible for telling NeoMutt how to assemble
 506         the tree.
 507         """
 508         KILL_LINE = r"\Ca\Ck"
 509
 510         if isinstance(item, Part):
 511             # We've hit a leaf-node, i.e. an alternative or a related part
 512             # with actual content.
 513
 514             # Let's add the part
 515             if item.orig:
 516                 # The original source already exists in the NeoMutt tree, but
 517                 # the underlying file may have been modified, so we need to
 518                 # update the encoding, but that's it:
 519                 cmds.push("<update-encoding>")
 520             else:
 521                 # … whereas all other parts need to be added, and they're all
 522                 # considered to be temporary and inline:
 523                 cmds.push(f"<attach-file>{item.path}<enter>")
 524                 cmds.push("<toggle-unlink><toggle-disposition>")
 525
 526             # If the item (including the original) comes with additional
 527             # information, then we might just as well update the NeoMutt
 528             # tree now:
 529             if item.cid:
 530                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 531
 532         elif isinstance(item, Multipart):
 533             # This node has children, but we already visited them (see
 534             # above), and so they have been tagged in NeoMutt's compose
 535             # window. Now it's just a matter of telling NeoMutt to do the
 536             # appropriate grouping:
 537             if item.subtype == "alternative":
 538                 cmds.push("<group-alternatives>")
 539             elif item.subtype in ("relative", "related"):
 540                 cmds.push("<group-related>")
 541             elif item.subtype == "multilingual":
 542                 cmds.push("<group-multilingual>")
 543
 544         else:
 545             # We should never get here
 546             assert not "is valid part"
 547
 548         # If the item has a description, we might just as well add it
 549         if item.desc:
 550             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 551
 552         # Finally, if we're at non-root level, tag the new container,
 553         # as it might itself be part of a container, to be processed
 554         # one level up:
 555         if stack:
 556             cmds.push("<tag-entry>")
 557
 558     # -----------------
 559     # End of visitor_fn
 560
 561     # Let's walk the tree and visit every node with our fancy visitor
 562     # function
 563     mimetree.walk(tree, visitor_fn=visitor_fn)
 564
 565     if not only_build:
 566         cmds.push("<send-message>")
 567
 568     # Finally, cleanup. Since we're responsible for removing the temporary
 569     # file, how's this for a little hack?
 570     try:
 571         filename = cmd_f.name
 572     except AttributeError:
 573         filename = "pytest_internal_file"
 574     cmds.cmd(f"source 'rm -f {filename}|'")
 575     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 576     cmds.flush()
 577
 578
 579 # [ CLI ENTRY ] ###############################################################
 580
 581 if __name__ == "__main__":
 582     args = parse_cli_args()
 583
 584     if args.mode is None:
 585         do_setup(
 586             tempdir=args.tempdir,
 587             debug_commands=args.debug_commands,
 588         )
 589
 590     elif args.mode == "massage":
 591         with open(args.MAILDRAFT, "r") as draft_f, open(
 592             args.cmdpath, "w"
 593         ) as cmd_f:
 594             do_massage(
 595                 draft_f,
 596                 pathlib.Path(args.MAILDRAFT),
 597                 cmd_f,
 598                 extensions=args.extensions,
 599                 cssfile=args.css_file,
 600                 only_build=args.only_build,
 601                 tempdir=args.tempdir,
 602                 debug_commands=args.debug_commands,
 603                 debug_walk=args.debug_walk,
 604             )
 605
 606
 607 # [ TESTS ] ###################################################################
 608
 609 try:
 610     import pytest
 611     from io import StringIO
 612
 613     class Tests:
 614         @pytest.fixture
 615         def const1(self):
 616             return "CONSTANT STRING 1"
 617
 618         @pytest.fixture
 619         def const2(self):
 620             return "CONSTANT STRING 2"
 621
 622         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 623         # functions they call, else old stdout is used and not captured
 624
 625         def test_MuttCommands_cmd(self, const1, const2, capsys):
 626             "Assert order of commands"
 627             cmds = MuttCommands(out_f=sys.stdout)
 628             cmds.cmd(const1)
 629             cmds.cmd(const2)
 630             cmds.flush()
 631             captured = capsys.readouterr()
 632             assert captured.out == "\n".join((const1, const2, ""))
 633
 634         def test_MuttCommands_push(self, const1, const2, capsys):
 635             "Assert reverse order of pushes"
 636             cmds = MuttCommands(out_f=sys.stdout)
 637             cmds.push(const1)
 638             cmds.push(const2)
 639             cmds.flush()
 640             captured = capsys.readouterr()
 641             assert (
 642                 captured.out
 643                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 644             )
 645
 646         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 647             "Assert reverse order of pushes"
 648             cmds = MuttCommands(out_f=sys.stdout)
 649             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 650             for i in range(2):
 651                 cmds.cmd(lines[4 * i + 0])
 652                 cmds.cmd(lines[4 * i + 1])
 653                 cmds.push(lines[4 * i + 2])
 654                 cmds.push(lines[4 * i + 3])
 655             cmds.flush()
 656
 657             captured = capsys.readouterr()
 658             lines_out = captured.out.splitlines()
 659             assert lines[0] in lines_out[0]
 660             assert lines[1] in lines_out[1]
 661             assert lines[7] in lines_out[2]
 662             assert lines[6] in lines_out[3]
 663             assert lines[3] in lines_out[4]
 664             assert lines[2] in lines_out[5]
 665             assert lines[4] in lines_out[6]
 666             assert lines[5] in lines_out[7]
 667
 668         @pytest.fixture
 669         def basic_mime_tree(self):
 670             return Multipart(
 671                 "relative",
 672                 children=[
 673                     Multipart(
 674                         "alternative",
 675                         children=[
 676                             Part(
 677                                 "text",
 678                                 "plain",
 679                                 "part.txt",
 680                                 desc="Plain",
 681                                 orig=True,
 682                             ),
 683                             Part("text", "html", "part.html", desc="HTML"),
 684                         ],
 685                         desc="Alternative",
 686                     ),
 687                     Part(
 688                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 689                     ),
 690                 ],
 691                 desc="Related",
 692             )
 693
 694         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 695             mimetree = MIMETreeDFWalker()
 696
 697             items = []
 698
 699             def visitor_fn(item, stack, debugprint):
 700                 items.append((item, len(stack)))
 701
 702             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 703             assert len(items) == 5
 704             assert items[0][0].subtype == "plain"
 705             assert items[0][1] == 2
 706             assert items[1][0].subtype == "html"
 707             assert items[1][1] == 2
 708             assert items[2][0].subtype == "alternative"
 709             assert items[2][1] == 1
 710             assert items[3][0].subtype == "png"
 711             assert items[3][1] == 1
 712             assert items[4][0].subtype == "relative"
 713             assert items[4][1] == 0
 714
 715         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
 716             mimetree = MIMETreeDFWalker()
 717             items = []
 718
 719             def visitor_fn(item, stack, debugprint):
 720                 items.append(item)
 721
 722             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
 723             assert items[-1].subtype == "mixed"
 724
 725         def test_MIMETreeDFWalker_visitor_in_constructor(
 726             self, basic_mime_tree
 727         ):
 728             items = []
 729
 730             def visitor_fn(item, stack, debugprint):
 731                 items.append(item)
 732
 733             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 734             mimetree.walk(basic_mime_tree)
 735             assert len(items) == 5
 736
 737         @pytest.fixture
 738         def string_io(self, const1, text=None):
 739             return StringIO(text or const1)
 740
 741         def test_do_massage_basic(self, const1, string_io, capsys):
 742             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 743                 return Part("text", "plain", draftpath, orig=True)
 744
 745             do_massage(
 746                 draft_f=string_io,
 747                 draftpath=const1,
 748                 cmd_f=sys.stdout,
 749                 converter=converter,
 750             )
 751
 752             captured = capsys.readouterr()
 753             lines = captured.out.splitlines()
 754             assert '="$my_editor"' in lines.pop(0)
 755             assert '="$my_edit_headers"' in lines.pop(0)
 756             assert "unset my_editor" == lines.pop(0)
 757             assert "unset my_edit_headers" == lines.pop(0)
 758             assert "send-message" in lines.pop(0)
 759             assert "update-encoding" in lines.pop(0)
 760             assert "source 'rm -f " in lines.pop(0)
 761             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 762
 763         def test_do_massage_fulltree(
 764             self, string_io, const1, basic_mime_tree, capsys
 765         ):
 766             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 767                 return basic_mime_tree
 768
 769             do_massage(
 770                 draft_f=string_io,
 771                 draftpath=const1,
 772                 cmd_f=sys.stdout,
 773                 converter=converter,
 774             )
 775
 776             captured = capsys.readouterr()
 777             lines = captured.out.splitlines()[4:]
 778             assert "send-message" in lines.pop(0)
 779             assert "Related" in lines.pop(0)
 780             assert "group-related" in lines.pop(0)
 781             assert "tag-entry" in lines.pop(0)
 782             assert "Logo" in lines.pop(0)
 783             assert "content-id" in lines.pop(0)
 784             assert "toggle-unlink" in lines.pop(0)
 785             assert "logo.png" in lines.pop(0)
 786             assert "tag-entry" in lines.pop(0)
 787             assert "Alternative" in lines.pop(0)
 788             assert "group-alternatives" in lines.pop(0)
 789             assert "tag-entry" in lines.pop(0)
 790             assert "HTML" in lines.pop(0)
 791             assert "toggle-unlink" in lines.pop(0)
 792             assert "part.html" in lines.pop(0)
 793             assert "tag-entry" in lines.pop(0)
 794             assert "Plain" in lines.pop(0)
 795             assert "update-encoding" in lines.pop(0)
 796             assert len(lines) == 2
 797
 798         @pytest.fixture
 799         def fake_filewriter(self):
 800             class FileWriter:
 801                 def __init__(self):
 802                     self._writes = []
 803
 804                 def __call__(self, path, content, mode="w", **kwargs):
 805                     self._writes.append((path, content))
 806
 807                 def pop(self, index=-1):
 808                     return self._writes.pop(index)
 809
 810             return FileWriter()
 811
 812         @pytest.fixture
 813         def markdown_non_converter(self, const1, const2):
 814             return lambda s, text: f"{const1}{text}{const2}"
 815
 816         def test_converter_tree_basic(
 817             self, const1, const2, fake_filewriter, markdown_non_converter
 818         ):
 819             path = pathlib.Path(const2)
 820             tree = convert_markdown_to_html(
 821                 const1, path, filewriter_fn=fake_filewriter
 822             )
 823
 824             assert tree.subtype == "alternative"
 825             assert len(tree.children) == 2
 826             assert tree.children[0].subtype == "plain"
 827             assert tree.children[0].path == path
 828             assert tree.children[0].orig
 829             assert tree.children[1].subtype == "html"
 830             assert tree.children[1].path == path.with_suffix(".html")
 831
 832         def test_converter_writes(
 833             self,
 834             const1,
 835             const2,
 836             fake_filewriter,
 837             monkeypatch,
 838             markdown_non_converter,
 839         ):
 840             path = pathlib.Path(const2)
 841
 842             with monkeypatch.context() as m:
 843                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 844                 convert_markdown_to_html(
 845                     const1, path, filewriter_fn=fake_filewriter
 846                 )
 847
 848             assert (path, const1) == fake_filewriter.pop(0)
 849             assert (
 850                 path.with_suffix(".html"),
 851                 markdown_non_converter(None, const1),
 852             ) == fake_filewriter.pop(0)
 853
 854         def test_markdown_inline_image_processor(self):
 855             imgpath1 = "file:/path/to/image.png"
 856             imgpath2 = "file:///path/to/image.png?url=params"
 857             imgpath3 = "/path/to/image.png"
 858             text = f"""![inline local image]({imgpath1})
 859                        ![image inlined
 860                          with newline]({imgpath2})
 861                        ![image local path]({imgpath3})"""
 862             text, html, images = markdown_with_inline_image_support(text)
 863
 864             # local paths have been normalised to URLs:
 865             imgpath3 = f"file://{imgpath3}"
 866
 867             assert 'src="cid:' in html
 868             assert "](cid:" in text
 869             assert len(images) == 3
 870             assert imgpath1 in images
 871             assert imgpath2 in images
 872             assert imgpath3 in images
 873             assert images[imgpath1].cid != images[imgpath2].cid
 874             assert images[imgpath1].cid != images[imgpath3].cid
 875             assert images[imgpath2].cid != images[imgpath3].cid
 876
 877         def test_markdown_inline_image_processor_title_to_desc(self, const1):
 878             imgpath = "file:///path/to/image.png"
 879             text = f'![inline local image]({imgpath} "{const1}")'
 880             text, html, images = markdown_with_inline_image_support(text)
 881             assert images[imgpath].desc == const1
 882
 883         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
 884             imgpath = "file:///path/to/image.png"
 885             text = f"![{const1}]({imgpath})"
 886             text, html, images = markdown_with_inline_image_support(text)
 887             assert images[imgpath].desc == const1
 888
 889         def test_markdown_inline_image_processor_title_over_alt_desc(
 890             self, const1, const2
 891         ):
 892             imgpath = "file:///path/to/image.png"
 893             text = f'![{const1}]({imgpath} "{const2}")'
 894             text, html, images = markdown_with_inline_image_support(text)
 895             assert images[imgpath].desc == const2
 896
 897         def test_markdown_inline_image_not_external(self):
 898             imgpath = "https://path/to/image.png"
 899             text = f"![inline image]({imgpath})"
 900             text, html, images = markdown_with_inline_image_support(text)
 901
 902             assert 'src="cid:' not in html
 903             assert "](cid:" not in text
 904             assert len(images) == 0
 905
 906         def test_markdown_inline_image_local_file(self):
 907             imgpath = "/path/to/image.png"
 908             text = f"![inline image]({imgpath})"
 909             text, html, images = markdown_with_inline_image_support(text)
 910
 911             for k, v in images.items():
 912                 assert k == f"file://{imgpath}"
 913                 break
 914
 915         @pytest.fixture
 916         def test_png(self):
 917             return (
 918                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
 919                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
 920             )
 921
 922         def test_markdown_inline_image_processor_base64(self, test_png):
 923             text = f"![1px white inlined]({test_png})"
 924             text, html, images = markdown_with_inline_image_support(text)
 925
 926             assert 'src="cid:' in html
 927             assert "](cid:" in text
 928             assert len(images) == 1
 929             assert test_png in images
 930
 931         def test_converter_tree_inline_image_base64(
 932             self, test_png, const1, fake_filewriter
 933         ):
 934             text = f"![inline base64 image]({test_png})"
 935             path = pathlib.Path(const1)
 936             tree = convert_markdown_to_html(
 937                 text, path, filewriter_fn=fake_filewriter
 938             )
 939
 940             assert tree.subtype == "relative"
 941             assert tree.children[1].subtype == "png"
 942             written = fake_filewriter.pop()
 943             assert tree.children[1].path == written[0]
 944             assert written[1] == request.urlopen(test_png).read()
 945
 946         def test_inline_image_collection(
 947             self, test_png, const1, const2, fake_filewriter
 948         ):
 949             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
 950             relparts = collect_inline_images(
 951                 test_images, filewriter_fn=fake_filewriter
 952             )
 953
 954             written = fake_filewriter.pop()
 955             assert b"PNG" in written[1]
 956
 957             assert relparts[0].subtype == "png"
 958             assert relparts[0].path == written[0]
 959             assert relparts[0].cid == const1
 960             assert relparts[0].desc.endswith(const2)
 961
 962         def test_apply_stylesheet(self):
 963             if _PYNLINER:
 964                 html = "<p>Hello, world!</p>"
 965                 css = "p { color:red }"
 966                 out = apply_styling(html, css)
 967                 assert 'p style="color' in out
 968
 969         def test_apply_stylesheet_pygments(self):
 970             if _PYGMENTS_CSS:
 971                 html = (
 972                     f'<div class="{_CODEHILITE_CLASS}">'
 973                     "<pre>def foo():\n    return</pre></div>"
 974                 )
 975                 out = apply_styling(html, _PYGMENTS_CSS)
 976                 assert f'{_CODEHILITE_CLASS}" style="' in out
 977
 978
 979 except ImportError:
 980     pass