.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--only-build",
  81         action="store_true",
  82         help="Only build, don't send the message",
  83     )
  84
  85     parser.add_argument(
  86         "--tempdir",
  87         default=None,
  88         help="Specify temporary directory to use for attachments",
  89     )
  90
  91     parser.add_argument(
  92         "--debug-commands",
  93         action="store_true",
  94         help="Turn on debug logging of commands generated to stderr",
  95     )
  96
  97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  98     massage_p = subp.add_parser(
  99         "massage", help="Massaging phase (internal use)"
 100     )
 101
 102     massage_p.add_argument(
 103         "--write-commands-to",
 104         "-o",
 105         metavar="PATH",
 106         dest="cmdpath",
 107         required=True,
 108         help="Temporary file path to write commands to",
 109     )
 110
 111     massage_p.add_argument(
 112         "--debug-walk",
 113         action="store_true",
 114         help="Turn on debugging to stderr of the MIME tree walk",
 115     )
 116
 117     massage_p.add_argument(
 118         "MAILDRAFT",
 119         nargs="?",
 120         help="If provided, the script is invoked as editor on the mail draft",
 121     )
 122
 123     return parser.parse_args(*args, **kwargs)
 124
 125
 126 # [ MARKDOWN WRAPPING ] #######################################################
 127
 128
 129 InlineImageInfo = namedtuple(
 130     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 131 )
 132
 133
 134 class InlineImageExtension(Extension):
 135     class RelatedImageInlineProcessor(ImageInlineProcessor):
 136         def __init__(self, re, md, ext):
 137             super().__init__(re, md)
 138             self._ext = ext
 139
 140         def handleMatch(self, m, data):
 141             el, start, end = super().handleMatch(m, data)
 142             if "src" in el.attrib:
 143                 src = el.attrib["src"]
 144                 if "://" not in src or src.startswith("file://"):
 145                     # We only inline local content
 146                     cid = self._ext.get_cid_for_image(el.attrib)
 147                     el.attrib["src"] = f"cid:{cid}"
 148             return el, start, end
 149
 150     def __init__(self):
 151         super().__init__()
 152         self._images = OrderedDict()
 153
 154     def extendMarkdown(self, md):
 155         md.registerExtension(self)
 156         inline_image_proc = self.RelatedImageInlineProcessor(
 157             IMAGE_LINK_RE, md, self
 158         )
 159         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 160
 161     def get_cid_for_image(self, attrib):
 162         msgid = make_msgid()[1:-1]
 163         path = attrib["src"]
 164         if path.startswith("/"):
 165             path = f"file://{path}"
 166         self._images[path] = InlineImageInfo(
 167             msgid, attrib.get("title", attrib.get("alt"))
 168         )
 169         return msgid
 170
 171     def get_images(self):
 172         return self._images
 173
 174
 175 def markdown_with_inline_image_support(
 176     text, *, extensions=None, extension_configs=None
 177 ):
 178     inline_image_handler = InlineImageExtension()
 179     extensions = extensions or []
 180     extensions.append(inline_image_handler)
 181     mdwn = markdown.Markdown(
 182         extensions=extensions, extension_configs=extension_configs
 183     )
 184     htmltext = mdwn.convert(text)
 185
 186     images = inline_image_handler.get_images()
 187
 188     def replace_image_with_cid(matchobj):
 189         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 190             if m in images:
 191                 return f"(cid:{images[m].cid}"
 192         return matchobj.group(0)
 193
 194     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 195     return text, htmltext, images
 196
 197
 198 # [ CSS STYLING ] #############################################################
 199
 200 try:
 201     import pynliner
 202
 203     _PYNLINER = True
 204
 205 except ImportError:
 206     _PYNLINER = False
 207
 208 try:
 209     from pygments.formatters import get_formatter_by_name
 210
 211     _CODEHILITE_CLASS = "codehilite"
 212
 213     _PYGMENTS_CSS = get_formatter_by_name(
 214         "html", style="default"
 215     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 216
 217 except ImportError:
 218     _PYGMENTS_CSS = None
 219
 220
 221 def apply_styling(html, css):
 222     return (
 223         pynliner.Pynliner()
 224         .from_string(html)
 225         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 226         .run()
 227     )
 228
 229
 230 # [ PARTS GENERATION ] ########################################################
 231
 232
 233 class Part(
 234     namedtuple(
 235         "Part",
 236         ["type", "subtype", "path", "desc", "cid", "orig"],
 237         defaults=[None, None, False],
 238     )
 239 ):
 240     def __str__(self):
 241         ret = f"<{self.type}/{self.subtype}>"
 242         if self.cid:
 243             ret = f"{ret} cid:{self.cid}"
 244         if self.orig:
 245             ret = f"{ret} ORIGINAL"
 246         return ret
 247
 248
 249 class Multipart(
 250     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 251 ):
 252     def __str__(self):
 253         return f"<multipart/{self.subtype}> children={len(self.children)}"
 254
 255     def __hash__(self):
 256         return hash(str(self.subtype) + "".join(str(self.children)))
 257
 258
 259 def filewriter_fn(path, content, mode="w", **kwargs):
 260     with open(path, mode, **kwargs) as out_f:
 261         out_f.write(content)
 262
 263
 264 def collect_inline_images(
 265     images, *, tempdir=None, filewriter_fn=filewriter_fn
 266 ):
 267     relparts = []
 268     for path, info in images.items():
 269         if path.startswith("cid:"):
 270             continue
 271
 272         data = request.urlopen(path)
 273
 274         mimetype = data.headers["Content-Type"]
 275         ext = mimetypes.guess_extension(mimetype)
 276         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 277         path = pathlib.Path(tempfilename[1])
 278
 279         filewriter_fn(path, data.read(), "w+b")
 280
 281         relparts.append(
 282             Part(
 283                 *mimetype.split("/"),
 284                 path,
 285                 cid=info.cid,
 286                 desc=f"Image: {info.desc}",
 287             )
 288         )
 289
 290     return relparts
 291
 292
 293 def convert_markdown_to_html(
 294     origtext,
 295     draftpath,
 296     *,
 297     cssfile=None,
 298     related_to_html_only=True,
 299     filewriter_fn=filewriter_fn,
 300     tempdir=None,
 301     extensions=None,
 302     extension_configs=None,
 303 ):
 304     # TODO extension_configs need to be handled differently
 305     extension_configs = extension_configs or {}
 306     extension_configs.setdefault("pymdownx.highlight", {})
 307     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 308
 309     origtext, htmltext, images = markdown_with_inline_image_support(
 310         origtext, extensions=extensions, extension_configs=extension_configs
 311     )
 312
 313     filewriter_fn(draftpath, origtext, encoding="utf-8")
 314     textpart = Part(
 315         "text", "plain", draftpath, "Plain-text version", orig=True
 316     )
 317
 318     htmltext = apply_styling(htmltext, cssfile)
 319
 320     htmlpath = draftpath.with_suffix(".html")
 321     filewriter_fn(
 322         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 323     )
 324     htmlpart = Part("text", "html", htmlpath, "HTML version")
 325
 326     imgparts = collect_inline_images(
 327         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 328     )
 329
 330     if related_to_html_only:
 331         # If there are inline image part, they will be contained within a
 332         # multipart/related part along with the HTML part only
 333         if imgparts:
 334             # replace htmlpart with a multipart/related container of the HTML
 335             # parts and the images
 336             htmlpart = Multipart(
 337                 "relative", [htmlpart] + imgparts, "Group of related content"
 338             )
 339
 340         return Multipart(
 341             "alternative", [textpart, htmlpart], "Group of alternative content"
 342         )
 343
 344     else:
 345         # If there are inline image part, they will be siblings to the
 346         # multipart/alternative tree within a multipart/related part
 347         altpart = Multipart(
 348             "alternative", [textpart, htmlpart], "Group of alternative content"
 349         )
 350         if imgparts:
 351             return Multipart(
 352                 "relative", [altpart] + imgparts, "Group of related content"
 353             )
 354         else:
 355             return altpart
 356
 357
 358 class MIMETreeDFWalker:
 359     def __init__(self, *, visitor_fn=None, debug=False):
 360         self._visitor_fn = visitor_fn or self._echovisit
 361         self._debug = debug
 362
 363     def _echovisit(self, node, ancestry, debugprint):
 364         debugprint(f"node={node} ancestry={ancestry}")
 365
 366     def walk(self, root, *, visitor_fn=None):
 367         """
 368         Recursive function to implement a depth-dirst walk of the MIME-tree
 369         rooted at `root`.
 370         """
 371         if isinstance(root, list):
 372             if len(root) > 1:
 373                 root = Multipart("mixed", children=root)
 374             else:
 375                 root = root[0]
 376
 377         self._walk(
 378             root,
 379             ancestry=[],
 380             visitor_fn=visitor_fn or self._visitor_fn,
 381         )
 382
 383     def _walk(self, node, *, ancestry, visitor_fn):
 384         # Let's start by enumerating the parts at the current level. At the
 385         # root level, ancestry will be the empty list, and we expect a
 386         # multipart/* container at this level. Later, e.g. within a
 387         # mutlipart/alternative container, the subtree will just be the
 388         # alternative parts, while the top of the ancestry will be the
 389         # multipart/alternative container, which we will process after the
 390         # following loop.
 391
 392         lead = f"{'│ '*len(ancestry)}"
 393         if isinstance(node, Multipart):
 394             self.debugprint(
 395                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
 396             )
 397
 398             # Depth-first, so push the current container onto the ancestry
 399             # stack, then descend …
 400             ancestry.append(node)
 401             self.debugprint(lead + "│ " * 2)
 402             for child in node.children:
 403                 self._walk(
 404                     child,
 405                     ancestry=ancestry,
 406                     visitor_fn=visitor_fn,
 407                 )
 408             assert ancestry.pop() == node
 409
 410         else:
 411             self.debugprint(f"{lead}├{node}")
 412
 413         if False and ancestry:
 414             self.debugprint(lead[:-1] + " │")
 415
 416         if visitor_fn:
 417             visitor_fn(node, ancestry, debugprint=self.debugprint)
 418
 419     def debugprint(self, s, **kwargs):
 420         if self._debug:
 421             print(s, file=sys.stderr, **kwargs)
 422
 423
 424 # [ RUN MODES ] ###############################################################
 425
 426
 427 class MuttCommands:
 428     """
 429     Stupid class to interface writing out Mutt commands. This is quite a hack
 430     to deal with the fact that Mutt runs "push" commands in reverse order, so
 431     all of a sudden, things become very complicated when mixing with "real"
 432     commands.
 433
 434     Hence we keep two sets of commands, and one set of pushes. Commands are
 435     added to the first until a push is added, after which commands are added to
 436     the second set of commands.
 437
 438     On flush(), the first set is printed, followed by the pushes in reverse,
 439     and then the second set is printed. All 3 sets are then cleared.
 440     """
 441
 442     def __init__(self, out_f=sys.stdout, *, debug=False):
 443         self._cmd1, self._push, self._cmd2 = [], [], []
 444         self._out_f = out_f
 445         self._debug = debug
 446
 447     def cmd(self, s):
 448         self.debugprint(s)
 449         if self._push:
 450             self._cmd2.append(s)
 451         else:
 452             self._cmd1.append(s)
 453
 454     def push(self, s):
 455         s = s.replace('"', '"')
 456         s = f'push "{s}"'
 457         self.debugprint(s)
 458         self._push.insert(0, s)
 459
 460     def flush(self):
 461         print(
 462             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 463         )
 464         self._cmd1, self._push, self._cmd2 = [], [], []
 465
 466     def debugprint(self, s, **kwargs):
 467         if self._debug:
 468             print(s, file=sys.stderr, **kwargs)
 469
 470
 471 def do_setup(
 472     *,
 473     out_f=sys.stdout,
 474     temppath=None,
 475     tempdir=None,
 476     debug_commands=False,
 477 ):
 478     temppath = temppath or pathlib.Path(
 479         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 480     )
 481     cmds = MuttCommands(out_f, debug=debug_commands)
 482
 483     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 484
 485     cmds.cmd('set my_editor="$editor"')
 486     cmds.cmd('set my_edit_headers="$edit_headers"')
 487     cmds.cmd(f'set editor="{editor}"')
 488     cmds.cmd("unset edit_headers")
 489     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 490     cmds.push("<first-entry><edit-file>")
 491     cmds.flush()
 492
 493
 494 def do_massage(
 495     draft_f,
 496     draftpath,
 497     cmd_f,
 498     *,
 499     extensions=None,
 500     cssfile=None,
 501     converter=convert_markdown_to_html,
 502     only_build=False,
 503     tempdir=None,
 504     debug_commands=False,
 505     debug_walk=False,
 506 ):
 507     # Here's the big picture: we're being invoked as the editor on the email
 508     # draft, and whatever commands we write to the file given as cmdpath will
 509     # be run by the second source command in the macro definition.
 510
 511     # Let's start by cleaning up what the setup did (see above), i.e. we
 512     # restore the $editor and $edit_headers variables, and also unset the
 513     # variable used to identify the command file we're currently writing
 514     # to.
 515     cmds = MuttCommands(cmd_f, debug=debug_commands)
 516     cmds.cmd('set editor="$my_editor"')
 517     cmds.cmd('set edit_headers="$my_edit_headers"')
 518     cmds.cmd("unset my_editor")
 519     cmds.cmd("unset my_edit_headers")
 520
 521     # let's flush those commands, as there'll be a lot of pushes from now
 522     # on, which need to be run in reverse order
 523     cmds.flush()
 524
 525     extensions = extensions.split(",") if extensions else []
 526     tree = converter(
 527         draft_f.read(),
 528         draftpath,
 529         cssfile=cssfile,
 530         tempdir=tempdir,
 531         extensions=extensions,
 532     )
 533
 534     mimetree = MIMETreeDFWalker(debug=debug_walk)
 535
 536     state = dict(pos=1, tags={}, parts=1)
 537
 538     def visitor_fn(item, ancestry, *, debugprint=None):
 539         """
 540         Visitor function called for every node (part) of the MIME tree,
 541         depth-first, and responsible for telling NeoMutt how to assemble
 542         the tree.
 543         """
 544         KILL_LINE = r"\Ca\Ck"
 545
 546         if isinstance(item, Part):
 547             # We've hit a leaf-node, i.e. an alternative or a related part
 548             # with actual content.
 549
 550             # Let's add the part
 551             if item.orig:
 552                 # The original source already exists in the NeoMutt tree, but
 553                 # the underlying file may have been modified, so we need to
 554                 # update the encoding, but that's it:
 555                 cmds.push("<first-entry>")
 556                 cmds.push("<update-encoding>")
 557
 558                 # We really just need to be able to assume that at this point,
 559                 # NeoMutt is at position 1, and that we've processed only this
 560                 # part so far. Nevermind about actual attachments, we can
 561                 # safely ignore those as they stay at the end.
 562                 assert state["pos"] == 1
 563                 assert state["parts"] == 1
 564             else:
 565                 # … whereas all other parts need to be added, and they're all
 566                 # considered to be temporary and inline:
 567                 cmds.push(f"<attach-file>{item.path}<enter>")
 568                 cmds.push("<toggle-unlink><toggle-disposition>")
 569
 570                 # This added a part at the end of the list of parts, and that's
 571                 # just how many parts we've seen so far, so it's position in
 572                 # the NeoMutt compose list is the count of parts
 573                 state["parts"] += 1
 574                 state["pos"] = state["parts"]
 575
 576             # If the item (including the original) comes with additional
 577             # information, then we might just as well update the NeoMutt
 578             # tree now:
 579             if item.cid:
 580                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 581
 582         elif isinstance(item, Multipart):
 583             # This node has children, but we already visited them (see
 584             # above). The tags dictionary of State should contain a list of
 585             # their positions in the NeoMutt compose window, so iterate those
 586             # and tag the parts there:
 587             for tag in state["tags"][item]:
 588                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
 589
 590             if item.subtype == "alternative":
 591                 cmds.push("<group-alternatives>")
 592             elif item.subtype in ("relative", "related"):
 593                 cmds.push("<group-related>")
 594             elif item.subtype == "multilingual":
 595                 cmds.push("<group-multilingual>")
 596             else:
 597                 raise NotImplementedError(
 598                     f"Handling of multipart/{item.subtype} is not implemented"
 599                 )
 600
 601             state["pos"] -= len(state["tags"][item]) - 1
 602             state["parts"] += 1
 603             del state["tags"][item]
 604
 605         else:
 606             # We should never get here
 607             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
 608
 609         # If the item has a description, we might just as well add it
 610         if item.desc:
 611             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 612
 613         if ancestry:
 614             # If there's an ancestry, record the current (assumed) position in
 615             # the NeoMutt compose window as needed-to-tag by our direct parent
 616             # (i.e. the last item of the ancestry)
 617             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
 618
 619             lead = "│ " * (len(ancestry) + 1) + "* "
 620             debugprint(
 621                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
 622                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
 623                 f"{lead}pos={state['pos']}, parts={state['parts']}"
 624             )
 625
 626     # -----------------
 627     # End of visitor_fn
 628
 629     # Let's walk the tree and visit every node with our fancy visitor
 630     # function
 631     mimetree.walk(tree, visitor_fn=visitor_fn)
 632
 633     if not only_build:
 634         cmds.push("<send-message>")
 635
 636     # Finally, cleanup. Since we're responsible for removing the temporary
 637     # file, how's this for a little hack?
 638     try:
 639         filename = cmd_f.name
 640     except AttributeError:
 641         filename = "pytest_internal_file"
 642     cmds.cmd(f"source 'rm -f {filename}|'")
 643     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 644     cmds.flush()
 645
 646
 647 # [ CLI ENTRY ] ###############################################################
 648
 649 if __name__ == "__main__":
 650     args = parse_cli_args()
 651
 652     if args.mode is None:
 653         do_setup(
 654             tempdir=args.tempdir,
 655             debug_commands=args.debug_commands,
 656         )
 657
 658     elif args.mode == "massage":
 659         with open(args.MAILDRAFT, "r") as draft_f, open(
 660             args.cmdpath, "w"
 661         ) as cmd_f:
 662             do_massage(
 663                 draft_f,
 664                 pathlib.Path(args.MAILDRAFT),
 665                 cmd_f,
 666                 extensions=args.extensions,
 667                 cssfile=args.css_file,
 668                 only_build=args.only_build,
 669                 tempdir=args.tempdir,
 670                 debug_commands=args.debug_commands,
 671                 debug_walk=args.debug_walk,
 672             )
 673
 674
 675 # [ TESTS ] ###################################################################
 676
 677 try:
 678     import pytest
 679     from io import StringIO
 680
 681     class Tests:
 682         @pytest.fixture
 683         def const1(self):
 684             return "CONSTANT STRING 1"
 685
 686         @pytest.fixture
 687         def const2(self):
 688             return "CONSTANT STRING 2"
 689
 690         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 691         # functions they call, else old stdout is used and not captured
 692
 693         def test_MuttCommands_cmd(self, const1, const2, capsys):
 694             "Assert order of commands"
 695             cmds = MuttCommands(out_f=sys.stdout)
 696             cmds.cmd(const1)
 697             cmds.cmd(const2)
 698             cmds.flush()
 699             captured = capsys.readouterr()
 700             assert captured.out == "\n".join((const1, const2, ""))
 701
 702         def test_MuttCommands_push(self, const1, const2, capsys):
 703             "Assert reverse order of pushes"
 704             cmds = MuttCommands(out_f=sys.stdout)
 705             cmds.push(const1)
 706             cmds.push(const2)
 707             cmds.flush()
 708             captured = capsys.readouterr()
 709             assert (
 710                 captured.out
 711                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 712             )
 713
 714         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 715             "Assert reverse order of pushes"
 716             cmds = MuttCommands(out_f=sys.stdout)
 717             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 718             for i in range(2):
 719                 cmds.cmd(lines[4 * i + 0])
 720                 cmds.cmd(lines[4 * i + 1])
 721                 cmds.push(lines[4 * i + 2])
 722                 cmds.push(lines[4 * i + 3])
 723             cmds.flush()
 724
 725             captured = capsys.readouterr()
 726             lines_out = captured.out.splitlines()
 727             assert lines[0] in lines_out[0]
 728             assert lines[1] in lines_out[1]
 729             assert lines[7] in lines_out[2]
 730             assert lines[6] in lines_out[3]
 731             assert lines[3] in lines_out[4]
 732             assert lines[2] in lines_out[5]
 733             assert lines[4] in lines_out[6]
 734             assert lines[5] in lines_out[7]
 735
 736         @pytest.fixture
 737         def mime_tree_related_to_alternative(self):
 738             return Multipart(
 739                 "relative",
 740                 children=[
 741                     Multipart(
 742                         "alternative",
 743                         children=[
 744                             Part(
 745                                 "text",
 746                                 "plain",
 747                                 "part.txt",
 748                                 desc="Plain",
 749                                 orig=True,
 750                             ),
 751                             Part("text", "html", "part.html", desc="HTML"),
 752                         ],
 753                         desc="Alternative",
 754                     ),
 755                     Part(
 756                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 757                     ),
 758                 ],
 759                 desc="Related",
 760             )
 761
 762         @pytest.fixture
 763         def mime_tree_related_to_html(self):
 764             return Multipart(
 765                 "alternative",
 766                 children=[
 767                     Part(
 768                         "text",
 769                         "plain",
 770                         "part.txt",
 771                         desc="Plain",
 772                         orig=True,
 773                     ),
 774                     Multipart(
 775                         "relative",
 776                         children=[
 777                             Part("text", "html", "part.html", desc="HTML"),
 778                             Part(
 779                                 "text",
 780                                 "png",
 781                                 "logo.png",
 782                                 cid="logo.png",
 783                                 desc="Logo",
 784                             ),
 785                         ],
 786                         desc="Related",
 787                     ),
 788                 ],
 789                 desc="Alternative",
 790             )
 791
 792         def test_MIMETreeDFWalker_depth_first_walk(
 793             self, mime_tree_related_to_alternative
 794         ):
 795             mimetree = MIMETreeDFWalker()
 796
 797             items = []
 798
 799             def visitor_fn(item, ancestry, debugprint):
 800                 items.append((item, len(ancestry)))
 801
 802             mimetree.walk(
 803                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
 804             )
 805             assert len(items) == 5
 806             assert items[0][0].subtype == "plain"
 807             assert items[0][1] == 2
 808             assert items[1][0].subtype == "html"
 809             assert items[1][1] == 2
 810             assert items[2][0].subtype == "alternative"
 811             assert items[2][1] == 1
 812             assert items[3][0].subtype == "png"
 813             assert items[3][1] == 1
 814             assert items[4][0].subtype == "relative"
 815             assert items[4][1] == 0
 816
 817         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
 818             mimetree = MIMETreeDFWalker()
 819             items = []
 820
 821             def visitor_fn(item, ancestry, debugprint):
 822                 items.append(item)
 823
 824             p = Part("text", "plain", const1)
 825             mimetree.walk([p], visitor_fn=visitor_fn)
 826             assert items[-1].subtype == "plain"
 827             mimetree.walk([p, p], visitor_fn=visitor_fn)
 828             assert items[-1].subtype == "mixed"
 829
 830         def test_MIMETreeDFWalker_visitor_in_constructor(
 831             self, mime_tree_related_to_alternative
 832         ):
 833             items = []
 834
 835             def visitor_fn(item, ancestry, debugprint):
 836                 items.append(item)
 837
 838             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 839             mimetree.walk(mime_tree_related_to_alternative)
 840             assert len(items) == 5
 841
 842         @pytest.fixture
 843         def string_io(self, const1, text=None):
 844             return StringIO(text or const1)
 845
 846         def test_do_massage_basic(self, const1, string_io, capsys):
 847             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 848                 return Part("text", "plain", draftpath, orig=True)
 849
 850             do_massage(
 851                 draft_f=string_io,
 852                 draftpath=const1,
 853                 cmd_f=sys.stdout,
 854                 converter=converter,
 855             )
 856
 857             captured = capsys.readouterr()
 858             lines = captured.out.splitlines()
 859             assert '="$my_editor"' in lines.pop(0)
 860             assert '="$my_edit_headers"' in lines.pop(0)
 861             assert "unset my_editor" == lines.pop(0)
 862             assert "unset my_edit_headers" == lines.pop(0)
 863             assert "send-message" in lines.pop(0)
 864             assert "update-encoding" in lines.pop(0)
 865             assert "first-entry" in lines.pop(0)
 866             assert "source 'rm -f " in lines.pop(0)
 867             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 868
 869         def test_do_massage_fulltree(
 870             self, string_io, const1, mime_tree_related_to_alternative, capsys
 871         ):
 872             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 873                 return mime_tree_related_to_alternative
 874
 875             do_massage(
 876                 draft_f=string_io,
 877                 draftpath=const1,
 878                 cmd_f=sys.stdout,
 879                 converter=converter,
 880             )
 881
 882             captured = capsys.readouterr()
 883             lines = captured.out.splitlines()[4:-2]
 884             print(lines)
 885             assert "first-entry" in lines.pop()
 886             assert "update-encoding" in lines.pop()
 887             assert "Plain" in lines.pop()
 888             assert "part.html" in lines.pop()
 889             assert "toggle-unlink" in lines.pop()
 890             assert "HTML" in lines.pop()
 891             assert "jump>1" in lines.pop()
 892             assert "jump>2" in lines.pop()
 893             assert "group-alternatives" in lines.pop()
 894             assert "Alternative" in lines.pop()
 895             assert "logo.png" in lines.pop()
 896             assert "toggle-unlink" in lines.pop()
 897             assert "content-id" in lines.pop()
 898             assert "Logo" in lines.pop()
 899             assert "jump>1" in lines.pop()
 900             assert "jump>4" in lines.pop()
 901             assert "group-related" in lines.pop()
 902             assert "Related" in lines.pop()
 903             assert "send-message" in lines.pop()
 904             assert len(lines) == 0
 905
 906         @pytest.fixture
 907         def fake_filewriter(self):
 908             class FileWriter:
 909                 def __init__(self):
 910                     self._writes = []
 911
 912                 def __call__(self, path, content, mode="w", **kwargs):
 913                     self._writes.append((path, content))
 914
 915                 def pop(self, index=-1):
 916                     return self._writes.pop(index)
 917
 918             return FileWriter()
 919
 920         @pytest.fixture
 921         def markdown_non_converter(self, const1, const2):
 922             return lambda s, text: f"{const1}{text}{const2}"
 923
 924         def test_converter_tree_basic(self, const1, const2, fake_filewriter):
 925             path = pathlib.Path(const2)
 926             tree = convert_markdown_to_html(
 927                 const1, path, filewriter_fn=fake_filewriter
 928             )
 929
 930             assert tree.subtype == "alternative"
 931             assert len(tree.children) == 2
 932             assert tree.children[0].subtype == "plain"
 933             assert tree.children[0].path == path
 934             assert tree.children[0].orig
 935             assert tree.children[1].subtype == "html"
 936             assert tree.children[1].path == path.with_suffix(".html")
 937
 938         def test_converter_writes(
 939             self,
 940             const1,
 941             const2,
 942             fake_filewriter,
 943             monkeypatch,
 944             markdown_non_converter,
 945         ):
 946             path = pathlib.Path(const2)
 947
 948             with monkeypatch.context() as m:
 949                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 950                 convert_markdown_to_html(
 951                     const1, path, filewriter_fn=fake_filewriter
 952                 )
 953
 954             assert (path, const1) == fake_filewriter.pop(0)
 955             assert (
 956                 path.with_suffix(".html"),
 957                 markdown_non_converter(None, const1),
 958             ) == fake_filewriter.pop(0)
 959
 960         def test_markdown_inline_image_processor(self):
 961             imgpath1 = "file:/path/to/image.png"
 962             imgpath2 = "file:///path/to/image.png?url=params"
 963             imgpath3 = "/path/to/image.png"
 964             text = f"""![inline local image]({imgpath1})
 965                        ![image inlined
 966                          with newline]({imgpath2})
 967                        ![image local path]({imgpath3})"""
 968             text, html, images = markdown_with_inline_image_support(text)
 969
 970             # local paths have been normalised to URLs:
 971             imgpath3 = f"file://{imgpath3}"
 972
 973             assert 'src="cid:' in html
 974             assert "](cid:" in text
 975             assert len(images) == 3
 976             assert imgpath1 in images
 977             assert imgpath2 in images
 978             assert imgpath3 in images
 979             assert images[imgpath1].cid != images[imgpath2].cid
 980             assert images[imgpath1].cid != images[imgpath3].cid
 981             assert images[imgpath2].cid != images[imgpath3].cid
 982
 983         def test_markdown_inline_image_processor_title_to_desc(self, const1):
 984             imgpath = "file:///path/to/image.png"
 985             text = f'![inline local image]({imgpath} "{const1}")'
 986             text, html, images = markdown_with_inline_image_support(text)
 987             assert images[imgpath].desc == const1
 988
 989         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
 990             imgpath = "file:///path/to/image.png"
 991             text = f"![{const1}]({imgpath})"
 992             text, html, images = markdown_with_inline_image_support(text)
 993             assert images[imgpath].desc == const1
 994
 995         def test_markdown_inline_image_processor_title_over_alt_desc(
 996             self, const1, const2
 997         ):
 998             imgpath = "file:///path/to/image.png"
 999             text = f'![{const1}]({imgpath} "{const2}")'
1000             text, html, images = markdown_with_inline_image_support(text)
1001             assert images[imgpath].desc == const2
1002
1003         def test_markdown_inline_image_not_external(self):
1004             imgpath = "https://path/to/image.png"
1005             text = f"![inline image]({imgpath})"
1006             text, html, images = markdown_with_inline_image_support(text)
1007
1008             assert 'src="cid:' not in html
1009             assert "](cid:" not in text
1010             assert len(images) == 0
1011
1012         def test_markdown_inline_image_local_file(self):
1013             imgpath = "/path/to/image.png"
1014             text = f"![inline image]({imgpath})"
1015             text, html, images = markdown_with_inline_image_support(text)
1016
1017             for k, v in images.items():
1018                 assert k == f"file://{imgpath}"
1019                 break
1020
1021         @pytest.fixture
1022         def test_png(self):
1023             return (
1024                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1025                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1026             )
1027
1028         def test_markdown_inline_image_processor_base64(self, test_png):
1029             text = f"![1px white inlined]({test_png})"
1030             text, html, images = markdown_with_inline_image_support(text)
1031
1032             assert 'src="cid:' in html
1033             assert "](cid:" in text
1034             assert len(images) == 1
1035             assert test_png in images
1036
1037         def test_converter_tree_inline_image_base64(
1038             self, test_png, const1, fake_filewriter
1039         ):
1040             text = f"![inline base64 image]({test_png})"
1041             path = pathlib.Path(const1)
1042             tree = convert_markdown_to_html(
1043                 text,
1044                 path,
1045                 filewriter_fn=fake_filewriter,
1046                 related_to_html_only=False,
1047             )
1048             assert tree.subtype == "relative"
1049             assert tree.children[0].subtype == "alternative"
1050             assert tree.children[1].subtype == "png"
1051             written = fake_filewriter.pop()
1052             assert tree.children[1].path == written[0]
1053             assert written[1] == request.urlopen(test_png).read()
1054
1055         def test_converter_tree_inline_image_base64_related_to_html(
1056             self, test_png, const1, fake_filewriter
1057         ):
1058             text = f"![inline base64 image]({test_png})"
1059             path = pathlib.Path(const1)
1060             tree = convert_markdown_to_html(
1061                 text,
1062                 path,
1063                 filewriter_fn=fake_filewriter,
1064                 related_to_html_only=True,
1065             )
1066             assert tree.subtype == "alternative"
1067             assert tree.children[1].subtype == "relative"
1068             assert tree.children[1].children[1].subtype == "png"
1069             written = fake_filewriter.pop()
1070             assert tree.children[1].children[1].path == written[0]
1071             assert written[1] == request.urlopen(test_png).read()
1072
1073         def test_converter_tree_inline_image_cid(
1074             self, const1, fake_filewriter
1075         ):
1076             text = f"![inline base64 image](cid:{const1})"
1077             path = pathlib.Path(const1)
1078             tree = convert_markdown_to_html(
1079                 text,
1080                 path,
1081                 filewriter_fn=fake_filewriter,
1082                 related_to_html_only=False,
1083             )
1084             assert len(tree.children) == 2
1085             assert tree.children[0].cid != const1
1086             assert tree.children[0].type != "image"
1087             assert tree.children[1].cid != const1
1088             assert tree.children[1].type != "image"
1089
1090         def test_inline_image_collection(
1091             self, test_png, const1, const2, fake_filewriter
1092         ):
1093             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1094             relparts = collect_inline_images(
1095                 test_images, filewriter_fn=fake_filewriter
1096             )
1097
1098             written = fake_filewriter.pop()
1099             assert b"PNG" in written[1]
1100
1101             assert relparts[0].subtype == "png"
1102             assert relparts[0].path == written[0]
1103             assert relparts[0].cid == const1
1104             assert relparts[0].desc.endswith(const2)
1105
1106         def test_apply_stylesheet(self):
1107             if _PYNLINER:
1108                 html = "<p>Hello, world!</p>"
1109                 css = "p { color:red }"
1110                 out = apply_styling(html, css)
1111                 assert 'p style="color' in out
1112
1113         def test_apply_stylesheet_pygments(self):
1114             if _PYGMENTS_CSS:
1115                 html = (
1116                     f'<div class="{_CODEHILITE_CLASS}">'
1117                     "<pre>def foo():\n    return</pre></div>"
1118                 )
1119                 out = apply_styling(html, _PYGMENTS_CSS)
1120                 assert f'{_CODEHILITE_CLASS}" style="' in out
1121
1122         def test_mime_tree_relative_within_alternative(
1123             self, string_io, const1, capsys, mime_tree_related_to_html
1124         ):
1125             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
1126                 return mime_tree_related_to_html
1127
1128             do_massage(
1129                 draft_f=string_io,
1130                 draftpath=const1,
1131                 cmd_f=sys.stdout,
1132                 converter=converter,
1133             )
1134
1135             captured = capsys.readouterr()
1136             lines = captured.out.splitlines()[4:-2]
1137             assert "first-entry" in lines.pop()
1138             assert "update-encoding" in lines.pop()
1139             assert "Plain" in lines.pop()
1140             assert "part.html" in lines.pop()
1141             assert "toggle-unlink" in lines.pop()
1142             assert "HTML" in lines.pop()
1143             assert "logo.png" in lines.pop()
1144             assert "toggle-unlink" in lines.pop()
1145             assert "content-id" in lines.pop()
1146             assert "Logo" in lines.pop()
1147             assert "jump>2" in lines.pop()
1148             assert "jump>3" in lines.pop()
1149             assert "group-related" in lines.pop()
1150             assert "Related" in lines.pop()
1151             assert "jump>1" in lines.pop()
1152             assert "jump>2" in lines.pop()
1153             assert "group-alternative" in lines.pop()
1154             assert "Alternative" in lines.pop()
1155             assert "send-message" in lines.pop()
1156             assert len(lines) == 0
1157
1158 except ImportError:
1159     pass