.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
  13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  14 #     " "Convert message into a modern MIME tree with inline images"
  15 #
  16 #     (Yes, we need to call source twice, as mutt only starts to process output
  17 #     from a source command when the command exits, and since we need to react
  18 #     to the output, we need to be invoked again, using a $my_ variable to pass
  19 #     information)
  20 #
  21 # Requirements:
  22 #   - python3
  23 #   - python3-markdown
  24 # Optional:
  25 #   - pytest
  26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  27 #   - Pygments, then syntax highlighting for fenced code is enabled
  28 #
  29 # Latest version:
  30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  31 #
  32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  33 # Released under the GPL-2+ licence, just like Mutt itself.
  34 #
  35
  36 import sys
  37 import pathlib
  38 import markdown
  39 import tempfile
  40 import argparse
  41 import re
  42 import mimetypes
  43 from collections import namedtuple, OrderedDict
  44 from markdown.extensions import Extension
  45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
  46 from email.utils import make_msgid
  47 from urllib import request
  48
  49
  50 def parse_cli_args(*args, **kwargs):
  51     parser = argparse.ArgumentParser(
  52         description=(
  53             "NeoMutt helper to turn text/markdown email parts "
  54             "into full-fledged MIME trees"
  55         )
  56     )
  57     parser.epilog = (
  58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  59         "Released under the MIT licence"
  60     )
  61
  62     parser.add_argument(
  63         "--extensions",
  64         type=str,
  65         default="",
  66         help="Markdown extension to use (comma-separated list)",
  67     )
  68
  69     if _PYNLINER:
  70         parser.add_argument(
  71             "--css-file",
  72             type=str,
  73             default="",
  74             help="CSS file to merge with the final HTML",
  75         )
  76     else:
  77         parser.set_defaults(css_file=None)
  78
  79     parser.add_argument(
  80         "--only-build",
  81         action="store_true",
  82         help="Only build, don't send the message",
  83     )
  84
  85     parser.add_argument(
  86         "--tempdir",
  87         default=None,
  88         help="Specify temporary directory to use for attachments",
  89     )
  90
  91     parser.add_argument(
  92         "--debug-commands",
  93         action="store_true",
  94         help="Turn on debug logging of commands generated to stderr",
  95     )
  96
  97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  98     massage_p = subp.add_parser(
  99         "massage", help="Massaging phase (internal use)"
 100     )
 101
 102     massage_p.add_argument(
 103         "--write-commands-to",
 104         "-o",
 105         metavar="PATH",
 106         dest="cmdpath",
 107         required=True,
 108         help="Temporary file path to write commands to",
 109     )
 110
 111     massage_p.add_argument(
 112         "--debug-walk",
 113         action="store_true",
 114         help="Turn on debugging to stderr of the MIME tree walk",
 115     )
 116
 117     massage_p.add_argument(
 118         "MAILDRAFT",
 119         nargs="?",
 120         help="If provided, the script is invoked as editor on the mail draft",
 121     )
 122
 123     return parser.parse_args(*args, **kwargs)
 124
 125
 126 # [ MARKDOWN WRAPPING ] #######################################################
 127
 128
 129 InlineImageInfo = namedtuple(
 130     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 131 )
 132
 133
 134 class InlineImageExtension(Extension):
 135     class RelatedImageInlineProcessor(ImageInlineProcessor):
 136         def __init__(self, re, md, ext):
 137             super().__init__(re, md)
 138             self._ext = ext
 139
 140         def handleMatch(self, m, data):
 141             el, start, end = super().handleMatch(m, data)
 142             if "src" in el.attrib:
 143                 src = el.attrib["src"]
 144                 if "://" not in src or src.startswith("file://"):
 145                     # We only inline local content
 146                     cid = self._ext.get_cid_for_image(el.attrib)
 147                     el.attrib["src"] = f"cid:{cid}"
 148             return el, start, end
 149
 150     def __init__(self):
 151         super().__init__()
 152         self._images = OrderedDict()
 153
 154     def extendMarkdown(self, md):
 155         md.registerExtension(self)
 156         inline_image_proc = self.RelatedImageInlineProcessor(
 157             IMAGE_LINK_RE, md, self
 158         )
 159         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
 160
 161     def get_cid_for_image(self, attrib):
 162         msgid = make_msgid()[1:-1]
 163         path = attrib["src"]
 164         if path.startswith("/"):
 165             path = f"file://{path}"
 166         self._images[path] = InlineImageInfo(
 167             msgid, attrib.get("title", attrib.get("alt"))
 168         )
 169         return msgid
 170
 171     def get_images(self):
 172         return self._images
 173
 174
 175 def markdown_with_inline_image_support(
 176     text, *, extensions=None, extension_configs=None
 177 ):
 178     inline_image_handler = InlineImageExtension()
 179     extensions = extensions or []
 180     extensions.append(inline_image_handler)
 181     mdwn = markdown.Markdown(
 182         extensions=extensions, extension_configs=extension_configs
 183     )
 184     htmltext = mdwn.convert(text)
 185
 186     images = inline_image_handler.get_images()
 187
 188     def replace_image_with_cid(matchobj):
 189         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 190             if m in images:
 191                 return f"(cid:{images[m].cid}"
 192         return matchobj.group(0)
 193
 194     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 195     return text, htmltext, images
 196
 197
 198 # [ CSS STYLING ] #############################################################
 199
 200 try:
 201     import pynliner
 202
 203     _PYNLINER = True
 204
 205 except ImportError:
 206     _PYNLINER = False
 207
 208 try:
 209     from pygments.formatters import get_formatter_by_name
 210
 211     _CODEHILITE_CLASS = "codehilite"
 212
 213     _PYGMENTS_CSS = get_formatter_by_name(
 214         "html", style="default"
 215     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 216
 217 except ImportError:
 218     _PYGMENTS_CSS = None
 219
 220
 221 def apply_styling(html, css):
 222     return (
 223         pynliner.Pynliner()
 224         .from_string(html)
 225         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 226         .run()
 227     )
 228
 229
 230 # [ PARTS GENERATION ] ########################################################
 231
 232
 233 class Part(
 234     namedtuple(
 235         "Part",
 236         ["type", "subtype", "path", "desc", "cid", "orig"],
 237         defaults=[None, None, False],
 238     )
 239 ):
 240     def __str__(self):
 241         ret = f"<{self.type}/{self.subtype}>"
 242         if self.cid:
 243             ret = f"{ret} cid:{self.cid}"
 244         if self.orig:
 245             ret = f"{ret} ORIGINAL"
 246         return ret
 247
 248
 249 class Multipart(
 250     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 251 ):
 252     def __str__(self):
 253         return f"<multipart/{self.subtype}> children={len(self.children)}"
 254
 255     def __hash__(self):
 256         return hash(str(self.subtype) + "".join(str(self.children)))
 257
 258
 259 def filewriter_fn(path, content, mode="w", **kwargs):
 260     with open(path, mode, **kwargs) as out_f:
 261         out_f.write(content)
 262
 263
 264 def collect_inline_images(
 265     images, *, tempdir=None, filewriter_fn=filewriter_fn
 266 ):
 267     relparts = []
 268     for path, info in images.items():
 269         if path.startswith("cid:"):
 270             continue
 271
 272         data = request.urlopen(path)
 273
 274         mimetype = data.headers["Content-Type"]
 275         ext = mimetypes.guess_extension(mimetype)
 276         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 277         path = pathlib.Path(tempfilename[1])
 278
 279         filewriter_fn(path, data.read(), "w+b")
 280
 281         relparts.append(
 282             Part(
 283                 *mimetype.split("/"),
 284                 path,
 285                 cid=info.cid,
 286                 desc=f"Image: {info.desc}",
 287             )
 288         )
 289
 290     return relparts
 291
 292
 293 def convert_markdown_to_html(
 294     origtext,
 295     draftpath,
 296     *,
 297     cssfile=None,
 298     filewriter_fn=filewriter_fn,
 299     tempdir=None,
 300     extensions=None,
 301     extension_configs=None,
 302 ):
 303     # TODO extension_configs need to be handled differently
 304     extension_configs = extension_configs or {}
 305     extension_configs.setdefault("pymdownx.highlight", {})
 306     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
 307
 308     origtext, htmltext, images = markdown_with_inline_image_support(
 309         origtext, extensions=extensions, extension_configs=extension_configs
 310     )
 311
 312     filewriter_fn(draftpath, origtext, encoding="utf-8")
 313     textpart = Part(
 314         "text", "plain", draftpath, "Plain-text version", orig=True
 315     )
 316
 317     htmltext = apply_styling(htmltext, cssfile)
 318
 319     htmlpath = draftpath.with_suffix(".html")
 320     filewriter_fn(
 321         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 322     )
 323     htmlpart = Part("text", "html", htmlpath, "HTML version")
 324
 325     altpart = Multipart(
 326         "alternative", [textpart, htmlpart], "Group of alternative content"
 327     )
 328
 329     imgparts = collect_inline_images(
 330         images, tempdir=tempdir, filewriter_fn=filewriter_fn
 331     )
 332     if imgparts:
 333         return Multipart(
 334             "relative", [altpart] + imgparts, "Group of related content"
 335         )
 336     else:
 337         return altpart
 338
 339
 340 class MIMETreeDFWalker:
 341     def __init__(self, *, visitor_fn=None, debug=False):
 342         self._visitor_fn = visitor_fn or self._echovisit
 343         self._debug = debug
 344
 345     def _echovisit(self, node, ancestry, debugprint):
 346         debugprint(f"node={node} ancestry={ancestry}")
 347
 348     def walk(self, root, *, visitor_fn=None):
 349         """
 350         Recursive function to implement a depth-dirst walk of the MIME-tree
 351         rooted at `root`.
 352         """
 353         if isinstance(root, list):
 354             if len(root) > 1:
 355                 root = Multipart("mixed", children=root)
 356             else:
 357                 root = root[0]
 358
 359         self._walk(
 360             root,
 361             ancestry=[],
 362             visitor_fn=visitor_fn or self._visitor_fn,
 363         )
 364
 365     def _walk(self, node, *, ancestry, visitor_fn):
 366         # Let's start by enumerating the parts at the current level. At the
 367         # root level, ancestry will be the empty list, and we expect a
 368         # multipart/* container at this level. Later, e.g. within a
 369         # mutlipart/alternative container, the subtree will just be the
 370         # alternative parts, while the top of the ancestry will be the
 371         # multipart/alternative container, which we will process after the
 372         # following loop.
 373
 374         lead = f"{'│ '*len(ancestry)}"
 375         if isinstance(node, Multipart):
 376             self.debugprint(
 377                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
 378             )
 379
 380             # Depth-first, so push the current container onto the ancestry
 381             # stack, then descend …
 382             ancestry.append(node)
 383             self.debugprint(lead + "│ " * 2)
 384             for child in node.children:
 385                 self._walk(
 386                     child,
 387                     ancestry=ancestry,
 388                     visitor_fn=visitor_fn,
 389                 )
 390             assert ancestry.pop() == node
 391
 392         else:
 393             self.debugprint(f"{lead}├{node}")
 394
 395         if False and ancestry:
 396             self.debugprint(lead[:-1] + " │")
 397
 398         if visitor_fn:
 399             visitor_fn(node, ancestry, debugprint=self.debugprint)
 400
 401     def debugprint(self, s, **kwargs):
 402         if self._debug:
 403             print(s, file=sys.stderr, **kwargs)
 404
 405
 406 # [ RUN MODES ] ###############################################################
 407
 408
 409 class MuttCommands:
 410     """
 411     Stupid class to interface writing out Mutt commands. This is quite a hack
 412     to deal with the fact that Mutt runs "push" commands in reverse order, so
 413     all of a sudden, things become very complicated when mixing with "real"
 414     commands.
 415
 416     Hence we keep two sets of commands, and one set of pushes. Commands are
 417     added to the first until a push is added, after which commands are added to
 418     the second set of commands.
 419
 420     On flush(), the first set is printed, followed by the pushes in reverse,
 421     and then the second set is printed. All 3 sets are then cleared.
 422     """
 423
 424     def __init__(self, out_f=sys.stdout, *, debug=False):
 425         self._cmd1, self._push, self._cmd2 = [], [], []
 426         self._out_f = out_f
 427         self._debug = debug
 428
 429     def cmd(self, s):
 430         self.debugprint(s)
 431         if self._push:
 432             self._cmd2.append(s)
 433         else:
 434             self._cmd1.append(s)
 435
 436     def push(self, s):
 437         s = s.replace('"', '"')
 438         s = f'push "{s}"'
 439         self.debugprint(s)
 440         self._push.insert(0, s)
 441
 442     def flush(self):
 443         print(
 444             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 445         )
 446         self._cmd1, self._push, self._cmd2 = [], [], []
 447
 448     def debugprint(self, s, **kwargs):
 449         if self._debug:
 450             print(s, file=sys.stderr, **kwargs)
 451
 452
 453 def do_setup(
 454     *,
 455     out_f=sys.stdout,
 456     temppath=None,
 457     tempdir=None,
 458     debug_commands=False,
 459 ):
 460     temppath = temppath or pathlib.Path(
 461         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 462     )
 463     cmds = MuttCommands(out_f, debug=debug_commands)
 464
 465     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 466
 467     cmds.cmd('set my_editor="$editor"')
 468     cmds.cmd('set my_edit_headers="$edit_headers"')
 469     cmds.cmd(f'set editor="{editor}"')
 470     cmds.cmd("unset edit_headers")
 471     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 472     cmds.push("<first-entry><edit-file>")
 473     cmds.flush()
 474
 475
 476 def do_massage(
 477     draft_f,
 478     draftpath,
 479     cmd_f,
 480     *,
 481     extensions=None,
 482     cssfile=None,
 483     converter=convert_markdown_to_html,
 484     only_build=False,
 485     tempdir=None,
 486     debug_commands=False,
 487     debug_walk=False,
 488 ):
 489     # Here's the big picture: we're being invoked as the editor on the email
 490     # draft, and whatever commands we write to the file given as cmdpath will
 491     # be run by the second source command in the macro definition.
 492
 493     # Let's start by cleaning up what the setup did (see above), i.e. we
 494     # restore the $editor and $edit_headers variables, and also unset the
 495     # variable used to identify the command file we're currently writing
 496     # to.
 497     cmds = MuttCommands(cmd_f, debug=debug_commands)
 498     cmds.cmd('set editor="$my_editor"')
 499     cmds.cmd('set edit_headers="$my_edit_headers"')
 500     cmds.cmd("unset my_editor")
 501     cmds.cmd("unset my_edit_headers")
 502
 503     # let's flush those commands, as there'll be a lot of pushes from now
 504     # on, which need to be run in reverse order
 505     cmds.flush()
 506
 507     extensions = extensions.split(",") if extensions else []
 508     tree = converter(
 509         draft_f.read(),
 510         draftpath,
 511         cssfile=cssfile,
 512         tempdir=tempdir,
 513         extensions=extensions,
 514     )
 515
 516     mimetree = MIMETreeDFWalker(debug=debug_walk)
 517
 518     state = dict(pos=1, tags={}, parts=1)
 519
 520     def visitor_fn(item, ancestry, *, debugprint=None):
 521         """
 522         Visitor function called for every node (part) of the MIME tree,
 523         depth-first, and responsible for telling NeoMutt how to assemble
 524         the tree.
 525         """
 526         KILL_LINE = r"\Ca\Ck"
 527
 528         if isinstance(item, Part):
 529             # We've hit a leaf-node, i.e. an alternative or a related part
 530             # with actual content.
 531
 532             # Let's add the part
 533             if item.orig:
 534                 # The original source already exists in the NeoMutt tree, but
 535                 # the underlying file may have been modified, so we need to
 536                 # update the encoding, but that's it:
 537                 cmds.push("<first-entry>")
 538                 cmds.push("<update-encoding>")
 539
 540                 # We really just need to be able to assume that at this point,
 541                 # NeoMutt is at position 1, and that we've processed only this
 542                 # part so far. Nevermind about actual attachments, we can
 543                 # safely ignore those as they stay at the end.
 544                 assert state["pos"] == 1
 545                 assert state["parts"] == 1
 546             else:
 547                 # … whereas all other parts need to be added, and they're all
 548                 # considered to be temporary and inline:
 549                 cmds.push(f"<attach-file>{item.path}<enter>")
 550                 cmds.push("<toggle-unlink><toggle-disposition>")
 551
 552                 # This added a part at the end of the list of parts, and that's
 553                 # just how many parts we've seen so far, so it's position in
 554                 # the NeoMutt compose list is the count of parts
 555                 state["parts"] += 1
 556                 state["pos"] = state["parts"]
 557
 558             # If the item (including the original) comes with additional
 559             # information, then we might just as well update the NeoMutt
 560             # tree now:
 561             if item.cid:
 562                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 563
 564         elif isinstance(item, Multipart):
 565             # This node has children, but we already visited them (see
 566             # above). The tags dictionary of State should contain a list of
 567             # their positions in the NeoMutt compose window, so iterate those
 568             # and tag the parts there:
 569             for tag in state["tags"][item]:
 570                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
 571
 572             if item.subtype == "alternative":
 573                 cmds.push("<group-alternatives>")
 574             elif item.subtype in ("relative", "related"):
 575                 cmds.push("<group-related>")
 576             elif item.subtype == "multilingual":
 577                 cmds.push("<group-multilingual>")
 578             else:
 579                 raise NotImplementedError(
 580                     f"Handling of multipart/{item.subtype} is not implemented"
 581                 )
 582
 583             state["pos"] -= len(state["tags"][item]) - 1
 584             state["parts"] += 1
 585             del state["tags"][item]
 586
 587         else:
 588             # We should never get here
 589             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
 590
 591         # If the item has a description, we might just as well add it
 592         if item.desc:
 593             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 594
 595         if ancestry:
 596             # If there's an ancestry, record the current (assumed) position in
 597             # the NeoMutt compose window as needed-to-tag by our direct parent
 598             # (i.e. the last item of the ancestry)
 599             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
 600
 601             lead = "│ " * (len(ancestry) + 1) + "* "
 602             debugprint(
 603                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
 604                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
 605                 f"{lead}pos={state['pos']}, parts={state['parts']}"
 606             )
 607
 608     # -----------------
 609     # End of visitor_fn
 610
 611     # Let's walk the tree and visit every node with our fancy visitor
 612     # function
 613     mimetree.walk(tree, visitor_fn=visitor_fn)
 614
 615     if not only_build:
 616         cmds.push("<send-message>")
 617
 618     # Finally, cleanup. Since we're responsible for removing the temporary
 619     # file, how's this for a little hack?
 620     try:
 621         filename = cmd_f.name
 622     except AttributeError:
 623         filename = "pytest_internal_file"
 624     cmds.cmd(f"source 'rm -f {filename}|'")
 625     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 626     cmds.flush()
 627
 628
 629 # [ CLI ENTRY ] ###############################################################
 630
 631 if __name__ == "__main__":
 632     args = parse_cli_args()
 633
 634     if args.mode is None:
 635         do_setup(
 636             tempdir=args.tempdir,
 637             debug_commands=args.debug_commands,
 638         )
 639
 640     elif args.mode == "massage":
 641         with open(args.MAILDRAFT, "r") as draft_f, open(
 642             args.cmdpath, "w"
 643         ) as cmd_f:
 644             do_massage(
 645                 draft_f,
 646                 pathlib.Path(args.MAILDRAFT),
 647                 cmd_f,
 648                 extensions=args.extensions,
 649                 cssfile=args.css_file,
 650                 only_build=args.only_build,
 651                 tempdir=args.tempdir,
 652                 debug_commands=args.debug_commands,
 653                 debug_walk=args.debug_walk,
 654             )
 655
 656
 657 # [ TESTS ] ###################################################################
 658
 659 try:
 660     import pytest
 661     from io import StringIO
 662
 663     class Tests:
 664         @pytest.fixture
 665         def const1(self):
 666             return "CONSTANT STRING 1"
 667
 668         @pytest.fixture
 669         def const2(self):
 670             return "CONSTANT STRING 2"
 671
 672         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 673         # functions they call, else old stdout is used and not captured
 674
 675         def test_MuttCommands_cmd(self, const1, const2, capsys):
 676             "Assert order of commands"
 677             cmds = MuttCommands(out_f=sys.stdout)
 678             cmds.cmd(const1)
 679             cmds.cmd(const2)
 680             cmds.flush()
 681             captured = capsys.readouterr()
 682             assert captured.out == "\n".join((const1, const2, ""))
 683
 684         def test_MuttCommands_push(self, const1, const2, capsys):
 685             "Assert reverse order of pushes"
 686             cmds = MuttCommands(out_f=sys.stdout)
 687             cmds.push(const1)
 688             cmds.push(const2)
 689             cmds.flush()
 690             captured = capsys.readouterr()
 691             assert (
 692                 captured.out
 693                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 694             )
 695
 696         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 697             "Assert reverse order of pushes"
 698             cmds = MuttCommands(out_f=sys.stdout)
 699             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 700             for i in range(2):
 701                 cmds.cmd(lines[4 * i + 0])
 702                 cmds.cmd(lines[4 * i + 1])
 703                 cmds.push(lines[4 * i + 2])
 704                 cmds.push(lines[4 * i + 3])
 705             cmds.flush()
 706
 707             captured = capsys.readouterr()
 708             lines_out = captured.out.splitlines()
 709             assert lines[0] in lines_out[0]
 710             assert lines[1] in lines_out[1]
 711             assert lines[7] in lines_out[2]
 712             assert lines[6] in lines_out[3]
 713             assert lines[3] in lines_out[4]
 714             assert lines[2] in lines_out[5]
 715             assert lines[4] in lines_out[6]
 716             assert lines[5] in lines_out[7]
 717
 718         @pytest.fixture
 719         def basic_mime_tree(self):
 720             return Multipart(
 721                 "relative",
 722                 children=[
 723                     Multipart(
 724                         "alternative",
 725                         children=[
 726                             Part(
 727                                 "text",
 728                                 "plain",
 729                                 "part.txt",
 730                                 desc="Plain",
 731                                 orig=True,
 732                             ),
 733                             Part("text", "html", "part.html", desc="HTML"),
 734                         ],
 735                         desc="Alternative",
 736                     ),
 737                     Part(
 738                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 739                     ),
 740                 ],
 741                 desc="Related",
 742             )
 743
 744         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 745             mimetree = MIMETreeDFWalker()
 746
 747             items = []
 748
 749             def visitor_fn(item, stack, debugprint):
 750                 items.append((item, len(stack)))
 751
 752             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 753             assert len(items) == 5
 754             assert items[0][0].subtype == "plain"
 755             assert items[0][1] == 2
 756             assert items[1][0].subtype == "html"
 757             assert items[1][1] == 2
 758             assert items[2][0].subtype == "alternative"
 759             assert items[2][1] == 1
 760             assert items[3][0].subtype == "png"
 761             assert items[3][1] == 1
 762             assert items[4][0].subtype == "relative"
 763             assert items[4][1] == 0
 764
 765         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
 766             mimetree = MIMETreeDFWalker()
 767             items = []
 768
 769             def visitor_fn(item, ancestry, debugprint):
 770                 items.append(item)
 771
 772             p = Part("text", "plain", const1)
 773             mimetree.walk([p], visitor_fn=visitor_fn)
 774             assert items[-1].subtype == "plain"
 775             mimetree.walk([p, p], visitor_fn=visitor_fn)
 776             assert items[-1].subtype == "mixed"
 777
 778         def test_MIMETreeDFWalker_visitor_in_constructor(
 779             self, basic_mime_tree
 780         ):
 781             items = []
 782
 783             def visitor_fn(item, ancestry, debugprint):
 784                 items.append(item)
 785
 786             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 787             mimetree.walk(basic_mime_tree)
 788             assert len(items) == 5
 789
 790         @pytest.fixture
 791         def string_io(self, const1, text=None):
 792             return StringIO(text or const1)
 793
 794         def test_do_massage_basic(self, const1, string_io, capsys):
 795             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 796                 return Part("text", "plain", draftpath, orig=True)
 797
 798             do_massage(
 799                 draft_f=string_io,
 800                 draftpath=const1,
 801                 cmd_f=sys.stdout,
 802                 converter=converter,
 803             )
 804
 805             captured = capsys.readouterr()
 806             lines = captured.out.splitlines()
 807             assert '="$my_editor"' in lines.pop(0)
 808             assert '="$my_edit_headers"' in lines.pop(0)
 809             assert "unset my_editor" == lines.pop(0)
 810             assert "unset my_edit_headers" == lines.pop(0)
 811             assert "send-message" in lines.pop(0)
 812             assert "update-encoding" in lines.pop(0)
 813             assert "source 'rm -f " in lines.pop(0)
 814             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 815
 816         def test_do_massage_fulltree(
 817             self, string_io, const1, basic_mime_tree, capsys
 818         ):
 819             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
 820                 return basic_mime_tree
 821
 822             do_massage(
 823                 draft_f=string_io,
 824                 draftpath=const1,
 825                 cmd_f=sys.stdout,
 826                 converter=converter,
 827             )
 828
 829             captured = capsys.readouterr()
 830             lines = captured.out.splitlines()[4:]
 831             assert "send-message" in lines.pop(0)
 832             assert "Related" in lines.pop(0)
 833             assert "group-related" in lines.pop(0)
 834             assert "tag-entry" in lines.pop(0)
 835             assert "Logo" in lines.pop(0)
 836             assert "content-id" in lines.pop(0)
 837             assert "toggle-unlink" in lines.pop(0)
 838             assert "logo.png" in lines.pop(0)
 839             assert "tag-entry" in lines.pop(0)
 840             assert "Alternative" in lines.pop(0)
 841             assert "group-alternatives" in lines.pop(0)
 842             assert "tag-entry" in lines.pop(0)
 843             assert "HTML" in lines.pop(0)
 844             assert "toggle-unlink" in lines.pop(0)
 845             assert "part.html" in lines.pop(0)
 846             assert "tag-entry" in lines.pop(0)
 847             assert "Plain" in lines.pop(0)
 848             assert "update-encoding" in lines.pop(0)
 849             assert len(lines) == 2
 850
 851         @pytest.fixture
 852         def fake_filewriter(self):
 853             class FileWriter:
 854                 def __init__(self):
 855                     self._writes = []
 856
 857                 def __call__(self, path, content, mode="w", **kwargs):
 858                     self._writes.append((path, content))
 859
 860                 def pop(self, index=-1):
 861                     return self._writes.pop(index)
 862
 863             return FileWriter()
 864
 865         @pytest.fixture
 866         def markdown_non_converter(self, const1, const2):
 867             return lambda s, text: f"{const1}{text}{const2}"
 868
 869         def test_converter_tree_basic(
 870             self, const1, const2, fake_filewriter, markdown_non_converter
 871         ):
 872             path = pathlib.Path(const2)
 873             tree = convert_markdown_to_html(
 874                 const1, path, filewriter_fn=fake_filewriter
 875             )
 876
 877             assert tree.subtype == "alternative"
 878             assert len(tree.children) == 2
 879             assert tree.children[0].subtype == "plain"
 880             assert tree.children[0].path == path
 881             assert tree.children[0].orig
 882             assert tree.children[1].subtype == "html"
 883             assert tree.children[1].path == path.with_suffix(".html")
 884
 885         def test_converter_writes(
 886             self,
 887             const1,
 888             const2,
 889             fake_filewriter,
 890             monkeypatch,
 891             markdown_non_converter,
 892         ):
 893             path = pathlib.Path(const2)
 894
 895             with monkeypatch.context() as m:
 896                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
 897                 convert_markdown_to_html(
 898                     const1, path, filewriter_fn=fake_filewriter
 899                 )
 900
 901             assert (path, const1) == fake_filewriter.pop(0)
 902             assert (
 903                 path.with_suffix(".html"),
 904                 markdown_non_converter(None, const1),
 905             ) == fake_filewriter.pop(0)
 906
 907         def test_markdown_inline_image_processor(self):
 908             imgpath1 = "file:/path/to/image.png"
 909             imgpath2 = "file:///path/to/image.png?url=params"
 910             imgpath3 = "/path/to/image.png"
 911             text = f"""![inline local image]({imgpath1})
 912                        ![image inlined
 913                          with newline]({imgpath2})
 914                        ![image local path]({imgpath3})"""
 915             text, html, images = markdown_with_inline_image_support(text)
 916
 917             # local paths have been normalised to URLs:
 918             imgpath3 = f"file://{imgpath3}"
 919
 920             assert 'src="cid:' in html
 921             assert "](cid:" in text
 922             assert len(images) == 3
 923             assert imgpath1 in images
 924             assert imgpath2 in images
 925             assert imgpath3 in images
 926             assert images[imgpath1].cid != images[imgpath2].cid
 927             assert images[imgpath1].cid != images[imgpath3].cid
 928             assert images[imgpath2].cid != images[imgpath3].cid
 929
 930         def test_markdown_inline_image_processor_title_to_desc(self, const1):
 931             imgpath = "file:///path/to/image.png"
 932             text = f'![inline local image]({imgpath} "{const1}")'
 933             text, html, images = markdown_with_inline_image_support(text)
 934             assert images[imgpath].desc == const1
 935
 936         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
 937             imgpath = "file:///path/to/image.png"
 938             text = f"![{const1}]({imgpath})"
 939             text, html, images = markdown_with_inline_image_support(text)
 940             assert images[imgpath].desc == const1
 941
 942         def test_markdown_inline_image_processor_title_over_alt_desc(
 943             self, const1, const2
 944         ):
 945             imgpath = "file:///path/to/image.png"
 946             text = f'![{const1}]({imgpath} "{const2}")'
 947             text, html, images = markdown_with_inline_image_support(text)
 948             assert images[imgpath].desc == const2
 949
 950         def test_markdown_inline_image_not_external(self):
 951             imgpath = "https://path/to/image.png"
 952             text = f"![inline image]({imgpath})"
 953             text, html, images = markdown_with_inline_image_support(text)
 954
 955             assert 'src="cid:' not in html
 956             assert "](cid:" not in text
 957             assert len(images) == 0
 958
 959         def test_markdown_inline_image_local_file(self):
 960             imgpath = "/path/to/image.png"
 961             text = f"![inline image]({imgpath})"
 962             text, html, images = markdown_with_inline_image_support(text)
 963
 964             for k, v in images.items():
 965                 assert k == f"file://{imgpath}"
 966                 break
 967
 968         @pytest.fixture
 969         def test_png(self):
 970             return (
 971                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
 972                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
 973             )
 974
 975         def test_markdown_inline_image_processor_base64(self, test_png):
 976             text = f"![1px white inlined]({test_png})"
 977             text, html, images = markdown_with_inline_image_support(text)
 978
 979             assert 'src="cid:' in html
 980             assert "](cid:" in text
 981             assert len(images) == 1
 982             assert test_png in images
 983
 984         def test_converter_tree_inline_image_base64(
 985             self, test_png, const1, fake_filewriter
 986         ):
 987             text = f"![inline base64 image]({test_png})"
 988             path = pathlib.Path(const1)
 989             tree = convert_markdown_to_html(
 990                 text, path, filewriter_fn=fake_filewriter
 991             )
 992
 993             assert tree.subtype == "relative"
 994             assert tree.children[1].subtype == "png"
 995             written = fake_filewriter.pop()
 996             assert tree.children[1].path == written[0]
 997             assert written[1] == request.urlopen(test_png).read()
 998
 999         def test_converter_tree_inline_image_cid(
1000             self, const1, fake_filewriter
1001         ):
1002             text = f"![inline base64 image](cid:{const1})"
1003             path = pathlib.Path(const1)
1004             tree = convert_markdown_to_html(
1005                 text,
1006                 path,
1007                 filewriter_fn=fake_filewriter,
1008                 related_to_html_only=False,
1009             )
1010             assert len(tree.children) == 2
1011             assert tree.children[0].cid != const1
1012             assert tree.children[0].type != "image"
1013             assert tree.children[1].cid != const1
1014             assert tree.children[1].type != "image"
1015
1016         def test_inline_image_collection(
1017             self, test_png, const1, const2, fake_filewriter
1018         ):
1019             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1020             relparts = collect_inline_images(
1021                 test_images, filewriter_fn=fake_filewriter
1022             )
1023
1024             written = fake_filewriter.pop()
1025             assert b"PNG" in written[1]
1026
1027             assert relparts[0].subtype == "png"
1028             assert relparts[0].path == written[0]
1029             assert relparts[0].cid == const1
1030             assert relparts[0].desc.endswith(const2)
1031
1032         def test_apply_stylesheet(self):
1033             if _PYNLINER:
1034                 html = "<p>Hello, world!</p>"
1035                 css = "p { color:red }"
1036                 out = apply_styling(html, css)
1037                 assert 'p style="color' in out
1038
1039         def test_apply_stylesheet_pygments(self):
1040             if _PYGMENTS_CSS:
1041                 html = (
1042                     f'<div class="{_CODEHILITE_CLASS}">'
1043                     "<pre>def foo():\n    return</pre></div>"
1044                 )
1045                 out = apply_styling(html, _PYGMENTS_CSS)
1046                 assert f'{_CODEHILITE_CLASS}" style="' in out
1047
1048
1049 except ImportError:
1050     pass