]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

e41ca611d2a86c158d0103844da3cb41c146a21a
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
10 #     macro compose B "\
11 #       <enter-command> source '$my_confdir/buildmimetree.py \
12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
14 #     " "Convert message into a modern MIME tree with inline images"
15 #
16 #     (Yes, we need to call source twice, as mutt only starts to process output
17 #     from a source command when the command exits, and since we need to react
18 #     to the output, we need to be invoked again, using a $my_ variable to pass
19 #     information)
20 #
21 # Requirements:
22 #   - python3
23 #   - python3-markdown
24 # Optional:
25 #   - pytest
26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
27 #   - Pygments, then syntax highlighting for fenced code is enabled
28 #
29 # Latest version:
30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
31 #
32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
33 # Released under the GPL-2+ licence, just like Mutt itself.
34 #
35
36 import sys
37 import pathlib
38 import markdown
39 import tempfile
40 import argparse
41 import re
42 import mimetypes
43 from collections import namedtuple, OrderedDict
44 from markdown.extensions import Extension
45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
46 from email.utils import make_msgid
47 from urllib import request
48
49
50 def parse_cli_args(*args, **kwargs):
51     parser = argparse.ArgumentParser(
52         description=(
53             "NeoMutt helper to turn text/markdown email parts "
54             "into full-fledged MIME trees"
55         )
56     )
57     parser.epilog = (
58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
59         "Released under the MIT licence"
60     )
61
62     parser.add_argument(
63         "--extensions",
64         type=str,
65         default="",
66         help="Markdown extension to use (comma-separated list)",
67     )
68
69     if _PYNLINER:
70         parser.add_argument(
71             "--css-file",
72             type=str,
73             default="",
74             help="CSS file to merge with the final HTML",
75         )
76     else:
77         parser.set_defaults(css_file=None)
78
79     parser.add_argument(
80         "--only-build",
81         action="store_true",
82         help="Only build, don't send the message",
83     )
84
85     parser.add_argument(
86         "--tempdir",
87         default=None,
88         help="Specify temporary directory to use for attachments",
89     )
90
91     parser.add_argument(
92         "--debug-commands",
93         action="store_true",
94         help="Turn on debug logging of commands generated to stderr",
95     )
96
97     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
98     massage_p = subp.add_parser(
99         "massage", help="Massaging phase (internal use)"
100     )
101
102     massage_p.add_argument(
103         "--write-commands-to",
104         "-o",
105         metavar="PATH",
106         dest="cmdpath",
107         required=True,
108         help="Temporary file path to write commands to",
109     )
110
111     massage_p.add_argument(
112         "--debug-walk",
113         action="store_true",
114         help="Turn on debugging to stderr of the MIME tree walk",
115     )
116
117     massage_p.add_argument(
118         "MAILDRAFT",
119         nargs="?",
120         help="If provided, the script is invoked as editor on the mail draft",
121     )
122
123     return parser.parse_args(*args, **kwargs)
124
125
126 # [ MARKDOWN WRAPPING ] #######################################################
127
128
129 InlineImageInfo = namedtuple(
130     "InlineImageInfo", ["cid", "desc"], defaults=[None]
131 )
132
133
134 class InlineImageExtension(Extension):
135     class RelatedImageInlineProcessor(ImageInlineProcessor):
136         def __init__(self, re, md, ext):
137             super().__init__(re, md)
138             self._ext = ext
139
140         def handleMatch(self, m, data):
141             el, start, end = super().handleMatch(m, data)
142             if "src" in el.attrib:
143                 src = el.attrib["src"]
144                 if "://" not in src or src.startswith("file://"):
145                     # We only inline local content
146                     cid = self._ext.get_cid_for_image(el.attrib)
147                     el.attrib["src"] = f"cid:{cid}"
148             return el, start, end
149
150     def __init__(self):
151         super().__init__()
152         self._images = OrderedDict()
153
154     def extendMarkdown(self, md):
155         md.registerExtension(self)
156         inline_image_proc = self.RelatedImageInlineProcessor(
157             IMAGE_LINK_RE, md, self
158         )
159         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
160
161     def get_cid_for_image(self, attrib):
162         msgid = make_msgid()[1:-1]
163         path = attrib["src"]
164         if path.startswith("/"):
165             path = f"file://{path}"
166         self._images[path] = InlineImageInfo(
167             msgid, attrib.get("title", attrib.get("alt"))
168         )
169         return msgid
170
171     def get_images(self):
172         return self._images
173
174
175 def markdown_with_inline_image_support(
176     text, *, extensions=None, extension_configs=None
177 ):
178     inline_image_handler = InlineImageExtension()
179     extensions = extensions or []
180     extensions.append(inline_image_handler)
181     mdwn = markdown.Markdown(
182         extensions=extensions, extension_configs=extension_configs
183     )
184     htmltext = mdwn.convert(text)
185
186     images = inline_image_handler.get_images()
187
188     def replace_image_with_cid(matchobj):
189         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
190             if m in images:
191                 return f"(cid:{images[m].cid}"
192         return matchobj.group(0)
193
194     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
195     return text, htmltext, images
196
197
198 # [ CSS STYLING ] #############################################################
199
200 try:
201     import pynliner
202
203     _PYNLINER = True
204
205 except ImportError:
206     _PYNLINER = False
207
208 try:
209     from pygments.formatters import get_formatter_by_name
210
211     _CODEHILITE_CLASS = "codehilite"
212
213     _PYGMENTS_CSS = get_formatter_by_name(
214         "html", style="default"
215     ).get_style_defs(f".{_CODEHILITE_CLASS}")
216
217 except ImportError:
218     _PYGMENTS_CSS = None
219
220
221 def apply_styling(html, css):
222     return (
223         pynliner.Pynliner()
224         .from_string(html)
225         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
226         .run()
227     )
228
229
230 # [ PARTS GENERATION ] ########################################################
231
232
233 class Part(
234     namedtuple(
235         "Part",
236         ["type", "subtype", "path", "desc", "cid", "orig"],
237         defaults=[None, None, False],
238     )
239 ):
240     def __str__(self):
241         ret = f"<{self.type}/{self.subtype}>"
242         if self.cid:
243             ret = f"{ret} cid:{self.cid}"
244         if self.orig:
245             ret = f"{ret} ORIGINAL"
246         return ret
247
248
249 class Multipart(
250     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
251 ):
252     def __str__(self):
253         return f"<multipart/{self.subtype}> children={len(self.children)}"
254
255     def __hash__(self):
256         return hash(str(self.subtype) + "".join(str(self.children)))
257
258
259 def filewriter_fn(path, content, mode="w", **kwargs):
260     with open(path, mode, **kwargs) as out_f:
261         out_f.write(content)
262
263
264 def collect_inline_images(
265     images, *, tempdir=None, filewriter_fn=filewriter_fn
266 ):
267     relparts = []
268     for path, info in images.items():
269         if path.startswith("cid:"):
270             continue
271
272         data = request.urlopen(path)
273
274         mimetype = data.headers["Content-Type"]
275         ext = mimetypes.guess_extension(mimetype)
276         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
277         path = pathlib.Path(tempfilename[1])
278
279         filewriter_fn(path, data.read(), "w+b")
280
281         relparts.append(
282             Part(
283                 *mimetype.split("/"),
284                 path,
285                 cid=info.cid,
286                 desc=f"Image: {info.desc}",
287             )
288         )
289
290     return relparts
291
292
293 def convert_markdown_to_html(
294     origtext,
295     draftpath,
296     *,
297     cssfile=None,
298     filewriter_fn=filewriter_fn,
299     tempdir=None,
300     extensions=None,
301     extension_configs=None,
302 ):
303     # TODO extension_configs need to be handled differently
304     extension_configs = extension_configs or {}
305     extension_configs.setdefault("pymdownx.highlight", {})
306     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
307
308     origtext, htmltext, images = markdown_with_inline_image_support(
309         origtext, extensions=extensions, extension_configs=extension_configs
310     )
311
312     filewriter_fn(draftpath, origtext, encoding="utf-8")
313     textpart = Part(
314         "text", "plain", draftpath, "Plain-text version", orig=True
315     )
316
317     htmltext = apply_styling(htmltext, cssfile)
318
319     htmlpath = draftpath.with_suffix(".html")
320     filewriter_fn(
321         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
322     )
323     htmlpart = Part("text", "html", htmlpath, "HTML version")
324
325     altpart = Multipart(
326         "alternative", [textpart, htmlpart], "Group of alternative content"
327     )
328
329     imgparts = collect_inline_images(
330         images, tempdir=tempdir, filewriter_fn=filewriter_fn
331     )
332     if imgparts:
333         return Multipart(
334             "relative", [altpart] + imgparts, "Group of related content"
335         )
336     else:
337         return altpart
338
339
340 class MIMETreeDFWalker:
341     def __init__(self, *, visitor_fn=None, debug=False):
342         self._visitor_fn = visitor_fn or self._echovisit
343         self._debug = debug
344
345     def _echovisit(self, node, ancestry, debugprint):
346         debugprint(f"node={node} ancestry={ancestry}")
347
348     def walk(self, root, *, visitor_fn=None):
349         """
350         Recursive function to implement a depth-dirst walk of the MIME-tree
351         rooted at `root`.
352         """
353         if isinstance(root, list):
354             if len(root) > 1:
355                 root = Multipart("mixed", children=root)
356             else:
357                 root = root[0]
358
359         self._walk(
360             root,
361             ancestry=[],
362             visitor_fn=visitor_fn or self._visitor_fn,
363         )
364
365     def _walk(self, node, *, ancestry, visitor_fn):
366         # Let's start by enumerating the parts at the current level. At the
367         # root level, ancestry will be the empty list, and we expect a
368         # multipart/* container at this level. Later, e.g. within a
369         # mutlipart/alternative container, the subtree will just be the
370         # alternative parts, while the top of the ancestry will be the
371         # multipart/alternative container, which we will process after the
372         # following loop.
373
374         lead = f"{'│ '*len(ancestry)}"
375         if isinstance(node, Multipart):
376             self.debugprint(
377                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
378             )
379
380             # Depth-first, so push the current container onto the ancestry
381             # stack, then descend …
382             ancestry.append(node)
383             self.debugprint(lead + "│ " * 2)
384             for child in node.children:
385                 self._walk(
386                     child,
387                     ancestry=ancestry,
388                     visitor_fn=visitor_fn,
389                 )
390             assert ancestry.pop() == node
391
392         else:
393             self.debugprint(f"{lead}├{node}")
394
395         if False and ancestry:
396             self.debugprint(lead[:-1] + " │")
397
398         if visitor_fn:
399             visitor_fn(node, ancestry, debugprint=self.debugprint)
400
401     def debugprint(self, s, **kwargs):
402         if self._debug:
403             print(s, file=sys.stderr, **kwargs)
404
405
406 # [ RUN MODES ] ###############################################################
407
408
409 class MuttCommands:
410     """
411     Stupid class to interface writing out Mutt commands. This is quite a hack
412     to deal with the fact that Mutt runs "push" commands in reverse order, so
413     all of a sudden, things become very complicated when mixing with "real"
414     commands.
415
416     Hence we keep two sets of commands, and one set of pushes. Commands are
417     added to the first until a push is added, after which commands are added to
418     the second set of commands.
419
420     On flush(), the first set is printed, followed by the pushes in reverse,
421     and then the second set is printed. All 3 sets are then cleared.
422     """
423
424     def __init__(self, out_f=sys.stdout, *, debug=False):
425         self._cmd1, self._push, self._cmd2 = [], [], []
426         self._out_f = out_f
427         self._debug = debug
428
429     def cmd(self, s):
430         self.debugprint(s)
431         if self._push:
432             self._cmd2.append(s)
433         else:
434             self._cmd1.append(s)
435
436     def push(self, s):
437         s = s.replace('"', '"')
438         s = f'push "{s}"'
439         self.debugprint(s)
440         self._push.insert(0, s)
441
442     def flush(self):
443         print(
444             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
445         )
446         self._cmd1, self._push, self._cmd2 = [], [], []
447
448     def debugprint(self, s, **kwargs):
449         if self._debug:
450             print(s, file=sys.stderr, **kwargs)
451
452
453 def do_setup(
454     *,
455     out_f=sys.stdout,
456     temppath=None,
457     tempdir=None,
458     debug_commands=False,
459 ):
460     temppath = temppath or pathlib.Path(
461         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
462     )
463     cmds = MuttCommands(out_f, debug=debug_commands)
464
465     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
466
467     cmds.cmd('set my_editor="$editor"')
468     cmds.cmd('set my_edit_headers="$edit_headers"')
469     cmds.cmd(f'set editor="{editor}"')
470     cmds.cmd("unset edit_headers")
471     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
472     cmds.push("<first-entry><edit-file>")
473     cmds.flush()
474
475
476 def do_massage(
477     draft_f,
478     draftpath,
479     cmd_f,
480     *,
481     extensions=None,
482     cssfile=None,
483     converter=convert_markdown_to_html,
484     only_build=False,
485     tempdir=None,
486     debug_commands=False,
487     debug_walk=False,
488 ):
489     # Here's the big picture: we're being invoked as the editor on the email
490     # draft, and whatever commands we write to the file given as cmdpath will
491     # be run by the second source command in the macro definition.
492
493     # Let's start by cleaning up what the setup did (see above), i.e. we
494     # restore the $editor and $edit_headers variables, and also unset the
495     # variable used to identify the command file we're currently writing
496     # to.
497     cmds = MuttCommands(cmd_f, debug=debug_commands)
498     cmds.cmd('set editor="$my_editor"')
499     cmds.cmd('set edit_headers="$my_edit_headers"')
500     cmds.cmd("unset my_editor")
501     cmds.cmd("unset my_edit_headers")
502
503     # let's flush those commands, as there'll be a lot of pushes from now
504     # on, which need to be run in reverse order
505     cmds.flush()
506
507     extensions = extensions.split(",") if extensions else []
508     tree = converter(
509         draft_f.read(),
510         draftpath,
511         cssfile=cssfile,
512         tempdir=tempdir,
513         extensions=extensions,
514     )
515
516     mimetree = MIMETreeDFWalker(debug=debug_walk)
517
518     state = dict(pos=1, tags={}, parts=1)
519
520     def visitor_fn(item, ancestry, *, debugprint=None):
521         """
522         Visitor function called for every node (part) of the MIME tree,
523         depth-first, and responsible for telling NeoMutt how to assemble
524         the tree.
525         """
526         KILL_LINE = r"\Ca\Ck"
527
528         if isinstance(item, Part):
529             # We've hit a leaf-node, i.e. an alternative or a related part
530             # with actual content.
531
532             # Let's add the part
533             if item.orig:
534                 # The original source already exists in the NeoMutt tree, but
535                 # the underlying file may have been modified, so we need to
536                 # update the encoding, but that's it:
537                 cmds.push("<first-entry>")
538                 cmds.push("<update-encoding>")
539
540                 # We really just need to be able to assume that at this point,
541                 # NeoMutt is at position 1, and that we've processed only this
542                 # part so far. Nevermind about actual attachments, we can
543                 # safely ignore those as they stay at the end.
544                 assert state["pos"] == 1
545                 assert state["parts"] == 1
546             else:
547                 # … whereas all other parts need to be added, and they're all
548                 # considered to be temporary and inline:
549                 cmds.push(f"<attach-file>{item.path}<enter>")
550                 cmds.push("<toggle-unlink><toggle-disposition>")
551
552                 # This added a part at the end of the list of parts, and that's
553                 # just how many parts we've seen so far, so it's position in
554                 # the NeoMutt compose list is the count of parts
555                 state["parts"] += 1
556                 state["pos"] = state["parts"]
557
558             # If the item (including the original) comes with additional
559             # information, then we might just as well update the NeoMutt
560             # tree now:
561             if item.cid:
562                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
563
564         elif isinstance(item, Multipart):
565             # This node has children, but we already visited them (see
566             # above). The tags dictionary of State should contain a list of
567             # their positions in the NeoMutt compose window, so iterate those
568             # and tag the parts there:
569             for tag in state["tags"][item]:
570                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
571
572             if item.subtype == "alternative":
573                 cmds.push("<group-alternatives>")
574             elif item.subtype in ("relative", "related"):
575                 cmds.push("<group-related>")
576             elif item.subtype == "multilingual":
577                 cmds.push("<group-multilingual>")
578             else:
579                 raise NotImplementedError(
580                     f"Handling of multipart/{item.subtype} is not implemented"
581                 )
582
583             state["pos"] -= len(state["tags"][item]) - 1
584             state["parts"] += 1
585             del state["tags"][item]
586
587         else:
588             # We should never get here
589             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
590
591         # If the item has a description, we might just as well add it
592         if item.desc:
593             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
594
595         if ancestry:
596             # If there's an ancestry, record the current (assumed) position in
597             # the NeoMutt compose window as needed-to-tag by our direct parent
598             # (i.e. the last item of the ancestry)
599             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
600
601             lead = "│ " * (len(ancestry) + 1) + "* "
602             debugprint(
603                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
604                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
605                 f"{lead}pos={state['pos']}, parts={state['parts']}"
606             )
607
608     # -----------------
609     # End of visitor_fn
610
611     # Let's walk the tree and visit every node with our fancy visitor
612     # function
613     mimetree.walk(tree, visitor_fn=visitor_fn)
614
615     if not only_build:
616         cmds.push("<send-message>")
617
618     # Finally, cleanup. Since we're responsible for removing the temporary
619     # file, how's this for a little hack?
620     try:
621         filename = cmd_f.name
622     except AttributeError:
623         filename = "pytest_internal_file"
624     cmds.cmd(f"source 'rm -f {filename}|'")
625     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
626     cmds.flush()
627
628
629 # [ CLI ENTRY ] ###############################################################
630
631 if __name__ == "__main__":
632     args = parse_cli_args()
633
634     if args.mode is None:
635         do_setup(
636             tempdir=args.tempdir,
637             debug_commands=args.debug_commands,
638         )
639
640     elif args.mode == "massage":
641         with open(args.MAILDRAFT, "r") as draft_f, open(
642             args.cmdpath, "w"
643         ) as cmd_f:
644             do_massage(
645                 draft_f,
646                 pathlib.Path(args.MAILDRAFT),
647                 cmd_f,
648                 extensions=args.extensions,
649                 cssfile=args.css_file,
650                 only_build=args.only_build,
651                 tempdir=args.tempdir,
652                 debug_commands=args.debug_commands,
653                 debug_walk=args.debug_walk,
654             )
655
656
657 # [ TESTS ] ###################################################################
658
659 try:
660     import pytest
661     from io import StringIO
662
663     class Tests:
664         @pytest.fixture
665         def const1(self):
666             return "CONSTANT STRING 1"
667
668         @pytest.fixture
669         def const2(self):
670             return "CONSTANT STRING 2"
671
672         # NOTE: tests using the capsys fixture must specify sys.stdout to the
673         # functions they call, else old stdout is used and not captured
674
675         def test_MuttCommands_cmd(self, const1, const2, capsys):
676             "Assert order of commands"
677             cmds = MuttCommands(out_f=sys.stdout)
678             cmds.cmd(const1)
679             cmds.cmd(const2)
680             cmds.flush()
681             captured = capsys.readouterr()
682             assert captured.out == "\n".join((const1, const2, ""))
683
684         def test_MuttCommands_push(self, const1, const2, capsys):
685             "Assert reverse order of pushes"
686             cmds = MuttCommands(out_f=sys.stdout)
687             cmds.push(const1)
688             cmds.push(const2)
689             cmds.flush()
690             captured = capsys.readouterr()
691             assert (
692                 captured.out
693                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
694             )
695
696         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
697             "Assert reverse order of pushes"
698             cmds = MuttCommands(out_f=sys.stdout)
699             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
700             for i in range(2):
701                 cmds.cmd(lines[4 * i + 0])
702                 cmds.cmd(lines[4 * i + 1])
703                 cmds.push(lines[4 * i + 2])
704                 cmds.push(lines[4 * i + 3])
705             cmds.flush()
706
707             captured = capsys.readouterr()
708             lines_out = captured.out.splitlines()
709             assert lines[0] in lines_out[0]
710             assert lines[1] in lines_out[1]
711             assert lines[7] in lines_out[2]
712             assert lines[6] in lines_out[3]
713             assert lines[3] in lines_out[4]
714             assert lines[2] in lines_out[5]
715             assert lines[4] in lines_out[6]
716             assert lines[5] in lines_out[7]
717
718         @pytest.fixture
719         def basic_mime_tree(self):
720             return Multipart(
721                 "relative",
722                 children=[
723                     Multipart(
724                         "alternative",
725                         children=[
726                             Part(
727                                 "text",
728                                 "plain",
729                                 "part.txt",
730                                 desc="Plain",
731                                 orig=True,
732                             ),
733                             Part("text", "html", "part.html", desc="HTML"),
734                         ],
735                         desc="Alternative",
736                     ),
737                     Part(
738                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
739                     ),
740                 ],
741                 desc="Related",
742             )
743
744         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
745             mimetree = MIMETreeDFWalker()
746
747             items = []
748
749             def visitor_fn(item, stack, debugprint):
750                 items.append((item, len(stack)))
751
752             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
753             assert len(items) == 5
754             assert items[0][0].subtype == "plain"
755             assert items[0][1] == 2
756             assert items[1][0].subtype == "html"
757             assert items[1][1] == 2
758             assert items[2][0].subtype == "alternative"
759             assert items[2][1] == 1
760             assert items[3][0].subtype == "png"
761             assert items[3][1] == 1
762             assert items[4][0].subtype == "relative"
763             assert items[4][1] == 0
764
765         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
766             mimetree = MIMETreeDFWalker()
767             items = []
768
769             def visitor_fn(item, ancestry, debugprint):
770                 items.append(item)
771
772             p = Part("text", "plain", const1)
773             mimetree.walk([p], visitor_fn=visitor_fn)
774             assert items[-1].subtype == "plain"
775             mimetree.walk([p, p], visitor_fn=visitor_fn)
776             assert items[-1].subtype == "mixed"
777
778         def test_MIMETreeDFWalker_visitor_in_constructor(
779             self, basic_mime_tree
780         ):
781             items = []
782
783             def visitor_fn(item, ancestry, debugprint):
784                 items.append(item)
785
786             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
787             mimetree.walk(basic_mime_tree)
788             assert len(items) == 5
789
790         @pytest.fixture
791         def string_io(self, const1, text=None):
792             return StringIO(text or const1)
793
794         def test_do_massage_basic(self, const1, string_io, capsys):
795             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
796                 return Part("text", "plain", draftpath, orig=True)
797
798             do_massage(
799                 draft_f=string_io,
800                 draftpath=const1,
801                 cmd_f=sys.stdout,
802                 converter=converter,
803             )
804
805             captured = capsys.readouterr()
806             lines = captured.out.splitlines()
807             assert '="$my_editor"' in lines.pop(0)
808             assert '="$my_edit_headers"' in lines.pop(0)
809             assert "unset my_editor" == lines.pop(0)
810             assert "unset my_edit_headers" == lines.pop(0)
811             assert "send-message" in lines.pop(0)
812             assert "update-encoding" in lines.pop(0)
813             assert "source 'rm -f " in lines.pop(0)
814             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
815
816         def test_do_massage_fulltree(
817             self, string_io, const1, basic_mime_tree, capsys
818         ):
819             def converter(drafttext, draftpath, cssfile, extensions, tempdir):
820                 return basic_mime_tree
821
822             do_massage(
823                 draft_f=string_io,
824                 draftpath=const1,
825                 cmd_f=sys.stdout,
826                 converter=converter,
827             )
828
829             captured = capsys.readouterr()
830             lines = captured.out.splitlines()[4:]
831             assert "send-message" in lines.pop(0)
832             assert "Related" in lines.pop(0)
833             assert "group-related" in lines.pop(0)
834             assert "tag-entry" in lines.pop(0)
835             assert "Logo" in lines.pop(0)
836             assert "content-id" in lines.pop(0)
837             assert "toggle-unlink" in lines.pop(0)
838             assert "logo.png" in lines.pop(0)
839             assert "tag-entry" in lines.pop(0)
840             assert "Alternative" in lines.pop(0)
841             assert "group-alternatives" in lines.pop(0)
842             assert "tag-entry" in lines.pop(0)
843             assert "HTML" in lines.pop(0)
844             assert "toggle-unlink" in lines.pop(0)
845             assert "part.html" in lines.pop(0)
846             assert "tag-entry" in lines.pop(0)
847             assert "Plain" in lines.pop(0)
848             assert "update-encoding" in lines.pop(0)
849             assert len(lines) == 2
850
851         @pytest.fixture
852         def fake_filewriter(self):
853             class FileWriter:
854                 def __init__(self):
855                     self._writes = []
856
857                 def __call__(self, path, content, mode="w", **kwargs):
858                     self._writes.append((path, content))
859
860                 def pop(self, index=-1):
861                     return self._writes.pop(index)
862
863             return FileWriter()
864
865         @pytest.fixture
866         def markdown_non_converter(self, const1, const2):
867             return lambda s, text: f"{const1}{text}{const2}"
868
869         def test_converter_tree_basic(
870             self, const1, const2, fake_filewriter, markdown_non_converter
871         ):
872             path = pathlib.Path(const2)
873             tree = convert_markdown_to_html(
874                 const1, path, filewriter_fn=fake_filewriter
875             )
876
877             assert tree.subtype == "alternative"
878             assert len(tree.children) == 2
879             assert tree.children[0].subtype == "plain"
880             assert tree.children[0].path == path
881             assert tree.children[0].orig
882             assert tree.children[1].subtype == "html"
883             assert tree.children[1].path == path.with_suffix(".html")
884
885         def test_converter_writes(
886             self,
887             const1,
888             const2,
889             fake_filewriter,
890             monkeypatch,
891             markdown_non_converter,
892         ):
893             path = pathlib.Path(const2)
894
895             with monkeypatch.context() as m:
896                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
897                 convert_markdown_to_html(
898                     const1, path, filewriter_fn=fake_filewriter
899                 )
900
901             assert (path, const1) == fake_filewriter.pop(0)
902             assert (
903                 path.with_suffix(".html"),
904                 markdown_non_converter(None, const1),
905             ) == fake_filewriter.pop(0)
906
907         def test_markdown_inline_image_processor(self):
908             imgpath1 = "file:/path/to/image.png"
909             imgpath2 = "file:///path/to/image.png?url=params"
910             imgpath3 = "/path/to/image.png"
911             text = f"""![inline local image]({imgpath1})
912                        ![image inlined
913                          with newline]({imgpath2})
914                        ![image local path]({imgpath3})"""
915             text, html, images = markdown_with_inline_image_support(text)
916
917             # local paths have been normalised to URLs:
918             imgpath3 = f"file://{imgpath3}"
919
920             assert 'src="cid:' in html
921             assert "](cid:" in text
922             assert len(images) == 3
923             assert imgpath1 in images
924             assert imgpath2 in images
925             assert imgpath3 in images
926             assert images[imgpath1].cid != images[imgpath2].cid
927             assert images[imgpath1].cid != images[imgpath3].cid
928             assert images[imgpath2].cid != images[imgpath3].cid
929
930         def test_markdown_inline_image_processor_title_to_desc(self, const1):
931             imgpath = "file:///path/to/image.png"
932             text = f'![inline local image]({imgpath} "{const1}")'
933             text, html, images = markdown_with_inline_image_support(text)
934             assert images[imgpath].desc == const1
935
936         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
937             imgpath = "file:///path/to/image.png"
938             text = f"![{const1}]({imgpath})"
939             text, html, images = markdown_with_inline_image_support(text)
940             assert images[imgpath].desc == const1
941
942         def test_markdown_inline_image_processor_title_over_alt_desc(
943             self, const1, const2
944         ):
945             imgpath = "file:///path/to/image.png"
946             text = f'![{const1}]({imgpath} "{const2}")'
947             text, html, images = markdown_with_inline_image_support(text)
948             assert images[imgpath].desc == const2
949
950         def test_markdown_inline_image_not_external(self):
951             imgpath = "https://path/to/image.png"
952             text = f"![inline image]({imgpath})"
953             text, html, images = markdown_with_inline_image_support(text)
954
955             assert 'src="cid:' not in html
956             assert "](cid:" not in text
957             assert len(images) == 0
958
959         def test_markdown_inline_image_local_file(self):
960             imgpath = "/path/to/image.png"
961             text = f"![inline image]({imgpath})"
962             text, html, images = markdown_with_inline_image_support(text)
963
964             for k, v in images.items():
965                 assert k == f"file://{imgpath}"
966                 break
967
968         @pytest.fixture
969         def test_png(self):
970             return (
971                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
972                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
973             )
974
975         def test_markdown_inline_image_processor_base64(self, test_png):
976             text = f"![1px white inlined]({test_png})"
977             text, html, images = markdown_with_inline_image_support(text)
978
979             assert 'src="cid:' in html
980             assert "](cid:" in text
981             assert len(images) == 1
982             assert test_png in images
983
984         def test_converter_tree_inline_image_base64(
985             self, test_png, const1, fake_filewriter
986         ):
987             text = f"![inline base64 image]({test_png})"
988             path = pathlib.Path(const1)
989             tree = convert_markdown_to_html(
990                 text, path, filewriter_fn=fake_filewriter
991             )
992
993             assert tree.subtype == "relative"
994             assert tree.children[1].subtype == "png"
995             written = fake_filewriter.pop()
996             assert tree.children[1].path == written[0]
997             assert written[1] == request.urlopen(test_png).read()
998
999         def test_converter_tree_inline_image_cid(
1000             self, const1, fake_filewriter
1001         ):
1002             text = f"![inline base64 image](cid:{const1})"
1003             path = pathlib.Path(const1)
1004             tree = convert_markdown_to_html(
1005                 text,
1006                 path,
1007                 filewriter_fn=fake_filewriter,
1008                 related_to_html_only=False,
1009             )
1010             assert len(tree.children) == 2
1011             assert tree.children[0].cid != const1
1012             assert tree.children[0].type != "image"
1013             assert tree.children[1].cid != const1
1014             assert tree.children[1].type != "image"
1015
1016         def test_inline_image_collection(
1017             self, test_png, const1, const2, fake_filewriter
1018         ):
1019             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1020             relparts = collect_inline_images(
1021                 test_images, filewriter_fn=fake_filewriter
1022             )
1023
1024             written = fake_filewriter.pop()
1025             assert b"PNG" in written[1]
1026
1027             assert relparts[0].subtype == "png"
1028             assert relparts[0].path == written[0]
1029             assert relparts[0].cid == const1
1030             assert relparts[0].desc.endswith(const2)
1031
1032         def test_apply_stylesheet(self):
1033             if _PYNLINER:
1034                 html = "<p>Hello, world!</p>"
1035                 css = "p { color:red }"
1036                 out = apply_styling(html, css)
1037                 assert 'p style="color' in out
1038
1039         def test_apply_stylesheet_pygments(self):
1040             if _PYGMENTS_CSS:
1041                 html = (
1042                     f'<div class="{_CODEHILITE_CLASS}">'
1043                     "<pre>def foo():\n    return</pre></div>"
1044                 )
1045                 out = apply_styling(html, _PYGMENTS_CSS)
1046                 assert f'{_CODEHILITE_CLASS}" style="' in out
1047
1048
1049 except ImportError:
1050     pass