]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

00526327ddca23a14446c9901d214eb368b6b2fe
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
10 #     macro compose B "\
11 #       <enter-command> source '$my_confdir/buildmimetree.py \
12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
14 #     " "Convert message into a modern MIME tree with inline images"
15 #
16 #     (Yes, we need to call source twice, as mutt only starts to process output
17 #     from a source command when the command exits, and since we need to react
18 #     to the output, we need to be invoked again, using a $my_ variable to pass
19 #     information)
20 #
21 # Requirements:
22 #   - python3
23 #   - python3-markdown
24 # Optional:
25 #   - pytest
26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
27 #   - Pygments, then syntax highlighting for fenced code is enabled
28 #
29 # Latest version:
30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
31 #
32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
33 # Released under the GPL-2+ licence, just like Mutt itself.
34 #
35
36 import sys
37 import pathlib
38 import markdown
39 import tempfile
40 import argparse
41 import re
42 import mimetypes
43 from collections import namedtuple, OrderedDict
44 from markdown.extensions import Extension
45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
46 from email.utils import make_msgid
47 from urllib import request
48
49
50 def parse_cli_args(*args, **kwargs):
51     parser = argparse.ArgumentParser(
52         description=(
53             "NeoMutt helper to turn text/markdown email parts "
54             "into full-fledged MIME trees"
55         )
56     )
57     parser.epilog = (
58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
59         "Released under the MIT licence"
60     )
61
62     parser.add_argument(
63         "--extensions",
64         type=str,
65         default="",
66         help="Markdown extension to use (comma-separated list)",
67     )
68
69     if _PYNLINER:
70         parser.add_argument(
71             "--css-file",
72             type=pathlib.Path,
73             help="CSS file to merge with the final HTML",
74         )
75     else:
76         parser.set_defaults(css_file=None)
77
78     parser.add_argument(
79         "--related-to-html-only",
80         action="store_true",
81         help="Make related content be sibling to HTML parts only",
82     )
83
84     def positive_integer(value):
85         try:
86             if int(value) > 0:
87                 return int(value)
88
89         except ValueError:
90             pass
91
92         raise ValueError(f"Must be a positive integer")
93
94     parser.add_argument(
95         "--max-number-other-attachments",
96         type=positive_integer,
97         help="Make related content be sibling to HTML parts only",
98     )
99
100     parser.add_argument(
101         "--only-build",
102         action="store_true",
103         help="Only build, don't send the message",
104     )
105
106     parser.add_argument(
107         "--tempdir",
108         type=pathlib.Path,
109         help="Specify temporary directory to use for attachments",
110     )
111
112     parser.add_argument(
113         "--debug-commands",
114         action="store_true",
115         help="Turn on debug logging of commands generated to stderr",
116     )
117
118     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
119     massage_p = subp.add_parser(
120         "massage", help="Massaging phase (internal use)"
121     )
122
123     massage_p.add_argument(
124         "--write-commands-to",
125         "-o",
126         metavar="PATH",
127         dest="cmdpath",
128         type=pathlib.Path,
129         required=True,
130         help="Temporary file path to write commands to",
131     )
132
133     massage_p.add_argument(
134         "--debug-walk",
135         action="store_true",
136         help="Turn on debugging to stderr of the MIME tree walk",
137     )
138
139     massage_p.add_argument(
140         "MAILDRAFT",
141         nargs="?",
142         type=pathlib.Path,
143         help="If provided, the script is invoked as editor on the mail draft",
144     )
145
146     return parser.parse_args(*args, **kwargs)
147
148
149 # [ MARKDOWN WRAPPING ] #######################################################
150
151
152 InlineImageInfo = namedtuple(
153     "InlineImageInfo", ["cid", "desc"], defaults=[None]
154 )
155
156
157 class InlineImageExtension(Extension):
158     class RelatedImageInlineProcessor(ImageInlineProcessor):
159         def __init__(self, re, md, ext):
160             super().__init__(re, md)
161             self._ext = ext
162
163         def handleMatch(self, m, data):
164             el, start, end = super().handleMatch(m, data)
165             if "src" in el.attrib:
166                 src = el.attrib["src"]
167                 if "://" not in src or src.startswith("file://"):
168                     # We only inline local content
169                     cid = self._ext.get_cid_for_image(el.attrib)
170                     el.attrib["src"] = f"cid:{cid}"
171             return el, start, end
172
173     def __init__(self):
174         super().__init__()
175         self._images = OrderedDict()
176
177     def extendMarkdown(self, md):
178         md.registerExtension(self)
179         inline_image_proc = self.RelatedImageInlineProcessor(
180             IMAGE_LINK_RE, md, self
181         )
182         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
183
184     def get_cid_for_image(self, attrib):
185         msgid = make_msgid()[1:-1]
186         path = attrib["src"]
187         if path.startswith("/"):
188             path = f"file://{path}"
189         self._images[path] = InlineImageInfo(
190             msgid, attrib.get("title", attrib.get("alt"))
191         )
192         return msgid
193
194     def get_images(self):
195         return self._images
196
197
198 def markdown_with_inline_image_support(
199     text, *, extensions=None, extension_configs=None
200 ):
201     inline_image_handler = InlineImageExtension()
202     extensions = extensions or []
203     extensions.append(inline_image_handler)
204     mdwn = markdown.Markdown(
205         extensions=extensions, extension_configs=extension_configs
206     )
207     htmltext = mdwn.convert(text)
208
209     images = inline_image_handler.get_images()
210
211     def replace_image_with_cid(matchobj):
212         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
213             if m in images:
214                 return f"(cid:{images[m].cid}"
215         return matchobj.group(0)
216
217     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
218     return text, htmltext, images
219
220
221 # [ CSS STYLING ] #############################################################
222
223 try:
224     import pynliner
225
226     _PYNLINER = True
227
228 except ImportError:
229     _PYNLINER = False
230
231 try:
232     from pygments.formatters import get_formatter_by_name
233
234     _CODEHILITE_CLASS = "codehilite"
235
236     _PYGMENTS_CSS = get_formatter_by_name(
237         "html", style="default"
238     ).get_style_defs(f".{_CODEHILITE_CLASS}")
239
240 except ImportError:
241     _PYGMENTS_CSS = None
242
243
244 def apply_styling(html, css):
245     return (
246         pynliner.Pynliner()
247         .from_string(html)
248         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
249         .run()
250     )
251
252
253 # [ PARTS GENERATION ] ########################################################
254
255
256 class Part(
257     namedtuple(
258         "Part",
259         ["type", "subtype", "path", "desc", "cid", "orig"],
260         defaults=[None, None, False],
261     )
262 ):
263     def __str__(self):
264         ret = f"<{self.type}/{self.subtype}>"
265         if self.cid:
266             ret = f"{ret} cid:{self.cid}"
267         if self.orig:
268             ret = f"{ret} ORIGINAL"
269         return ret
270
271
272 class Multipart(
273     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
274 ):
275     def __str__(self):
276         return f"<multipart/{self.subtype}> children={len(self.children)}"
277
278     def __hash__(self):
279         return hash(str(self.subtype) + "".join(str(self.children)))
280
281
282 def filewriter_fn(path, content, mode="w", **kwargs):
283     with open(path, mode, **kwargs) as out_f:
284         out_f.write(content)
285
286
287 def collect_inline_images(
288     images, *, tempdir=None, filewriter_fn=filewriter_fn
289 ):
290     relparts = []
291     for path, info in images.items():
292         if path.startswith("cid:"):
293             continue
294
295         data = request.urlopen(path)
296
297         mimetype = data.headers["Content-Type"]
298         ext = mimetypes.guess_extension(mimetype)
299         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
300         path = pathlib.Path(tempfilename[1])
301
302         filewriter_fn(path, data.read(), "w+b")
303
304         relparts.append(
305             Part(
306                 *mimetype.split("/"),
307                 path,
308                 cid=info.cid,
309                 desc=f"Image: {info.desc}",
310             )
311         )
312
313     return relparts
314
315
316 def convert_markdown_to_html(
317     origtext,
318     draftpath,
319     *,
320     related_to_html_only=False,
321     cssfile=None,
322     filewriter_fn=filewriter_fn,
323     tempdir=None,
324     extensions=None,
325     extension_configs=None,
326 ):
327     # TODO extension_configs need to be handled differently
328     extension_configs = extension_configs or {}
329     extension_configs.setdefault("pymdownx.highlight", {})
330     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
331
332     origtext, htmltext, images = markdown_with_inline_image_support(
333         origtext, extensions=extensions, extension_configs=extension_configs
334     )
335
336     filewriter_fn(draftpath, origtext, encoding="utf-8")
337     textpart = Part(
338         "text", "plain", draftpath, "Plain-text version", orig=True
339     )
340
341     htmltext = apply_styling(htmltext, cssfile)
342
343     htmlpath = draftpath.with_suffix(".html")
344     filewriter_fn(
345         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
346     )
347     htmlpart = Part("text", "html", htmlpath, "HTML version")
348
349     imgparts = collect_inline_images(
350         images, tempdir=tempdir, filewriter_fn=filewriter_fn
351     )
352
353     if related_to_html_only:
354         # If there are inline image part, they will be contained within a
355         # multipart/related part along with the HTML part only
356         if imgparts:
357             # replace htmlpart with a multipart/related container of the HTML
358             # parts and the images
359             htmlpart = Multipart(
360                 "relative", [htmlpart] + imgparts, "Group of related content"
361             )
362
363         return Multipart(
364             "alternative", [textpart, htmlpart], "Group of alternative content"
365         )
366
367     else:
368         # If there are inline image part, they will be siblings to the
369         # multipart/alternative tree within a multipart/related part
370         altpart = Multipart(
371             "alternative", [textpart, htmlpart], "Group of alternative content"
372         )
373         if imgparts:
374             return Multipart(
375                 "relative", [altpart] + imgparts, "Group of related content"
376             )
377         else:
378             return altpart
379
380
381 class MIMETreeDFWalker:
382     def __init__(self, *, visitor_fn=None, debug=False):
383         self._visitor_fn = visitor_fn or self._echovisit
384         self._debug = debug
385
386     def _echovisit(self, node, ancestry, debugprint):
387         debugprint(f"node={node} ancestry={ancestry}")
388
389     def walk(self, root, *, visitor_fn=None):
390         """
391         Recursive function to implement a depth-dirst walk of the MIME-tree
392         rooted at `root`.
393         """
394         if isinstance(root, list):
395             if len(root) > 1:
396                 root = Multipart("mixed", children=root)
397             else:
398                 root = root[0]
399
400         self._walk(
401             root,
402             ancestry=[],
403             descendents=[],
404             visitor_fn=visitor_fn or self._visitor_fn,
405         )
406
407     def _walk(self, node, *, ancestry, descendents, visitor_fn):
408         # Let's start by enumerating the parts at the current level. At the
409         # root level, ancestry will be the empty list, and we expect a
410         # multipart/* container at this level. Later, e.g. within a
411         # mutlipart/alternative container, the subtree will just be the
412         # alternative parts, while the top of the ancestry will be the
413         # multipart/alternative container, which we will process after the
414         # following loop.
415
416         lead = f"{'│ '*len(ancestry)}"
417         if isinstance(node, Multipart):
418             self.debugprint(
419                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
420             )
421
422             # Depth-first, so push the current container onto the ancestry
423             # stack, then descend …
424             ancestry.append(node)
425             self.debugprint(lead + "│ " * 2)
426             for child in node.children:
427                 self._walk(
428                     child,
429                     ancestry=ancestry,
430                     descendents=descendents,
431                     visitor_fn=visitor_fn,
432                 )
433             assert ancestry.pop() == node
434             sibling_descendents = descendents
435             descendents.extend(node.children)
436
437         else:
438             self.debugprint(f"{lead}├{node}")
439             sibling_descendents = descendents
440
441         if False and ancestry:
442             self.debugprint(lead[:-1] + " │")
443
444         if visitor_fn:
445             visitor_fn(
446                 node, ancestry, sibling_descendents, debugprint=self.debugprint
447             )
448
449     def debugprint(self, s, **kwargs):
450         if self._debug:
451             print(s, file=sys.stderr, **kwargs)
452
453
454 # [ RUN MODES ] ###############################################################
455
456
457 class MuttCommands:
458     """
459     Stupid class to interface writing out Mutt commands. This is quite a hack
460     to deal with the fact that Mutt runs "push" commands in reverse order, so
461     all of a sudden, things become very complicated when mixing with "real"
462     commands.
463
464     Hence we keep two sets of commands, and one set of pushes. Commands are
465     added to the first until a push is added, after which commands are added to
466     the second set of commands.
467
468     On flush(), the first set is printed, followed by the pushes in reverse,
469     and then the second set is printed. All 3 sets are then cleared.
470     """
471
472     def __init__(self, out_f=sys.stdout, *, debug=False):
473         self._cmd1, self._push, self._cmd2 = [], [], []
474         self._out_f = out_f
475         self._debug = debug
476
477     def cmd(self, s):
478         self.debugprint(s)
479         if self._push:
480             self._cmd2.append(s)
481         else:
482             self._cmd1.append(s)
483
484     def push(self, s):
485         s = s.replace('"', '"')
486         s = f'push "{s}"'
487         self.debugprint(s)
488         self._push.insert(0, s)
489
490     def flush(self):
491         print(
492             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
493         )
494         self._cmd1, self._push, self._cmd2 = [], [], []
495
496     def debugprint(self, s, **kwargs):
497         if self._debug:
498             print(s, file=sys.stderr, **kwargs)
499
500
501 def do_setup(
502     *,
503     out_f=sys.stdout,
504     temppath=None,
505     tempdir=None,
506     debug_commands=False,
507 ):
508     temppath = temppath or pathlib.Path(
509         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
510     )
511     cmds = MuttCommands(out_f, debug=debug_commands)
512
513     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
514
515     cmds.cmd('set my_editor="$editor"')
516     cmds.cmd('set my_edit_headers="$edit_headers"')
517     cmds.cmd(f'set editor="{editor}"')
518     cmds.cmd("unset edit_headers")
519     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
520     cmds.push("<first-entry><edit-file>")
521     cmds.flush()
522
523
524 def do_massage(
525     draft_f,
526     draftpath,
527     cmd_f,
528     *,
529     extensions=None,
530     cssfile=None,
531     converter=convert_markdown_to_html,
532     related_to_html_only=True,
533     only_build=False,
534     max_other_attachments=20,
535     tempdir=None,
536     debug_commands=False,
537     debug_walk=False,
538 ):
539     # Here's the big picture: we're being invoked as the editor on the email
540     # draft, and whatever commands we write to the file given as cmdpath will
541     # be run by the second source command in the macro definition.
542
543     # Let's start by cleaning up what the setup did (see above), i.e. we
544     # restore the $editor and $edit_headers variables, and also unset the
545     # variable used to identify the command file we're currently writing
546     # to.
547     cmds = MuttCommands(cmd_f, debug=debug_commands)
548     cmds.cmd('set editor="$my_editor"')
549     cmds.cmd('set edit_headers="$my_edit_headers"')
550     cmds.cmd("unset my_editor")
551     cmds.cmd("unset my_edit_headers")
552
553     # let's flush those commands, as there'll be a lot of pushes from now
554     # on, which need to be run in reverse order
555     cmds.flush()
556
557     extensions = extensions.split(",") if extensions else []
558     tree = converter(
559         draft_f.read(),
560         draftpath,
561         cssfile=cssfile,
562         related_to_html_only=related_to_html_only,
563         tempdir=tempdir,
564         extensions=extensions,
565     )
566
567     mimetree = MIMETreeDFWalker(debug=debug_walk)
568
569     state = dict(pos=1, tags={}, parts=1)
570
571     def visitor_fn(item, ancestry, descendents, *, debugprint=None):
572         """
573         Visitor function called for every node (part) of the MIME tree,
574         depth-first, and responsible for telling NeoMutt how to assemble
575         the tree.
576         """
577         KILL_LINE = r"\Ca\Ck"
578
579         if isinstance(item, Part):
580             # We've hit a leaf-node, i.e. an alternative or a related part
581             # with actual content.
582
583             # Let's add the part
584             if item.orig:
585                 # The original source already exists in the NeoMutt tree, but
586                 # the underlying file may have been modified, so we need to
587                 # update the encoding, but that's it:
588                 cmds.push("<first-entry>")
589                 cmds.push("<update-encoding>")
590
591                 # We really just need to be able to assume that at this point,
592                 # NeoMutt is at position 1, and that we've processed only this
593                 # part so far. Nevermind about actual attachments, we can
594                 # safely ignore those as they stay at the end.
595                 assert state["pos"] == 1
596                 assert state["parts"] == 1
597             else:
598                 # … whereas all other parts need to be added, and they're all
599                 # considered to be temporary and inline:
600                 cmds.push(f"<attach-file>{item.path}<enter>")
601                 cmds.push("<toggle-unlink><toggle-disposition>")
602
603                 # This added a part at the end of the list of parts, and that's
604                 # just how many parts we've seen so far, so it's position in
605                 # the NeoMutt compose list is the count of parts
606                 state["parts"] += 1
607                 state["pos"] = state["parts"]
608
609             # If the item (including the original) comes with additional
610             # information, then we might just as well update the NeoMutt
611             # tree now:
612             if item.cid:
613                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
614
615             # Now for the biggest hack in this script, which is to handle
616             # attachments, such as PDFs, that aren't related or alternatives.
617             # The problem is that when we add an inline image, it always gets
618             # appended to the list, i.e. inserted *after* other attachments.
619             # Since we don't know the number of attachments, we also cannot
620             # infer the postition of the new attachment. Therefore, we bubble
621             # it all the way to the top, only to then move it down again:
622             if state["pos"] > 1:  # skip for the first part
623                 for i in range(max_other_attachments):
624                     # could use any number here, but has to be larger than the
625                     # number of possible attachments. The performance
626                     # difference of using a high number is negligible.
627                     # Bubble up the new part
628                     cmds.push(f"<move-up>")
629
630                 # As we push the part to the right position in the list (i.e.
631                 # the last of the subset of attachments this script added), we
632                 # must handle the situation that subtrees are skipped by
633                 # NeoMutt. Hence, the actual number of positions to move down
634                 # is decremented by the number of descendents so far
635                 # encountered.
636                 for i in range(1, state["pos"] - len(descendents)):
637                     cmds.push(f"<move-down>")
638
639         elif isinstance(item, Multipart):
640             # This node has children, but we already visited them (see
641             # above). The tags dictionary of State should contain a list of
642             # their positions in the NeoMutt compose window, so iterate those
643             # and tag the parts there:
644             n_tags = len(state["tags"][item])
645             for tag in state["tags"][item]:
646                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
647
648             if item.subtype == "alternative":
649                 cmds.push("<group-alternatives>")
650             elif item.subtype in ("relative", "related"):
651                 cmds.push("<group-related>")
652             elif item.subtype == "multilingual":
653                 cmds.push("<group-multilingual>")
654             else:
655                 raise NotImplementedError(
656                     f"Handling of multipart/{item.subtype} is not implemented"
657                 )
658
659             state["pos"] -= n_tags - 1
660             state["parts"] += 1
661
662         else:
663             # We should never get here
664             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
665
666         # If the item has a description, we might just as well add it
667         if item.desc:
668             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
669
670         if ancestry:
671             # If there's an ancestry, record the current (assumed) position in
672             # the NeoMutt compose window as needed-to-tag by our direct parent
673             # (i.e. the last item of the ancestry)
674             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
675
676             lead = "│ " * (len(ancestry) + 1) + "* "
677             debugprint(
678                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
679                 f"{lead}descendents={[d.subtype for d in descendents]}\n"
680                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
681                 f"{lead}pos={state['pos']}, parts={state['parts']}"
682             )
683
684     # -----------------
685     # End of visitor_fn
686
687     # Let's walk the tree and visit every node with our fancy visitor
688     # function
689     mimetree.walk(tree, visitor_fn=visitor_fn)
690
691     if not only_build:
692         cmds.push("<send-message>")
693
694     # Finally, cleanup. Since we're responsible for removing the temporary
695     # file, how's this for a little hack?
696     try:
697         filename = cmd_f.name
698     except AttributeError:
699         filename = "pytest_internal_file"
700     cmds.cmd(f"source 'rm -f {filename}|'")
701     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
702     cmds.flush()
703
704
705 # [ CLI ENTRY ] ###############################################################
706
707 if __name__ == "__main__":
708     args = parse_cli_args()
709
710     if args.mode is None:
711         do_setup(
712             tempdir=args.tempdir,
713             debug_commands=args.debug_commands,
714         )
715
716     elif args.mode == "massage":
717         with open(args.MAILDRAFT, "r") as draft_f, open(
718             args.cmdpath, "w"
719         ) as cmd_f:
720             do_massage(
721                 draft_f,
722                 args.MAILDRAFT,
723                 cmd_f,
724                 extensions=args.extensions,
725                 cssfile=args.css_file,
726                 related_to_html_only=args.related_to_html_only,
727                 max_other_attachments=args.max_number_other_attachments,
728                 only_build=args.only_build,
729                 tempdir=args.tempdir,
730                 debug_commands=args.debug_commands,
731                 debug_walk=args.debug_walk,
732             )
733
734
735 # [ TESTS ] ###################################################################
736
737 try:
738     import pytest
739     from io import StringIO
740
741     class Tests:
742         @pytest.fixture
743         def const1(self):
744             return "CONSTANT STRING 1"
745
746         @pytest.fixture
747         def const2(self):
748             return "CONSTANT STRING 2"
749
750         # NOTE: tests using the capsys fixture must specify sys.stdout to the
751         # functions they call, else old stdout is used and not captured
752
753         def test_MuttCommands_cmd(self, const1, const2, capsys):
754             "Assert order of commands"
755             cmds = MuttCommands(out_f=sys.stdout)
756             cmds.cmd(const1)
757             cmds.cmd(const2)
758             cmds.flush()
759             captured = capsys.readouterr()
760             assert captured.out == "\n".join((const1, const2, ""))
761
762         def test_MuttCommands_push(self, const1, const2, capsys):
763             "Assert reverse order of pushes"
764             cmds = MuttCommands(out_f=sys.stdout)
765             cmds.push(const1)
766             cmds.push(const2)
767             cmds.flush()
768             captured = capsys.readouterr()
769             assert (
770                 captured.out
771                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
772             )
773
774         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
775             "Assert reverse order of pushes"
776             cmds = MuttCommands(out_f=sys.stdout)
777             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
778             for i in range(2):
779                 cmds.cmd(lines[4 * i + 0])
780                 cmds.cmd(lines[4 * i + 1])
781                 cmds.push(lines[4 * i + 2])
782                 cmds.push(lines[4 * i + 3])
783             cmds.flush()
784
785             captured = capsys.readouterr()
786             lines_out = captured.out.splitlines()
787             assert lines[0] in lines_out[0]
788             assert lines[1] in lines_out[1]
789             assert lines[7] in lines_out[2]
790             assert lines[6] in lines_out[3]
791             assert lines[3] in lines_out[4]
792             assert lines[2] in lines_out[5]
793             assert lines[4] in lines_out[6]
794             assert lines[5] in lines_out[7]
795
796         @pytest.fixture
797         def mime_tree_related_to_alternative(self):
798             return Multipart(
799                 "relative",
800                 children=[
801                     Multipart(
802                         "alternative",
803                         children=[
804                             Part(
805                                 "text",
806                                 "plain",
807                                 "part.txt",
808                                 desc="Plain",
809                                 orig=True,
810                             ),
811                             Part("text", "html", "part.html", desc="HTML"),
812                         ],
813                         desc="Alternative",
814                     ),
815                     Part(
816                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
817                     ),
818                 ],
819                 desc="Related",
820             )
821
822         @pytest.fixture
823         def mime_tree_related_to_html(self):
824             return Multipart(
825                 "alternative",
826                 children=[
827                     Part(
828                         "text",
829                         "plain",
830                         "part.txt",
831                         desc="Plain",
832                         orig=True,
833                     ),
834                     Multipart(
835                         "relative",
836                         children=[
837                             Part("text", "html", "part.html", desc="HTML"),
838                             Part(
839                                 "text",
840                                 "png",
841                                 "logo.png",
842                                 cid="logo.png",
843                                 desc="Logo",
844                             ),
845                         ],
846                         desc="Related",
847                     ),
848                 ],
849                 desc="Alternative",
850             )
851
852         def test_MIMETreeDFWalker_depth_first_walk(
853             self, mime_tree_related_to_alternative
854         ):
855             mimetree = MIMETreeDFWalker()
856
857             items = []
858
859             def visitor_fn(item, ancestry, descendents, debugprint):
860                 items.append((item, len(ancestry), len(descendents)))
861
862             mimetree.walk(
863                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
864             )
865             assert len(items) == 5
866             assert items[0][0].subtype == "plain"
867             assert items[0][1] == 2
868             assert items[0][2] == 0
869             assert items[1][0].subtype == "html"
870             assert items[1][1] == 2
871             assert items[1][2] == 0
872             assert items[2][0].subtype == "alternative"
873             assert items[2][1] == 1
874             assert items[2][2] == 2
875             assert items[3][0].subtype == "png"
876             assert items[3][1] == 1
877             assert items[3][2] == 2
878             assert items[4][0].subtype == "relative"
879             assert items[4][1] == 0
880             assert items[4][2] == 4
881
882         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
883             mimetree = MIMETreeDFWalker()
884             items = []
885
886             def visitor_fn(item, ancestry, descendents, debugprint):
887                 items.append(item)
888
889             p = Part("text", "plain", const1)
890             mimetree.walk([p], visitor_fn=visitor_fn)
891             assert items[-1].subtype == "plain"
892             mimetree.walk([p, p], visitor_fn=visitor_fn)
893             assert items[-1].subtype == "mixed"
894
895         def test_MIMETreeDFWalker_visitor_in_constructor(
896             self, mime_tree_related_to_alternative
897         ):
898             items = []
899
900             def visitor_fn(item, ancestry, descendents, debugprint):
901                 items.append(item)
902
903             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
904             mimetree.walk(mime_tree_related_to_alternative)
905             assert len(items) == 5
906
907         @pytest.fixture
908         def string_io(self, const1, text=None):
909             return StringIO(text or const1)
910
911         def test_do_massage_basic(self, const1, string_io, capsys):
912             def converter(
913                 drafttext,
914                 draftpath,
915                 cssfile,
916                 related_to_html_only,
917                 extensions,
918                 tempdir,
919             ):
920                 return Part("text", "plain", draftpath, orig=True)
921
922             do_massage(
923                 draft_f=string_io,
924                 draftpath=const1,
925                 cmd_f=sys.stdout,
926                 converter=converter,
927             )
928
929             captured = capsys.readouterr()
930             lines = captured.out.splitlines()
931             assert '="$my_editor"' in lines.pop(0)
932             assert '="$my_edit_headers"' in lines.pop(0)
933             assert "unset my_editor" == lines.pop(0)
934             assert "unset my_edit_headers" == lines.pop(0)
935             assert "send-message" in lines.pop(0)
936             assert "update-encoding" in lines.pop(0)
937             assert "first-entry" in lines.pop(0)
938             assert "source 'rm -f " in lines.pop(0)
939             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
940
941         def test_do_massage_fulltree(
942             self, string_io, const1, mime_tree_related_to_alternative, capsys
943         ):
944             def converter(
945                 drafttext,
946                 draftpath,
947                 cssfile,
948                 related_to_html_only,
949                 extensions,
950                 tempdir,
951             ):
952                 return mime_tree_related_to_alternative
953
954             max_attachments = 5
955             do_massage(
956                 draft_f=string_io,
957                 draftpath=const1,
958                 cmd_f=sys.stdout,
959                 max_other_attachments=max_attachments,
960                 converter=converter,
961             )
962
963             captured = capsys.readouterr()
964             lines = captured.out.splitlines()[4:-2]
965             assert "first-entry" in lines.pop()
966             assert "update-encoding" in lines.pop()
967             assert "Plain" in lines.pop()
968             assert "part.html" in lines.pop()
969             assert "toggle-unlink" in lines.pop()
970             for i in range(max_attachments):
971                 assert "move-up" in lines.pop()
972             assert "move-down" in lines.pop()
973             assert "HTML" in lines.pop()
974             assert "jump>1" in lines.pop()
975             assert "jump>2" in lines.pop()
976             assert "group-alternatives" in lines.pop()
977             assert "Alternative" in lines.pop()
978             assert "logo.png" in lines.pop()
979             assert "toggle-unlink" in lines.pop()
980             assert "content-id" in lines.pop()
981             for i in range(max_attachments):
982                 assert "move-up" in lines.pop()
983             assert "move-down" in lines.pop()
984             assert "Logo" in lines.pop()
985             assert "jump>1" in lines.pop()
986             assert "jump>4" in lines.pop()
987             assert "group-related" in lines.pop()
988             assert "Related" in lines.pop()
989             assert "send-message" in lines.pop()
990             assert len(lines) == 0
991
992         @pytest.fixture
993         def fake_filewriter(self):
994             class FileWriter:
995                 def __init__(self):
996                     self._writes = []
997
998                 def __call__(self, path, content, mode="w", **kwargs):
999                     self._writes.append((path, content))
1000
1001                 def pop(self, index=-1):
1002                     return self._writes.pop(index)
1003
1004             return FileWriter()
1005
1006         @pytest.fixture
1007         def markdown_non_converter(self, const1, const2):
1008             return lambda s, text: f"{const1}{text}{const2}"
1009
1010         def test_converter_tree_basic(self, const1, const2, fake_filewriter):
1011             path = pathlib.Path(const2)
1012             tree = convert_markdown_to_html(
1013                 const1, path, filewriter_fn=fake_filewriter
1014             )
1015
1016             assert tree.subtype == "alternative"
1017             assert len(tree.children) == 2
1018             assert tree.children[0].subtype == "plain"
1019             assert tree.children[0].path == path
1020             assert tree.children[0].orig
1021             assert tree.children[1].subtype == "html"
1022             assert tree.children[1].path == path.with_suffix(".html")
1023
1024         def test_converter_writes(
1025             self,
1026             const1,
1027             const2,
1028             fake_filewriter,
1029             monkeypatch,
1030             markdown_non_converter,
1031         ):
1032             path = pathlib.Path(const2)
1033
1034             with monkeypatch.context() as m:
1035                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
1036                 convert_markdown_to_html(
1037                     const1, path, filewriter_fn=fake_filewriter
1038                 )
1039
1040             assert (path, const1) == fake_filewriter.pop(0)
1041             assert (
1042                 path.with_suffix(".html"),
1043                 markdown_non_converter(None, const1),
1044             ) == fake_filewriter.pop(0)
1045
1046         def test_markdown_inline_image_processor(self):
1047             imgpath1 = "file:/path/to/image.png"
1048             imgpath2 = "file:///path/to/image.png?url=params"
1049             imgpath3 = "/path/to/image.png"
1050             text = f"""![inline local image]({imgpath1})
1051                        ![image inlined
1052                          with newline]({imgpath2})
1053                        ![image local path]({imgpath3})"""
1054             text, html, images = markdown_with_inline_image_support(text)
1055
1056             # local paths have been normalised to URLs:
1057             imgpath3 = f"file://{imgpath3}"
1058
1059             assert 'src="cid:' in html
1060             assert "](cid:" in text
1061             assert len(images) == 3
1062             assert imgpath1 in images
1063             assert imgpath2 in images
1064             assert imgpath3 in images
1065             assert images[imgpath1].cid != images[imgpath2].cid
1066             assert images[imgpath1].cid != images[imgpath3].cid
1067             assert images[imgpath2].cid != images[imgpath3].cid
1068
1069         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1070             imgpath = "file:///path/to/image.png"
1071             text = f'![inline local image]({imgpath} "{const1}")'
1072             text, html, images = markdown_with_inline_image_support(text)
1073             assert images[imgpath].desc == const1
1074
1075         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1076             imgpath = "file:///path/to/image.png"
1077             text = f"![{const1}]({imgpath})"
1078             text, html, images = markdown_with_inline_image_support(text)
1079             assert images[imgpath].desc == const1
1080
1081         def test_markdown_inline_image_processor_title_over_alt_desc(
1082             self, const1, const2
1083         ):
1084             imgpath = "file:///path/to/image.png"
1085             text = f'![{const1}]({imgpath} "{const2}")'
1086             text, html, images = markdown_with_inline_image_support(text)
1087             assert images[imgpath].desc == const2
1088
1089         def test_markdown_inline_image_not_external(self):
1090             imgpath = "https://path/to/image.png"
1091             text = f"![inline image]({imgpath})"
1092             text, html, images = markdown_with_inline_image_support(text)
1093
1094             assert 'src="cid:' not in html
1095             assert "](cid:" not in text
1096             assert len(images) == 0
1097
1098         def test_markdown_inline_image_local_file(self):
1099             imgpath = "/path/to/image.png"
1100             text = f"![inline image]({imgpath})"
1101             text, html, images = markdown_with_inline_image_support(text)
1102
1103             for k, v in images.items():
1104                 assert k == f"file://{imgpath}"
1105                 break
1106
1107         @pytest.fixture
1108         def test_png(self):
1109             return (
1110                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1111                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1112             )
1113
1114         def test_markdown_inline_image_processor_base64(self, test_png):
1115             text = f"![1px white inlined]({test_png})"
1116             text, html, images = markdown_with_inline_image_support(text)
1117
1118             assert 'src="cid:' in html
1119             assert "](cid:" in text
1120             assert len(images) == 1
1121             assert test_png in images
1122
1123         def test_converter_tree_inline_image_base64(
1124             self, test_png, const1, fake_filewriter
1125         ):
1126             text = f"![inline base64 image]({test_png})"
1127             path = pathlib.Path(const1)
1128             tree = convert_markdown_to_html(
1129                 text,
1130                 path,
1131                 filewriter_fn=fake_filewriter,
1132                 related_to_html_only=False,
1133             )
1134             assert tree.subtype == "relative"
1135             assert tree.children[0].subtype == "alternative"
1136             assert tree.children[1].subtype == "png"
1137             written = fake_filewriter.pop()
1138             assert tree.children[1].path == written[0]
1139             assert written[1] == request.urlopen(test_png).read()
1140
1141         def test_converter_tree_inline_image_base64_related_to_html(
1142             self, test_png, const1, fake_filewriter
1143         ):
1144             text = f"![inline base64 image]({test_png})"
1145             path = pathlib.Path(const1)
1146             tree = convert_markdown_to_html(
1147                 text,
1148                 path,
1149                 filewriter_fn=fake_filewriter,
1150                 related_to_html_only=True,
1151             )
1152             assert tree.subtype == "alternative"
1153             assert tree.children[1].subtype == "relative"
1154             assert tree.children[1].children[1].subtype == "png"
1155             written = fake_filewriter.pop()
1156             assert tree.children[1].children[1].path == written[0]
1157             assert written[1] == request.urlopen(test_png).read()
1158
1159         def test_converter_tree_inline_image_cid(
1160             self, const1, fake_filewriter
1161         ):
1162             text = f"![inline base64 image](cid:{const1})"
1163             path = pathlib.Path(const1)
1164             tree = convert_markdown_to_html(
1165                 text,
1166                 path,
1167                 filewriter_fn=fake_filewriter,
1168                 related_to_html_only=False,
1169             )
1170             assert len(tree.children) == 2
1171             assert tree.children[0].cid != const1
1172             assert tree.children[0].type != "image"
1173             assert tree.children[1].cid != const1
1174             assert tree.children[1].type != "image"
1175
1176         def test_inline_image_collection(
1177             self, test_png, const1, const2, fake_filewriter
1178         ):
1179             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1180             relparts = collect_inline_images(
1181                 test_images, filewriter_fn=fake_filewriter
1182             )
1183
1184             written = fake_filewriter.pop()
1185             assert b"PNG" in written[1]
1186
1187             assert relparts[0].subtype == "png"
1188             assert relparts[0].path == written[0]
1189             assert relparts[0].cid == const1
1190             assert relparts[0].desc.endswith(const2)
1191
1192         def test_apply_stylesheet(self):
1193             if _PYNLINER:
1194                 html = "<p>Hello, world!</p>"
1195                 css = "p { color:red }"
1196                 out = apply_styling(html, css)
1197                 assert 'p style="color' in out
1198
1199         def test_apply_stylesheet_pygments(self):
1200             if _PYGMENTS_CSS:
1201                 html = (
1202                     f'<div class="{_CODEHILITE_CLASS}">'
1203                     "<pre>def foo():\n    return</pre></div>"
1204                 )
1205                 out = apply_styling(html, _PYGMENTS_CSS)
1206                 assert f'{_CODEHILITE_CLASS}" style="' in out
1207
1208         def test_mime_tree_relative_within_alternative(
1209             self, string_io, const1, capsys, mime_tree_related_to_html
1210         ):
1211             def converter(
1212                 drafttext,
1213                 draftpath,
1214                 cssfile,
1215                 related_to_html_only,
1216                 extensions,
1217                 tempdir,
1218             ):
1219                 return mime_tree_related_to_html
1220
1221             do_massage(
1222                 draft_f=string_io,
1223                 draftpath=const1,
1224                 cmd_f=sys.stdout,
1225                 converter=converter,
1226             )
1227
1228             captured = capsys.readouterr()
1229             lines = captured.out.splitlines()[4:-2]
1230             assert "first-entry" in lines.pop()
1231             assert "update-encoding" in lines.pop()
1232             assert "Plain" in lines.pop()
1233             assert "part.html" in lines.pop()
1234             assert "toggle-unlink" in lines.pop()
1235             assert "move-up" in lines.pop()
1236             while True:
1237                 top = lines.pop()
1238                 if "move-up" not in top:
1239                     break
1240             assert "move-down" in top
1241             assert "HTML" in lines.pop()
1242             assert "logo.png" in lines.pop()
1243             assert "toggle-unlink" in lines.pop()
1244             assert "content-id" in lines.pop()
1245             assert "move-up" in lines.pop()
1246             while True:
1247                 top = lines.pop()
1248                 if "move-up" not in top:
1249                     break
1250             assert "move-down" in top
1251             assert "move-down" in lines.pop()
1252             assert "Logo" in lines.pop()
1253             assert "jump>2" in lines.pop()
1254             assert "jump>3" in lines.pop()
1255             assert "group-related" in lines.pop()
1256             assert "Related" in lines.pop()
1257             assert "jump>1" in lines.pop()
1258             assert "jump>2" in lines.pop()
1259             assert "group-alternative" in lines.pop()
1260             assert "Alternative" in lines.pop()
1261             assert "send-message" in lines.pop()
1262             assert len(lines) == 0
1263
1264         def test_mime_tree_nested_trees_does_not_break_positioning(
1265             self, string_io, const1, capsys
1266         ):
1267             def converter(
1268                 drafttext,
1269                 draftpath,
1270                 cssfile,
1271                 related_to_html_only,
1272                 extensions,
1273                 tempdir,
1274             ):
1275                 return Multipart(
1276                     "relative",
1277                     children=[
1278                         Multipart(
1279                             "alternative",
1280                             children=[
1281                                 Part(
1282                                     "text",
1283                                     "plain",
1284                                     "part.txt",
1285                                     desc="Plain",
1286                                     orig=True,
1287                                 ),
1288                                 Multipart(
1289                                     "alternative",
1290                                     children=[
1291                                         Part(
1292                                             "text",
1293                                             "plain",
1294                                             "part.txt",
1295                                             desc="Nested plain",
1296                                         ),
1297                                         Part(
1298                                             "text",
1299                                             "html",
1300                                             "part.html",
1301                                             desc="Nested HTML",
1302                                         ),
1303                                     ],
1304                                     desc="Nested alternative",
1305                                 ),
1306                             ],
1307                             desc="Alternative",
1308                         ),
1309                         Part(
1310                             "text",
1311                             "png",
1312                             "logo.png",
1313                             cid="logo.png",
1314                             desc="Logo",
1315                         ),
1316                     ],
1317                     desc="Related",
1318                 )
1319
1320             do_massage(
1321                 draft_f=string_io,
1322                 draftpath=const1,
1323                 cmd_f=sys.stdout,
1324                 converter=converter,
1325             )
1326
1327             captured = capsys.readouterr()
1328             lines = captured.out.splitlines()
1329             while not "logo.png" in lines.pop():
1330                 pass
1331             lines.pop()
1332             assert "content-id" in lines.pop()
1333             assert "move-up" in lines.pop()
1334             while True:
1335                 top = lines.pop()
1336                 if "move-up" not in top:
1337                     break
1338             assert "move-down" in top
1339             # Due to the nested trees, the number of descendents of the sibling
1340             # actually needs to be considered, not just the nieces. So to move
1341             # from position 1 to position 6, it only needs one <move-down>
1342             # because that jumps over the entire sibling tree. Thus what
1343             # follows next must not be another <move-down>
1344             assert "Logo" in lines.pop()
1345
1346 except ImportError:
1347     pass