]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

buildmimetree.py: handle counting of extended family in presence of other attachments
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
10 #     macro compose B "\
11 #       <enter-command> source '$my_confdir/buildmimetree.py \
12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
14 #     " "Convert message into a modern MIME tree with inline images"
15 #
16 #     (Yes, we need to call source twice, as mutt only starts to process output
17 #     from a source command when the command exits, and since we need to react
18 #     to the output, we need to be invoked again, using a $my_ variable to pass
19 #     information)
20 #
21 # Requirements:
22 #   - python3
23 #   - python3-markdown
24 # Optional:
25 #   - pytest
26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
27 #   - Pygments, then syntax highlighting for fenced code is enabled
28 #
29 # Latest version:
30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
31 #
32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
33 # Released under the GPL-2+ licence, just like Mutt itself.
34 #
35
36 import sys
37 import pathlib
38 import markdown
39 import tempfile
40 import argparse
41 import re
42 import mimetypes
43 from collections import namedtuple, OrderedDict
44 from markdown.extensions import Extension
45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
46 from email.utils import make_msgid
47 from urllib import request
48
49
50 def parse_cli_args(*args, **kwargs):
51     parser = argparse.ArgumentParser(
52         description=(
53             "NeoMutt helper to turn text/markdown email parts "
54             "into full-fledged MIME trees"
55         )
56     )
57     parser.epilog = (
58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
59         "Released under the MIT licence"
60     )
61
62     parser.add_argument(
63         "--extensions",
64         type=str,
65         default="",
66         help="Markdown extension to use (comma-separated list)",
67     )
68
69     if _PYNLINER:
70         parser.add_argument(
71             "--css-file",
72             type=str,
73             default="",
74             help="CSS file to merge with the final HTML",
75         )
76     else:
77         parser.set_defaults(css_file=None)
78
79     parser.add_argument(
80         "--related-to-html-only",
81         action="store_true",
82         help="Make related content be sibling to HTML parts only",
83     )
84
85     def positive_integer(value):
86         try:
87             if int(value) > 0:
88                 return int(value)
89
90         except ValueError:
91             pass
92
93         raise ValueError(f"Must be a positive integer")
94
95     parser.add_argument(
96         "--max-number-other-attachments",
97         type=positive_integer,
98         help="Make related content be sibling to HTML parts only",
99     )
100
101     parser.add_argument(
102         "--only-build",
103         action="store_true",
104         help="Only build, don't send the message",
105     )
106
107     parser.add_argument(
108         "--tempdir",
109         default=None,
110         help="Specify temporary directory to use for attachments",
111     )
112
113     parser.add_argument(
114         "--debug-commands",
115         action="store_true",
116         help="Turn on debug logging of commands generated to stderr",
117     )
118
119     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
120     massage_p = subp.add_parser(
121         "massage", help="Massaging phase (internal use)"
122     )
123
124     massage_p.add_argument(
125         "--write-commands-to",
126         "-o",
127         metavar="PATH",
128         dest="cmdpath",
129         required=True,
130         help="Temporary file path to write commands to",
131     )
132
133     massage_p.add_argument(
134         "--debug-walk",
135         action="store_true",
136         help="Turn on debugging to stderr of the MIME tree walk",
137     )
138
139     massage_p.add_argument(
140         "MAILDRAFT",
141         nargs="?",
142         help="If provided, the script is invoked as editor on the mail draft",
143     )
144
145     return parser.parse_args(*args, **kwargs)
146
147
148 # [ MARKDOWN WRAPPING ] #######################################################
149
150
151 InlineImageInfo = namedtuple(
152     "InlineImageInfo", ["cid", "desc"], defaults=[None]
153 )
154
155
156 class InlineImageExtension(Extension):
157     class RelatedImageInlineProcessor(ImageInlineProcessor):
158         def __init__(self, re, md, ext):
159             super().__init__(re, md)
160             self._ext = ext
161
162         def handleMatch(self, m, data):
163             el, start, end = super().handleMatch(m, data)
164             if "src" in el.attrib:
165                 src = el.attrib["src"]
166                 if "://" not in src or src.startswith("file://"):
167                     # We only inline local content
168                     cid = self._ext.get_cid_for_image(el.attrib)
169                     el.attrib["src"] = f"cid:{cid}"
170             return el, start, end
171
172     def __init__(self):
173         super().__init__()
174         self._images = OrderedDict()
175
176     def extendMarkdown(self, md):
177         md.registerExtension(self)
178         inline_image_proc = self.RelatedImageInlineProcessor(
179             IMAGE_LINK_RE, md, self
180         )
181         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
182
183     def get_cid_for_image(self, attrib):
184         msgid = make_msgid()[1:-1]
185         path = attrib["src"]
186         if path.startswith("/"):
187             path = f"file://{path}"
188         self._images[path] = InlineImageInfo(
189             msgid, attrib.get("title", attrib.get("alt"))
190         )
191         return msgid
192
193     def get_images(self):
194         return self._images
195
196
197 def markdown_with_inline_image_support(
198     text, *, extensions=None, extension_configs=None
199 ):
200     inline_image_handler = InlineImageExtension()
201     extensions = extensions or []
202     extensions.append(inline_image_handler)
203     mdwn = markdown.Markdown(
204         extensions=extensions, extension_configs=extension_configs
205     )
206     htmltext = mdwn.convert(text)
207
208     images = inline_image_handler.get_images()
209
210     def replace_image_with_cid(matchobj):
211         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
212             if m in images:
213                 return f"(cid:{images[m].cid}"
214         return matchobj.group(0)
215
216     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
217     return text, htmltext, images
218
219
220 # [ CSS STYLING ] #############################################################
221
222 try:
223     import pynliner
224
225     _PYNLINER = True
226
227 except ImportError:
228     _PYNLINER = False
229
230 try:
231     from pygments.formatters import get_formatter_by_name
232
233     _CODEHILITE_CLASS = "codehilite"
234
235     _PYGMENTS_CSS = get_formatter_by_name(
236         "html", style="default"
237     ).get_style_defs(f".{_CODEHILITE_CLASS}")
238
239 except ImportError:
240     _PYGMENTS_CSS = None
241
242
243 def apply_styling(html, css):
244     return (
245         pynliner.Pynliner()
246         .from_string(html)
247         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
248         .run()
249     )
250
251
252 # [ PARTS GENERATION ] ########################################################
253
254
255 class Part(
256     namedtuple(
257         "Part",
258         ["type", "subtype", "path", "desc", "cid", "orig"],
259         defaults=[None, None, False],
260     )
261 ):
262     def __str__(self):
263         ret = f"<{self.type}/{self.subtype}>"
264         if self.cid:
265             ret = f"{ret} cid:{self.cid}"
266         if self.orig:
267             ret = f"{ret} ORIGINAL"
268         return ret
269
270
271 class Multipart(
272     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
273 ):
274     def __str__(self):
275         return f"<multipart/{self.subtype}> children={len(self.children)}"
276
277     def __hash__(self):
278         return hash(str(self.subtype) + "".join(str(self.children)))
279
280
281 def filewriter_fn(path, content, mode="w", **kwargs):
282     with open(path, mode, **kwargs) as out_f:
283         out_f.write(content)
284
285
286 def collect_inline_images(
287     images, *, tempdir=None, filewriter_fn=filewriter_fn
288 ):
289     relparts = []
290     for path, info in images.items():
291         if path.startswith("cid:"):
292             continue
293
294         data = request.urlopen(path)
295
296         mimetype = data.headers["Content-Type"]
297         ext = mimetypes.guess_extension(mimetype)
298         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
299         path = pathlib.Path(tempfilename[1])
300
301         filewriter_fn(path, data.read(), "w+b")
302
303         relparts.append(
304             Part(
305                 *mimetype.split("/"),
306                 path,
307                 cid=info.cid,
308                 desc=f"Image: {info.desc}",
309             )
310         )
311
312     return relparts
313
314
315 def convert_markdown_to_html(
316     origtext,
317     draftpath,
318     *,
319     related_to_html_only=False,
320     cssfile=None,
321     filewriter_fn=filewriter_fn,
322     tempdir=None,
323     extensions=None,
324     extension_configs=None,
325 ):
326     # TODO extension_configs need to be handled differently
327     extension_configs = extension_configs or {}
328     extension_configs.setdefault("pymdownx.highlight", {})
329     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
330
331     origtext, htmltext, images = markdown_with_inline_image_support(
332         origtext, extensions=extensions, extension_configs=extension_configs
333     )
334
335     filewriter_fn(draftpath, origtext, encoding="utf-8")
336     textpart = Part(
337         "text", "plain", draftpath, "Plain-text version", orig=True
338     )
339
340     htmltext = apply_styling(htmltext, cssfile)
341
342     htmlpath = draftpath.with_suffix(".html")
343     filewriter_fn(
344         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
345     )
346     htmlpart = Part("text", "html", htmlpath, "HTML version")
347
348     imgparts = collect_inline_images(
349         images, tempdir=tempdir, filewriter_fn=filewriter_fn
350     )
351
352     if related_to_html_only:
353         # If there are inline image part, they will be contained within a
354         # multipart/related part along with the HTML part only
355         if imgparts:
356             # replace htmlpart with a multipart/related container of the HTML
357             # parts and the images
358             htmlpart = Multipart(
359                 "relative", [htmlpart] + imgparts, "Group of related content"
360             )
361
362         return Multipart(
363             "alternative", [textpart, htmlpart], "Group of alternative content"
364         )
365
366     else:
367         # If there are inline image part, they will be siblings to the
368         # multipart/alternative tree within a multipart/related part
369         altpart = Multipart(
370             "alternative", [textpart, htmlpart], "Group of alternative content"
371         )
372         if imgparts:
373             return Multipart(
374                 "relative", [altpart] + imgparts, "Group of related content"
375             )
376         else:
377             return altpart
378
379
380 class MIMETreeDFWalker:
381     def __init__(self, *, visitor_fn=None, debug=False):
382         self._visitor_fn = visitor_fn or self._echovisit
383         self._debug = debug
384
385     def _echovisit(self, node, ancestry, debugprint):
386         debugprint(f"node={node} ancestry={ancestry}")
387
388     def walk(self, root, *, visitor_fn=None):
389         """
390         Recursive function to implement a depth-dirst walk of the MIME-tree
391         rooted at `root`.
392         """
393         if isinstance(root, list):
394             if len(root) > 1:
395                 root = Multipart("mixed", children=root)
396             else:
397                 root = root[0]
398
399         self._walk(
400             root,
401             ancestry=[],
402             descendents=[],
403             visitor_fn=visitor_fn or self._visitor_fn,
404         )
405
406     def _walk(self, node, *, ancestry, descendents, visitor_fn):
407         # Let's start by enumerating the parts at the current level. At the
408         # root level, ancestry will be the empty list, and we expect a
409         # multipart/* container at this level. Later, e.g. within a
410         # mutlipart/alternative container, the subtree will just be the
411         # alternative parts, while the top of the ancestry will be the
412         # multipart/alternative container, which we will process after the
413         # following loop.
414
415         lead = f"{'│ '*len(ancestry)}"
416         if isinstance(node, Multipart):
417             self.debugprint(
418                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
419             )
420
421             # Depth-first, so push the current container onto the ancestry
422             # stack, then descend …
423             ancestry.append(node)
424             self.debugprint(lead + "│ " * 2)
425             for child in node.children:
426                 self._walk(
427                     child,
428                     ancestry=ancestry,
429                     descendents=descendents,
430                     visitor_fn=visitor_fn,
431                 )
432             assert ancestry.pop() == node
433             sibling_descendents = descendents
434             descendents.extend(node.children)
435
436         else:
437             self.debugprint(f"{lead}├{node}")
438             sibling_descendents = descendents
439
440         if False and ancestry:
441             self.debugprint(lead[:-1] + " │")
442
443         if visitor_fn:
444             visitor_fn(
445                 node, ancestry, sibling_descendents, debugprint=self.debugprint
446             )
447
448     def debugprint(self, s, **kwargs):
449         if self._debug:
450             print(s, file=sys.stderr, **kwargs)
451
452
453 # [ RUN MODES ] ###############################################################
454
455
456 class MuttCommands:
457     """
458     Stupid class to interface writing out Mutt commands. This is quite a hack
459     to deal with the fact that Mutt runs "push" commands in reverse order, so
460     all of a sudden, things become very complicated when mixing with "real"
461     commands.
462
463     Hence we keep two sets of commands, and one set of pushes. Commands are
464     added to the first until a push is added, after which commands are added to
465     the second set of commands.
466
467     On flush(), the first set is printed, followed by the pushes in reverse,
468     and then the second set is printed. All 3 sets are then cleared.
469     """
470
471     def __init__(self, out_f=sys.stdout, *, debug=False):
472         self._cmd1, self._push, self._cmd2 = [], [], []
473         self._out_f = out_f
474         self._debug = debug
475
476     def cmd(self, s):
477         self.debugprint(s)
478         if self._push:
479             self._cmd2.append(s)
480         else:
481             self._cmd1.append(s)
482
483     def push(self, s):
484         s = s.replace('"', '"')
485         s = f'push "{s}"'
486         self.debugprint(s)
487         self._push.insert(0, s)
488
489     def flush(self):
490         print(
491             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
492         )
493         self._cmd1, self._push, self._cmd2 = [], [], []
494
495     def debugprint(self, s, **kwargs):
496         if self._debug:
497             print(s, file=sys.stderr, **kwargs)
498
499
500 def do_setup(
501     *,
502     out_f=sys.stdout,
503     temppath=None,
504     tempdir=None,
505     debug_commands=False,
506 ):
507     temppath = temppath or pathlib.Path(
508         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
509     )
510     cmds = MuttCommands(out_f, debug=debug_commands)
511
512     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
513
514     cmds.cmd('set my_editor="$editor"')
515     cmds.cmd('set my_edit_headers="$edit_headers"')
516     cmds.cmd(f'set editor="{editor}"')
517     cmds.cmd("unset edit_headers")
518     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
519     cmds.push("<first-entry><edit-file>")
520     cmds.flush()
521
522
523 def do_massage(
524     draft_f,
525     draftpath,
526     cmd_f,
527     *,
528     extensions=None,
529     cssfile=None,
530     converter=convert_markdown_to_html,
531     related_to_html_only=True,
532     only_build=False,
533     max_other_attachments=20,
534     tempdir=None,
535     debug_commands=False,
536     debug_walk=False,
537 ):
538     # Here's the big picture: we're being invoked as the editor on the email
539     # draft, and whatever commands we write to the file given as cmdpath will
540     # be run by the second source command in the macro definition.
541
542     # Let's start by cleaning up what the setup did (see above), i.e. we
543     # restore the $editor and $edit_headers variables, and also unset the
544     # variable used to identify the command file we're currently writing
545     # to.
546     cmds = MuttCommands(cmd_f, debug=debug_commands)
547     cmds.cmd('set editor="$my_editor"')
548     cmds.cmd('set edit_headers="$my_edit_headers"')
549     cmds.cmd("unset my_editor")
550     cmds.cmd("unset my_edit_headers")
551
552     # let's flush those commands, as there'll be a lot of pushes from now
553     # on, which need to be run in reverse order
554     cmds.flush()
555
556     extensions = extensions.split(",") if extensions else []
557     tree = converter(
558         draft_f.read(),
559         draftpath,
560         cssfile=cssfile,
561         related_to_html_only=related_to_html_only,
562         tempdir=tempdir,
563         extensions=extensions,
564     )
565
566     mimetree = MIMETreeDFWalker(debug=debug_walk)
567
568     state = dict(pos=1, tags={}, parts=1)
569
570     def visitor_fn(item, ancestry, descendents, *, debugprint=None):
571         """
572         Visitor function called for every node (part) of the MIME tree,
573         depth-first, and responsible for telling NeoMutt how to assemble
574         the tree.
575         """
576         KILL_LINE = r"\Ca\Ck"
577
578         if isinstance(item, Part):
579             # We've hit a leaf-node, i.e. an alternative or a related part
580             # with actual content.
581
582             # Let's add the part
583             if item.orig:
584                 # The original source already exists in the NeoMutt tree, but
585                 # the underlying file may have been modified, so we need to
586                 # update the encoding, but that's it:
587                 cmds.push("<first-entry>")
588                 cmds.push("<update-encoding>")
589
590                 # We really just need to be able to assume that at this point,
591                 # NeoMutt is at position 1, and that we've processed only this
592                 # part so far. Nevermind about actual attachments, we can
593                 # safely ignore those as they stay at the end.
594                 assert state["pos"] == 1
595                 assert state["parts"] == 1
596             else:
597                 # … whereas all other parts need to be added, and they're all
598                 # considered to be temporary and inline:
599                 cmds.push(f"<attach-file>{item.path}<enter>")
600                 cmds.push("<toggle-unlink><toggle-disposition>")
601
602                 # This added a part at the end of the list of parts, and that's
603                 # just how many parts we've seen so far, so it's position in
604                 # the NeoMutt compose list is the count of parts
605                 state["parts"] += 1
606                 state["pos"] = state["parts"]
607
608             # If the item (including the original) comes with additional
609             # information, then we might just as well update the NeoMutt
610             # tree now:
611             if item.cid:
612                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
613
614             # Now for the biggest hack in this script, which is to handle
615             # attachments, such as PDFs, that aren't related or alternatives.
616             # The problem is that when we add an inline image, it always gets
617             # appended to the list, i.e. inserted *after* other attachments.
618             # Since we don't know the number of attachments, we also cannot
619             # infer the postition of the new attachment. Therefore, we bubble
620             # it all the way to the top, only to then move it down again:
621             if state["pos"] > 1:  # skip for the first part
622                 for i in range(max_other_attachments):
623                     # could use any number here, but has to be larger than the
624                     # number of possible attachments. The performance
625                     # difference of using a high number is negligible.
626                     # Bubble up the new part
627                     cmds.push(f"<move-up>")
628
629                 # As we push the part to the right position in the list (i.e.
630                 # the last of the subset of attachments this script added), we
631                 # must handle the situation that subtrees are skipped by
632                 # NeoMutt. Hence, the actual number of positions to move down
633                 # is decremented by the number of descendents so far
634                 # encountered.
635                 for i in range(1, state["pos"] - len(descendents)):
636                     cmds.push(f"<move-down>")
637
638         elif isinstance(item, Multipart):
639             # This node has children, but we already visited them (see
640             # above). The tags dictionary of State should contain a list of
641             # their positions in the NeoMutt compose window, so iterate those
642             # and tag the parts there:
643             n_tags = len(state["tags"][item])
644             for tag in state["tags"][item]:
645                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
646
647             if item.subtype == "alternative":
648                 cmds.push("<group-alternatives>")
649             elif item.subtype in ("relative", "related"):
650                 cmds.push("<group-related>")
651             elif item.subtype == "multilingual":
652                 cmds.push("<group-multilingual>")
653             else:
654                 raise NotImplementedError(
655                     f"Handling of multipart/{item.subtype} is not implemented"
656                 )
657
658             state["pos"] -= n_tags - 1
659             state["parts"] += 1
660
661         else:
662             # We should never get here
663             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
664
665         # If the item has a description, we might just as well add it
666         if item.desc:
667             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
668
669         if ancestry:
670             # If there's an ancestry, record the current (assumed) position in
671             # the NeoMutt compose window as needed-to-tag by our direct parent
672             # (i.e. the last item of the ancestry)
673             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
674
675             lead = "│ " * (len(ancestry) + 1) + "* "
676             debugprint(
677                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
678                 f"{lead}descendents={[d.subtype for d in descendents]}\n"
679                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
680                 f"{lead}pos={state['pos']}, parts={state['parts']}"
681             )
682
683     # -----------------
684     # End of visitor_fn
685
686     # Let's walk the tree and visit every node with our fancy visitor
687     # function
688     mimetree.walk(tree, visitor_fn=visitor_fn)
689
690     if not only_build:
691         cmds.push("<send-message>")
692
693     # Finally, cleanup. Since we're responsible for removing the temporary
694     # file, how's this for a little hack?
695     try:
696         filename = cmd_f.name
697     except AttributeError:
698         filename = "pytest_internal_file"
699     cmds.cmd(f"source 'rm -f {filename}|'")
700     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
701     cmds.flush()
702
703
704 # [ CLI ENTRY ] ###############################################################
705
706 if __name__ == "__main__":
707     args = parse_cli_args()
708
709     if args.mode is None:
710         do_setup(
711             tempdir=args.tempdir,
712             debug_commands=args.debug_commands,
713         )
714
715     elif args.mode == "massage":
716         with open(args.MAILDRAFT, "r") as draft_f, open(
717             args.cmdpath, "w"
718         ) as cmd_f:
719             do_massage(
720                 draft_f,
721                 pathlib.Path(args.MAILDRAFT),
722                 cmd_f,
723                 extensions=args.extensions,
724                 cssfile=args.css_file,
725                 related_to_html_only=args.related_to_html_only,
726                 max_other_attachments=args.max_number_other_attachments,
727                 only_build=args.only_build,
728                 tempdir=args.tempdir,
729                 debug_commands=args.debug_commands,
730                 debug_walk=args.debug_walk,
731             )
732
733
734 # [ TESTS ] ###################################################################
735
736 try:
737     import pytest
738     from io import StringIO
739
740     class Tests:
741         @pytest.fixture
742         def const1(self):
743             return "CONSTANT STRING 1"
744
745         @pytest.fixture
746         def const2(self):
747             return "CONSTANT STRING 2"
748
749         # NOTE: tests using the capsys fixture must specify sys.stdout to the
750         # functions they call, else old stdout is used and not captured
751
752         def test_MuttCommands_cmd(self, const1, const2, capsys):
753             "Assert order of commands"
754             cmds = MuttCommands(out_f=sys.stdout)
755             cmds.cmd(const1)
756             cmds.cmd(const2)
757             cmds.flush()
758             captured = capsys.readouterr()
759             assert captured.out == "\n".join((const1, const2, ""))
760
761         def test_MuttCommands_push(self, const1, const2, capsys):
762             "Assert reverse order of pushes"
763             cmds = MuttCommands(out_f=sys.stdout)
764             cmds.push(const1)
765             cmds.push(const2)
766             cmds.flush()
767             captured = capsys.readouterr()
768             assert (
769                 captured.out
770                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
771             )
772
773         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
774             "Assert reverse order of pushes"
775             cmds = MuttCommands(out_f=sys.stdout)
776             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
777             for i in range(2):
778                 cmds.cmd(lines[4 * i + 0])
779                 cmds.cmd(lines[4 * i + 1])
780                 cmds.push(lines[4 * i + 2])
781                 cmds.push(lines[4 * i + 3])
782             cmds.flush()
783
784             captured = capsys.readouterr()
785             lines_out = captured.out.splitlines()
786             assert lines[0] in lines_out[0]
787             assert lines[1] in lines_out[1]
788             assert lines[7] in lines_out[2]
789             assert lines[6] in lines_out[3]
790             assert lines[3] in lines_out[4]
791             assert lines[2] in lines_out[5]
792             assert lines[4] in lines_out[6]
793             assert lines[5] in lines_out[7]
794
795         @pytest.fixture
796         def mime_tree_related_to_alternative(self):
797             return Multipart(
798                 "relative",
799                 children=[
800                     Multipart(
801                         "alternative",
802                         children=[
803                             Part(
804                                 "text",
805                                 "plain",
806                                 "part.txt",
807                                 desc="Plain",
808                                 orig=True,
809                             ),
810                             Part("text", "html", "part.html", desc="HTML"),
811                         ],
812                         desc="Alternative",
813                     ),
814                     Part(
815                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
816                     ),
817                 ],
818                 desc="Related",
819             )
820
821         @pytest.fixture
822         def mime_tree_related_to_html(self):
823             return Multipart(
824                 "alternative",
825                 children=[
826                     Part(
827                         "text",
828                         "plain",
829                         "part.txt",
830                         desc="Plain",
831                         orig=True,
832                     ),
833                     Multipart(
834                         "relative",
835                         children=[
836                             Part("text", "html", "part.html", desc="HTML"),
837                             Part(
838                                 "text",
839                                 "png",
840                                 "logo.png",
841                                 cid="logo.png",
842                                 desc="Logo",
843                             ),
844                         ],
845                         desc="Related",
846                     ),
847                 ],
848                 desc="Alternative",
849             )
850
851         def test_MIMETreeDFWalker_depth_first_walk(
852             self, mime_tree_related_to_alternative
853         ):
854             mimetree = MIMETreeDFWalker()
855
856             items = []
857
858             def visitor_fn(item, ancestry, descendents, debugprint):
859                 items.append((item, len(ancestry), len(descendents)))
860
861             mimetree.walk(
862                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
863             )
864             assert len(items) == 5
865             assert items[0][0].subtype == "plain"
866             assert items[0][1] == 2
867             assert items[0][2] == 0
868             assert items[1][0].subtype == "html"
869             assert items[1][1] == 2
870             assert items[1][2] == 0
871             assert items[2][0].subtype == "alternative"
872             assert items[2][1] == 1
873             assert items[2][2] == 2
874             assert items[3][0].subtype == "png"
875             assert items[3][1] == 1
876             assert items[3][2] == 2
877             assert items[4][0].subtype == "relative"
878             assert items[4][1] == 0
879             assert items[4][2] == 4
880
881         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
882             mimetree = MIMETreeDFWalker()
883             items = []
884
885             def visitor_fn(item, ancestry, descendents, debugprint):
886                 items.append(item)
887
888             p = Part("text", "plain", const1)
889             mimetree.walk([p], visitor_fn=visitor_fn)
890             assert items[-1].subtype == "plain"
891             mimetree.walk([p, p], visitor_fn=visitor_fn)
892             assert items[-1].subtype == "mixed"
893
894         def test_MIMETreeDFWalker_visitor_in_constructor(
895             self, mime_tree_related_to_alternative
896         ):
897             items = []
898
899             def visitor_fn(item, ancestry, descendents, debugprint):
900                 items.append(item)
901
902             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
903             mimetree.walk(mime_tree_related_to_alternative)
904             assert len(items) == 5
905
906         @pytest.fixture
907         def string_io(self, const1, text=None):
908             return StringIO(text or const1)
909
910         def test_do_massage_basic(self, const1, string_io, capsys):
911             def converter(
912                 drafttext,
913                 draftpath,
914                 cssfile,
915                 related_to_html_only,
916                 extensions,
917                 tempdir,
918             ):
919                 return Part("text", "plain", draftpath, orig=True)
920
921             do_massage(
922                 draft_f=string_io,
923                 draftpath=const1,
924                 cmd_f=sys.stdout,
925                 converter=converter,
926             )
927
928             captured = capsys.readouterr()
929             lines = captured.out.splitlines()
930             assert '="$my_editor"' in lines.pop(0)
931             assert '="$my_edit_headers"' in lines.pop(0)
932             assert "unset my_editor" == lines.pop(0)
933             assert "unset my_edit_headers" == lines.pop(0)
934             assert "send-message" in lines.pop(0)
935             assert "update-encoding" in lines.pop(0)
936             assert "first-entry" in lines.pop(0)
937             assert "source 'rm -f " in lines.pop(0)
938             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
939
940         def test_do_massage_fulltree(
941             self, string_io, const1, mime_tree_related_to_alternative, capsys
942         ):
943             def converter(
944                 drafttext,
945                 draftpath,
946                 cssfile,
947                 related_to_html_only,
948                 extensions,
949                 tempdir,
950             ):
951                 return mime_tree_related_to_alternative
952
953             max_attachments = 5
954             do_massage(
955                 draft_f=string_io,
956                 draftpath=const1,
957                 cmd_f=sys.stdout,
958                 max_other_attachments=max_attachments,
959                 converter=converter,
960             )
961
962             captured = capsys.readouterr()
963             lines = captured.out.splitlines()[4:-2]
964             assert "first-entry" in lines.pop()
965             assert "update-encoding" in lines.pop()
966             assert "Plain" in lines.pop()
967             assert "part.html" in lines.pop()
968             assert "toggle-unlink" in lines.pop()
969             for i in range(max_attachments):
970                 assert "move-up" in lines.pop()
971             assert "move-down" in lines.pop()
972             assert "HTML" in lines.pop()
973             assert "jump>1" in lines.pop()
974             assert "jump>2" in lines.pop()
975             assert "group-alternatives" in lines.pop()
976             assert "Alternative" in lines.pop()
977             assert "logo.png" in lines.pop()
978             assert "toggle-unlink" in lines.pop()
979             assert "content-id" in lines.pop()
980             for i in range(max_attachments):
981                 assert "move-up" in lines.pop()
982             assert "move-down" in lines.pop()
983             assert "Logo" in lines.pop()
984             assert "jump>1" in lines.pop()
985             assert "jump>4" in lines.pop()
986             assert "group-related" in lines.pop()
987             assert "Related" in lines.pop()
988             assert "send-message" in lines.pop()
989             assert len(lines) == 0
990
991         @pytest.fixture
992         def fake_filewriter(self):
993             class FileWriter:
994                 def __init__(self):
995                     self._writes = []
996
997                 def __call__(self, path, content, mode="w", **kwargs):
998                     self._writes.append((path, content))
999
1000                 def pop(self, index=-1):
1001                     return self._writes.pop(index)
1002
1003             return FileWriter()
1004
1005         @pytest.fixture
1006         def markdown_non_converter(self, const1, const2):
1007             return lambda s, text: f"{const1}{text}{const2}"
1008
1009         def test_converter_tree_basic(self, const1, const2, fake_filewriter):
1010             path = pathlib.Path(const2)
1011             tree = convert_markdown_to_html(
1012                 const1, path, filewriter_fn=fake_filewriter
1013             )
1014
1015             assert tree.subtype == "alternative"
1016             assert len(tree.children) == 2
1017             assert tree.children[0].subtype == "plain"
1018             assert tree.children[0].path == path
1019             assert tree.children[0].orig
1020             assert tree.children[1].subtype == "html"
1021             assert tree.children[1].path == path.with_suffix(".html")
1022
1023         def test_converter_writes(
1024             self,
1025             const1,
1026             const2,
1027             fake_filewriter,
1028             monkeypatch,
1029             markdown_non_converter,
1030         ):
1031             path = pathlib.Path(const2)
1032
1033             with monkeypatch.context() as m:
1034                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
1035                 convert_markdown_to_html(
1036                     const1, path, filewriter_fn=fake_filewriter
1037                 )
1038
1039             assert (path, const1) == fake_filewriter.pop(0)
1040             assert (
1041                 path.with_suffix(".html"),
1042                 markdown_non_converter(None, const1),
1043             ) == fake_filewriter.pop(0)
1044
1045         def test_markdown_inline_image_processor(self):
1046             imgpath1 = "file:/path/to/image.png"
1047             imgpath2 = "file:///path/to/image.png?url=params"
1048             imgpath3 = "/path/to/image.png"
1049             text = f"""![inline local image]({imgpath1})
1050                        ![image inlined
1051                          with newline]({imgpath2})
1052                        ![image local path]({imgpath3})"""
1053             text, html, images = markdown_with_inline_image_support(text)
1054
1055             # local paths have been normalised to URLs:
1056             imgpath3 = f"file://{imgpath3}"
1057
1058             assert 'src="cid:' in html
1059             assert "](cid:" in text
1060             assert len(images) == 3
1061             assert imgpath1 in images
1062             assert imgpath2 in images
1063             assert imgpath3 in images
1064             assert images[imgpath1].cid != images[imgpath2].cid
1065             assert images[imgpath1].cid != images[imgpath3].cid
1066             assert images[imgpath2].cid != images[imgpath3].cid
1067
1068         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1069             imgpath = "file:///path/to/image.png"
1070             text = f'![inline local image]({imgpath} "{const1}")'
1071             text, html, images = markdown_with_inline_image_support(text)
1072             assert images[imgpath].desc == const1
1073
1074         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1075             imgpath = "file:///path/to/image.png"
1076             text = f"![{const1}]({imgpath})"
1077             text, html, images = markdown_with_inline_image_support(text)
1078             assert images[imgpath].desc == const1
1079
1080         def test_markdown_inline_image_processor_title_over_alt_desc(
1081             self, const1, const2
1082         ):
1083             imgpath = "file:///path/to/image.png"
1084             text = f'![{const1}]({imgpath} "{const2}")'
1085             text, html, images = markdown_with_inline_image_support(text)
1086             assert images[imgpath].desc == const2
1087
1088         def test_markdown_inline_image_not_external(self):
1089             imgpath = "https://path/to/image.png"
1090             text = f"![inline image]({imgpath})"
1091             text, html, images = markdown_with_inline_image_support(text)
1092
1093             assert 'src="cid:' not in html
1094             assert "](cid:" not in text
1095             assert len(images) == 0
1096
1097         def test_markdown_inline_image_local_file(self):
1098             imgpath = "/path/to/image.png"
1099             text = f"![inline image]({imgpath})"
1100             text, html, images = markdown_with_inline_image_support(text)
1101
1102             for k, v in images.items():
1103                 assert k == f"file://{imgpath}"
1104                 break
1105
1106         @pytest.fixture
1107         def test_png(self):
1108             return (
1109                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1110                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1111             )
1112
1113         def test_markdown_inline_image_processor_base64(self, test_png):
1114             text = f"![1px white inlined]({test_png})"
1115             text, html, images = markdown_with_inline_image_support(text)
1116
1117             assert 'src="cid:' in html
1118             assert "](cid:" in text
1119             assert len(images) == 1
1120             assert test_png in images
1121
1122         def test_converter_tree_inline_image_base64(
1123             self, test_png, const1, fake_filewriter
1124         ):
1125             text = f"![inline base64 image]({test_png})"
1126             path = pathlib.Path(const1)
1127             tree = convert_markdown_to_html(
1128                 text,
1129                 path,
1130                 filewriter_fn=fake_filewriter,
1131                 related_to_html_only=False,
1132             )
1133             assert tree.subtype == "relative"
1134             assert tree.children[0].subtype == "alternative"
1135             assert tree.children[1].subtype == "png"
1136             written = fake_filewriter.pop()
1137             assert tree.children[1].path == written[0]
1138             assert written[1] == request.urlopen(test_png).read()
1139
1140         def test_converter_tree_inline_image_base64_related_to_html(
1141             self, test_png, const1, fake_filewriter
1142         ):
1143             text = f"![inline base64 image]({test_png})"
1144             path = pathlib.Path(const1)
1145             tree = convert_markdown_to_html(
1146                 text,
1147                 path,
1148                 filewriter_fn=fake_filewriter,
1149                 related_to_html_only=True,
1150             )
1151             assert tree.subtype == "alternative"
1152             assert tree.children[1].subtype == "relative"
1153             assert tree.children[1].children[1].subtype == "png"
1154             written = fake_filewriter.pop()
1155             assert tree.children[1].children[1].path == written[0]
1156             assert written[1] == request.urlopen(test_png).read()
1157
1158         def test_converter_tree_inline_image_cid(
1159             self, const1, fake_filewriter
1160         ):
1161             text = f"![inline base64 image](cid:{const1})"
1162             path = pathlib.Path(const1)
1163             tree = convert_markdown_to_html(
1164                 text,
1165                 path,
1166                 filewriter_fn=fake_filewriter,
1167                 related_to_html_only=False,
1168             )
1169             assert len(tree.children) == 2
1170             assert tree.children[0].cid != const1
1171             assert tree.children[0].type != "image"
1172             assert tree.children[1].cid != const1
1173             assert tree.children[1].type != "image"
1174
1175         def test_inline_image_collection(
1176             self, test_png, const1, const2, fake_filewriter
1177         ):
1178             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1179             relparts = collect_inline_images(
1180                 test_images, filewriter_fn=fake_filewriter
1181             )
1182
1183             written = fake_filewriter.pop()
1184             assert b"PNG" in written[1]
1185
1186             assert relparts[0].subtype == "png"
1187             assert relparts[0].path == written[0]
1188             assert relparts[0].cid == const1
1189             assert relparts[0].desc.endswith(const2)
1190
1191         def test_apply_stylesheet(self):
1192             if _PYNLINER:
1193                 html = "<p>Hello, world!</p>"
1194                 css = "p { color:red }"
1195                 out = apply_styling(html, css)
1196                 assert 'p style="color' in out
1197
1198         def test_apply_stylesheet_pygments(self):
1199             if _PYGMENTS_CSS:
1200                 html = (
1201                     f'<div class="{_CODEHILITE_CLASS}">'
1202                     "<pre>def foo():\n    return</pre></div>"
1203                 )
1204                 out = apply_styling(html, _PYGMENTS_CSS)
1205                 assert f'{_CODEHILITE_CLASS}" style="' in out
1206
1207         def test_mime_tree_relative_within_alternative(
1208             self, string_io, const1, capsys, mime_tree_related_to_html
1209         ):
1210             def converter(
1211                 drafttext,
1212                 draftpath,
1213                 cssfile,
1214                 related_to_html_only,
1215                 extensions,
1216                 tempdir,
1217             ):
1218                 return mime_tree_related_to_html
1219
1220             do_massage(
1221                 draft_f=string_io,
1222                 draftpath=const1,
1223                 cmd_f=sys.stdout,
1224                 converter=converter,
1225             )
1226
1227             captured = capsys.readouterr()
1228             lines = captured.out.splitlines()[4:-2]
1229             assert "first-entry" in lines.pop()
1230             assert "update-encoding" in lines.pop()
1231             assert "Plain" in lines.pop()
1232             assert "part.html" in lines.pop()
1233             assert "toggle-unlink" in lines.pop()
1234             assert "move-up" in lines.pop()
1235             while True:
1236                 top = lines.pop()
1237                 if "move-up" not in top:
1238                     break
1239             assert "move-down" in top
1240             assert "HTML" in lines.pop()
1241             assert "logo.png" in lines.pop()
1242             assert "toggle-unlink" in lines.pop()
1243             assert "content-id" in lines.pop()
1244             assert "move-up" in lines.pop()
1245             while True:
1246                 top = lines.pop()
1247                 if "move-up" not in top:
1248                     break
1249             assert "move-down" in top
1250             assert "move-down" in lines.pop()
1251             assert "Logo" in lines.pop()
1252             assert "jump>2" in lines.pop()
1253             assert "jump>3" in lines.pop()
1254             assert "group-related" in lines.pop()
1255             assert "Related" in lines.pop()
1256             assert "jump>1" in lines.pop()
1257             assert "jump>2" in lines.pop()
1258             assert "group-alternative" in lines.pop()
1259             assert "Alternative" in lines.pop()
1260             assert "send-message" in lines.pop()
1261             assert len(lines) == 0
1262
1263         def test_mime_tree_nested_trees_does_not_break_positioning(
1264             self, string_io, const1, capsys
1265         ):
1266             def converter(
1267                 drafttext,
1268                 draftpath,
1269                 cssfile,
1270                 related_to_html_only,
1271                 extensions,
1272                 tempdir,
1273             ):
1274                 return Multipart(
1275                     "relative",
1276                     children=[
1277                         Multipart(
1278                             "alternative",
1279                             children=[
1280                                 Part(
1281                                     "text",
1282                                     "plain",
1283                                     "part.txt",
1284                                     desc="Plain",
1285                                     orig=True,
1286                                 ),
1287                                 Multipart(
1288                                     "alternative",
1289                                     children=[
1290                                         Part(
1291                                             "text",
1292                                             "plain",
1293                                             "part.txt",
1294                                             desc="Nested plain",
1295                                         ),
1296                                         Part(
1297                                             "text",
1298                                             "html",
1299                                             "part.html",
1300                                             desc="Nested HTML",
1301                                         ),
1302                                     ],
1303                                     desc="Nested alternative",
1304                                 ),
1305                             ],
1306                             desc="Alternative",
1307                         ),
1308                         Part(
1309                             "text",
1310                             "png",
1311                             "logo.png",
1312                             cid="logo.png",
1313                             desc="Logo",
1314                         ),
1315                     ],
1316                     desc="Related",
1317                 )
1318
1319             do_massage(
1320                 draft_f=string_io,
1321                 draftpath=const1,
1322                 cmd_f=sys.stdout,
1323                 converter=converter,
1324             )
1325
1326             captured = capsys.readouterr()
1327             lines = captured.out.splitlines()
1328             while not "logo.png" in lines.pop():
1329                 pass
1330             lines.pop()
1331             assert "content-id" in lines.pop()
1332             assert "move-up" in lines.pop()
1333             while True:
1334                 top = lines.pop()
1335                 if "move-up" not in top:
1336                     break
1337             assert "move-down" in top
1338             # Due to the nested trees, the number of descendents of the sibling
1339             # actually needs to be considered, not just the nieces. So to move
1340             # from position 1 to position 6, it only needs one <move-down>
1341             # because that jumps over the entire sibling tree. Thus what
1342             # follows next must not be another <move-down>
1343             assert "Logo" in lines.pop()
1344
1345 except ImportError:
1346     pass