]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

fixup! Support both mp/related inline tree structures
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
10 #     macro compose B "\
11 #       <enter-command> source '$my_confdir/buildmimetree.py \
12 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
13 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
14 #     " "Convert message into a modern MIME tree with inline images"
15 #
16 #     (Yes, we need to call source twice, as mutt only starts to process output
17 #     from a source command when the command exits, and since we need to react
18 #     to the output, we need to be invoked again, using a $my_ variable to pass
19 #     information)
20 #
21 # Requirements:
22 #   - python3
23 #   - python3-markdown
24 # Optional:
25 #   - pytest
26 #   - Pynliner, provides --css-file and thus inline styling of HTML output
27 #   - Pygments, then syntax highlighting for fenced code is enabled
28 #
29 # Latest version:
30 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
31 #
32 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
33 # Released under the GPL-2+ licence, just like Mutt itself.
34 #
35
36 import sys
37 import pathlib
38 import markdown
39 import tempfile
40 import argparse
41 import re
42 import mimetypes
43 from collections import namedtuple, OrderedDict
44 from markdown.extensions import Extension
45 from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
46 from email.utils import make_msgid
47 from urllib import request
48
49
50 def parse_cli_args(*args, **kwargs):
51     parser = argparse.ArgumentParser(
52         description=(
53             "NeoMutt helper to turn text/markdown email parts "
54             "into full-fledged MIME trees"
55         )
56     )
57     parser.epilog = (
58         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
59         "Released under the MIT licence"
60     )
61
62     parser.add_argument(
63         "--extensions",
64         type=str,
65         default="",
66         help="Markdown extension to use (comma-separated list)",
67     )
68
69     if _PYNLINER:
70         parser.add_argument(
71             "--css-file",
72             type=str,
73             default="",
74             help="CSS file to merge with the final HTML",
75         )
76     else:
77         parser.set_defaults(css_file=None)
78
79     parser.add_argument(
80         "--related-to-html-only",
81         action="store_true",
82         help="Make related content be sibling to HTML parts only"
83     )
84
85     parser.add_argument(
86         "--only-build",
87         action="store_true",
88         help="Only build, don't send the message",
89     )
90
91     parser.add_argument(
92         "--tempdir",
93         default=None,
94         help="Specify temporary directory to use for attachments",
95     )
96
97     parser.add_argument(
98         "--debug-commands",
99         action="store_true",
100         help="Turn on debug logging of commands generated to stderr",
101     )
102
103     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
104     massage_p = subp.add_parser(
105         "massage", help="Massaging phase (internal use)"
106     )
107
108     massage_p.add_argument(
109         "--write-commands-to",
110         "-o",
111         metavar="PATH",
112         dest="cmdpath",
113         required=True,
114         help="Temporary file path to write commands to",
115     )
116
117     massage_p.add_argument(
118         "--debug-walk",
119         action="store_true",
120         help="Turn on debugging to stderr of the MIME tree walk",
121     )
122
123     massage_p.add_argument(
124         "MAILDRAFT",
125         nargs="?",
126         help="If provided, the script is invoked as editor on the mail draft",
127     )
128
129     return parser.parse_args(*args, **kwargs)
130
131
132 # [ MARKDOWN WRAPPING ] #######################################################
133
134
135 InlineImageInfo = namedtuple(
136     "InlineImageInfo", ["cid", "desc"], defaults=[None]
137 )
138
139
140 class InlineImageExtension(Extension):
141     class RelatedImageInlineProcessor(ImageInlineProcessor):
142         def __init__(self, re, md, ext):
143             super().__init__(re, md)
144             self._ext = ext
145
146         def handleMatch(self, m, data):
147             el, start, end = super().handleMatch(m, data)
148             if "src" in el.attrib:
149                 src = el.attrib["src"]
150                 if "://" not in src or src.startswith("file://"):
151                     # We only inline local content
152                     cid = self._ext.get_cid_for_image(el.attrib)
153                     el.attrib["src"] = f"cid:{cid}"
154             return el, start, end
155
156     def __init__(self):
157         super().__init__()
158         self._images = OrderedDict()
159
160     def extendMarkdown(self, md):
161         md.registerExtension(self)
162         inline_image_proc = self.RelatedImageInlineProcessor(
163             IMAGE_LINK_RE, md, self
164         )
165         md.inlinePatterns.register(inline_image_proc, "image_link", 150)
166
167     def get_cid_for_image(self, attrib):
168         msgid = make_msgid()[1:-1]
169         path = attrib["src"]
170         if path.startswith("/"):
171             path = f"file://{path}"
172         self._images[path] = InlineImageInfo(
173             msgid, attrib.get("title", attrib.get("alt"))
174         )
175         return msgid
176
177     def get_images(self):
178         return self._images
179
180
181 def markdown_with_inline_image_support(
182     text, *, extensions=None, extension_configs=None
183 ):
184     inline_image_handler = InlineImageExtension()
185     extensions = extensions or []
186     extensions.append(inline_image_handler)
187     mdwn = markdown.Markdown(
188         extensions=extensions, extension_configs=extension_configs
189     )
190     htmltext = mdwn.convert(text)
191
192     images = inline_image_handler.get_images()
193
194     def replace_image_with_cid(matchobj):
195         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
196             if m in images:
197                 return f"(cid:{images[m].cid}"
198         return matchobj.group(0)
199
200     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
201     return text, htmltext, images
202
203
204 # [ CSS STYLING ] #############################################################
205
206 try:
207     import pynliner
208
209     _PYNLINER = True
210
211 except ImportError:
212     _PYNLINER = False
213
214 try:
215     from pygments.formatters import get_formatter_by_name
216
217     _CODEHILITE_CLASS = "codehilite"
218
219     _PYGMENTS_CSS = get_formatter_by_name(
220         "html", style="default"
221     ).get_style_defs(f".{_CODEHILITE_CLASS}")
222
223 except ImportError:
224     _PYGMENTS_CSS = None
225
226
227 def apply_styling(html, css):
228     return (
229         pynliner.Pynliner()
230         .from_string(html)
231         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
232         .run()
233     )
234
235
236 # [ PARTS GENERATION ] ########################################################
237
238
239 class Part(
240     namedtuple(
241         "Part",
242         ["type", "subtype", "path", "desc", "cid", "orig"],
243         defaults=[None, None, False],
244     )
245 ):
246     def __str__(self):
247         ret = f"<{self.type}/{self.subtype}>"
248         if self.cid:
249             ret = f"{ret} cid:{self.cid}"
250         if self.orig:
251             ret = f"{ret} ORIGINAL"
252         return ret
253
254
255 class Multipart(
256     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
257 ):
258     def __str__(self):
259         return f"<multipart/{self.subtype}> children={len(self.children)}"
260
261     def __hash__(self):
262         return hash(str(self.subtype) + "".join(str(self.children)))
263
264
265 def filewriter_fn(path, content, mode="w", **kwargs):
266     with open(path, mode, **kwargs) as out_f:
267         out_f.write(content)
268
269
270 def collect_inline_images(
271     images, *, tempdir=None, filewriter_fn=filewriter_fn
272 ):
273     relparts = []
274     for path, info in images.items():
275         if path.startswith("cid:"):
276             continue
277
278         data = request.urlopen(path)
279
280         mimetype = data.headers["Content-Type"]
281         ext = mimetypes.guess_extension(mimetype)
282         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
283         path = pathlib.Path(tempfilename[1])
284
285         filewriter_fn(path, data.read(), "w+b")
286
287         relparts.append(
288             Part(
289                 *mimetype.split("/"),
290                 path,
291                 cid=info.cid,
292                 desc=f"Image: {info.desc}",
293             )
294         )
295
296     return relparts
297
298
299 def convert_markdown_to_html(
300     origtext,
301     draftpath,
302     *,
303     related_to_html_only=False,
304     cssfile=None,
305     filewriter_fn=filewriter_fn,
306     tempdir=None,
307     extensions=None,
308     extension_configs=None,
309 ):
310     # TODO extension_configs need to be handled differently
311     extension_configs = extension_configs or {}
312     extension_configs.setdefault("pymdownx.highlight", {})
313     extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
314
315     origtext, htmltext, images = markdown_with_inline_image_support(
316         origtext, extensions=extensions, extension_configs=extension_configs
317     )
318
319     filewriter_fn(draftpath, origtext, encoding="utf-8")
320     textpart = Part(
321         "text", "plain", draftpath, "Plain-text version", orig=True
322     )
323
324     htmltext = apply_styling(htmltext, cssfile)
325
326     htmlpath = draftpath.with_suffix(".html")
327     filewriter_fn(
328         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
329     )
330     htmlpart = Part("text", "html", htmlpath, "HTML version")
331
332     imgparts = collect_inline_images(
333         images, tempdir=tempdir, filewriter_fn=filewriter_fn
334     )
335
336     if related_to_html_only:
337         # If there are inline image part, they will be contained within a
338         # multipart/related part along with the HTML part only
339         if imgparts:
340             # replace htmlpart with a multipart/related container of the HTML
341             # parts and the images
342             htmlpart = Multipart(
343                 "relative", [htmlpart] + imgparts, "Group of related content"
344             )
345
346         return Multipart(
347             "alternative", [textpart, htmlpart], "Group of alternative content"
348         )
349
350     else:
351         # If there are inline image part, they will be siblings to the
352         # multipart/alternative tree within a multipart/related part
353         altpart = Multipart(
354             "alternative", [textpart, htmlpart], "Group of alternative content"
355         )
356         if imgparts:
357             return Multipart(
358                 "relative", [altpart] + imgparts, "Group of related content"
359             )
360         else:
361             return altpart
362
363
364 class MIMETreeDFWalker:
365     def __init__(self, *, visitor_fn=None, debug=False):
366         self._visitor_fn = visitor_fn or self._echovisit
367         self._debug = debug
368
369     def _echovisit(self, node, ancestry, debugprint):
370         debugprint(f"node={node} ancestry={ancestry}")
371
372     def walk(self, root, *, visitor_fn=None):
373         """
374         Recursive function to implement a depth-dirst walk of the MIME-tree
375         rooted at `root`.
376         """
377         if isinstance(root, list):
378             if len(root) > 1:
379                 root = Multipart("mixed", children=root)
380             else:
381                 root = root[0]
382
383         self._walk(
384             root,
385             ancestry=[],
386             visitor_fn=visitor_fn or self._visitor_fn,
387         )
388
389     def _walk(self, node, *, ancestry, visitor_fn):
390         # Let's start by enumerating the parts at the current level. At the
391         # root level, ancestry will be the empty list, and we expect a
392         # multipart/* container at this level. Later, e.g. within a
393         # mutlipart/alternative container, the subtree will just be the
394         # alternative parts, while the top of the ancestry will be the
395         # multipart/alternative container, which we will process after the
396         # following loop.
397
398         lead = f"{'│ '*len(ancestry)}"
399         if isinstance(node, Multipart):
400             self.debugprint(
401                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
402             )
403
404             # Depth-first, so push the current container onto the ancestry
405             # stack, then descend …
406             ancestry.append(node)
407             self.debugprint(lead + "│ " * 2)
408             for child in node.children:
409                 self._walk(
410                     child,
411                     ancestry=ancestry,
412                     visitor_fn=visitor_fn,
413                 )
414             assert ancestry.pop() == node
415
416         else:
417             self.debugprint(f"{lead}├{node}")
418
419         if False and ancestry:
420             self.debugprint(lead[:-1] + " │")
421
422         if visitor_fn:
423             visitor_fn(node, ancestry, debugprint=self.debugprint)
424
425     def debugprint(self, s, **kwargs):
426         if self._debug:
427             print(s, file=sys.stderr, **kwargs)
428
429
430 # [ RUN MODES ] ###############################################################
431
432
433 class MuttCommands:
434     """
435     Stupid class to interface writing out Mutt commands. This is quite a hack
436     to deal with the fact that Mutt runs "push" commands in reverse order, so
437     all of a sudden, things become very complicated when mixing with "real"
438     commands.
439
440     Hence we keep two sets of commands, and one set of pushes. Commands are
441     added to the first until a push is added, after which commands are added to
442     the second set of commands.
443
444     On flush(), the first set is printed, followed by the pushes in reverse,
445     and then the second set is printed. All 3 sets are then cleared.
446     """
447
448     def __init__(self, out_f=sys.stdout, *, debug=False):
449         self._cmd1, self._push, self._cmd2 = [], [], []
450         self._out_f = out_f
451         self._debug = debug
452
453     def cmd(self, s):
454         self.debugprint(s)
455         if self._push:
456             self._cmd2.append(s)
457         else:
458             self._cmd1.append(s)
459
460     def push(self, s):
461         s = s.replace('"', '"')
462         s = f'push "{s}"'
463         self.debugprint(s)
464         self._push.insert(0, s)
465
466     def flush(self):
467         print(
468             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
469         )
470         self._cmd1, self._push, self._cmd2 = [], [], []
471
472     def debugprint(self, s, **kwargs):
473         if self._debug:
474             print(s, file=sys.stderr, **kwargs)
475
476
477 def do_setup(
478     *,
479     out_f=sys.stdout,
480     temppath=None,
481     tempdir=None,
482     debug_commands=False,
483 ):
484     temppath = temppath or pathlib.Path(
485         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
486     )
487     cmds = MuttCommands(out_f, debug=debug_commands)
488
489     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
490
491     cmds.cmd('set my_editor="$editor"')
492     cmds.cmd('set my_edit_headers="$edit_headers"')
493     cmds.cmd(f'set editor="{editor}"')
494     cmds.cmd("unset edit_headers")
495     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
496     cmds.push("<first-entry><edit-file>")
497     cmds.flush()
498
499
500 def do_massage(
501     draft_f,
502     draftpath,
503     cmd_f,
504     *,
505     extensions=None,
506     cssfile=None,
507     converter=convert_markdown_to_html,
508     related_to_html_only=True,
509     only_build=False,
510     tempdir=None,
511     debug_commands=False,
512     debug_walk=False,
513 ):
514     # Here's the big picture: we're being invoked as the editor on the email
515     # draft, and whatever commands we write to the file given as cmdpath will
516     # be run by the second source command in the macro definition.
517
518     # Let's start by cleaning up what the setup did (see above), i.e. we
519     # restore the $editor and $edit_headers variables, and also unset the
520     # variable used to identify the command file we're currently writing
521     # to.
522     cmds = MuttCommands(cmd_f, debug=debug_commands)
523     cmds.cmd('set editor="$my_editor"')
524     cmds.cmd('set edit_headers="$my_edit_headers"')
525     cmds.cmd("unset my_editor")
526     cmds.cmd("unset my_edit_headers")
527
528     # let's flush those commands, as there'll be a lot of pushes from now
529     # on, which need to be run in reverse order
530     cmds.flush()
531
532     extensions = extensions.split(",") if extensions else []
533     tree = converter(
534         draft_f.read(),
535         draftpath,
536         cssfile=cssfile,
537         related_to_html_only=related_to_html_only,
538         tempdir=tempdir,
539         extensions=extensions,
540     )
541
542     mimetree = MIMETreeDFWalker(debug=debug_walk)
543
544     state = dict(pos=1, tags={}, parts=1)
545
546     def visitor_fn(item, ancestry, *, debugprint=None):
547         """
548         Visitor function called for every node (part) of the MIME tree,
549         depth-first, and responsible for telling NeoMutt how to assemble
550         the tree.
551         """
552         KILL_LINE = r"\Ca\Ck"
553
554         if isinstance(item, Part):
555             # We've hit a leaf-node, i.e. an alternative or a related part
556             # with actual content.
557
558             # Let's add the part
559             if item.orig:
560                 # The original source already exists in the NeoMutt tree, but
561                 # the underlying file may have been modified, so we need to
562                 # update the encoding, but that's it:
563                 cmds.push("<first-entry>")
564                 cmds.push("<update-encoding>")
565
566                 # We really just need to be able to assume that at this point,
567                 # NeoMutt is at position 1, and that we've processed only this
568                 # part so far. Nevermind about actual attachments, we can
569                 # safely ignore those as they stay at the end.
570                 assert state["pos"] == 1
571                 assert state["parts"] == 1
572             else:
573                 # … whereas all other parts need to be added, and they're all
574                 # considered to be temporary and inline:
575                 cmds.push(f"<attach-file>{item.path}<enter>")
576                 cmds.push("<toggle-unlink><toggle-disposition>")
577
578                 # This added a part at the end of the list of parts, and that's
579                 # just how many parts we've seen so far, so it's position in
580                 # the NeoMutt compose list is the count of parts
581                 state["parts"] += 1
582                 state["pos"] = state["parts"]
583
584             # If the item (including the original) comes with additional
585             # information, then we might just as well update the NeoMutt
586             # tree now:
587             if item.cid:
588                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
589
590         elif isinstance(item, Multipart):
591             # This node has children, but we already visited them (see
592             # above). The tags dictionary of State should contain a list of
593             # their positions in the NeoMutt compose window, so iterate those
594             # and tag the parts there:
595             for tag in state["tags"][item]:
596                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
597
598             if item.subtype == "alternative":
599                 cmds.push("<group-alternatives>")
600             elif item.subtype in ("relative", "related"):
601                 cmds.push("<group-related>")
602             elif item.subtype == "multilingual":
603                 cmds.push("<group-multilingual>")
604             else:
605                 raise NotImplementedError(
606                     f"Handling of multipart/{item.subtype} is not implemented"
607                 )
608
609             state["pos"] -= len(state["tags"][item]) - 1
610             state["parts"] += 1
611             del state["tags"][item]
612
613         else:
614             # We should never get here
615             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
616
617         # If the item has a description, we might just as well add it
618         if item.desc:
619             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
620
621         if ancestry:
622             # If there's an ancestry, record the current (assumed) position in
623             # the NeoMutt compose window as needed-to-tag by our direct parent
624             # (i.e. the last item of the ancestry)
625             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
626
627             lead = "│ " * (len(ancestry) + 1) + "* "
628             debugprint(
629                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
630                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
631                 f"{lead}pos={state['pos']}, parts={state['parts']}"
632             )
633
634     # -----------------
635     # End of visitor_fn
636
637     # Let's walk the tree and visit every node with our fancy visitor
638     # function
639     mimetree.walk(tree, visitor_fn=visitor_fn)
640
641     if not only_build:
642         cmds.push("<send-message>")
643
644     # Finally, cleanup. Since we're responsible for removing the temporary
645     # file, how's this for a little hack?
646     try:
647         filename = cmd_f.name
648     except AttributeError:
649         filename = "pytest_internal_file"
650     cmds.cmd(f"source 'rm -f {filename}|'")
651     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
652     cmds.flush()
653
654
655 # [ CLI ENTRY ] ###############################################################
656
657 if __name__ == "__main__":
658     args = parse_cli_args()
659
660     if args.mode is None:
661         do_setup(
662             tempdir=args.tempdir,
663             debug_commands=args.debug_commands,
664         )
665
666     elif args.mode == "massage":
667         with open(args.MAILDRAFT, "r") as draft_f, open(
668             args.cmdpath, "w"
669         ) as cmd_f:
670             do_massage(
671                 draft_f,
672                 pathlib.Path(args.MAILDRAFT),
673                 cmd_f,
674                 extensions=args.extensions,
675                 cssfile=args.css_file,
676                 related_to_html_only=args.related_to_html_only,
677                 only_build=args.only_build,
678                 tempdir=args.tempdir,
679                 debug_commands=args.debug_commands,
680                 debug_walk=args.debug_walk,
681             )
682
683
684 # [ TESTS ] ###################################################################
685
686 try:
687     import pytest
688     from io import StringIO
689
690     class Tests:
691         @pytest.fixture
692         def const1(self):
693             return "CONSTANT STRING 1"
694
695         @pytest.fixture
696         def const2(self):
697             return "CONSTANT STRING 2"
698
699         # NOTE: tests using the capsys fixture must specify sys.stdout to the
700         # functions they call, else old stdout is used and not captured
701
702         def test_MuttCommands_cmd(self, const1, const2, capsys):
703             "Assert order of commands"
704             cmds = MuttCommands(out_f=sys.stdout)
705             cmds.cmd(const1)
706             cmds.cmd(const2)
707             cmds.flush()
708             captured = capsys.readouterr()
709             assert captured.out == "\n".join((const1, const2, ""))
710
711         def test_MuttCommands_push(self, const1, const2, capsys):
712             "Assert reverse order of pushes"
713             cmds = MuttCommands(out_f=sys.stdout)
714             cmds.push(const1)
715             cmds.push(const2)
716             cmds.flush()
717             captured = capsys.readouterr()
718             assert (
719                 captured.out
720                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
721             )
722
723         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
724             "Assert reverse order of pushes"
725             cmds = MuttCommands(out_f=sys.stdout)
726             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
727             for i in range(2):
728                 cmds.cmd(lines[4 * i + 0])
729                 cmds.cmd(lines[4 * i + 1])
730                 cmds.push(lines[4 * i + 2])
731                 cmds.push(lines[4 * i + 3])
732             cmds.flush()
733
734             captured = capsys.readouterr()
735             lines_out = captured.out.splitlines()
736             assert lines[0] in lines_out[0]
737             assert lines[1] in lines_out[1]
738             assert lines[7] in lines_out[2]
739             assert lines[6] in lines_out[3]
740             assert lines[3] in lines_out[4]
741             assert lines[2] in lines_out[5]
742             assert lines[4] in lines_out[6]
743             assert lines[5] in lines_out[7]
744
745         @pytest.fixture
746         def mime_tree_related_to_alternative(self):
747             return Multipart(
748                 "relative",
749                 children=[
750                     Multipart(
751                         "alternative",
752                         children=[
753                             Part(
754                                 "text",
755                                 "plain",
756                                 "part.txt",
757                                 desc="Plain",
758                                 orig=True,
759                             ),
760                             Part("text", "html", "part.html", desc="HTML"),
761                         ],
762                         desc="Alternative",
763                     ),
764                     Part(
765                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
766                     ),
767                 ],
768                 desc="Related",
769             )
770
771         @pytest.fixture
772         def mime_tree_related_to_html(self):
773             return Multipart(
774                 "alternative",
775                 children=[
776                     Part(
777                         "text",
778                         "plain",
779                         "part.txt",
780                         desc="Plain",
781                         orig=True,
782                     ),
783                     Multipart(
784                         "relative",
785                         children=[
786                             Part("text", "html", "part.html", desc="HTML"),
787                             Part(
788                                 "text",
789                                 "png",
790                                 "logo.png",
791                                 cid="logo.png",
792                                 desc="Logo",
793                             ),
794                         ],
795                         desc="Related",
796                     ),
797                 ],
798                 desc="Alternative",
799             )
800
801         def test_MIMETreeDFWalker_depth_first_walk(
802             self, mime_tree_related_to_alternative
803         ):
804             mimetree = MIMETreeDFWalker()
805
806             items = []
807
808             def visitor_fn(item, ancestry, debugprint):
809                 items.append((item, len(ancestry)))
810
811             mimetree.walk(
812                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
813             )
814             assert len(items) == 5
815             assert items[0][0].subtype == "plain"
816             assert items[0][1] == 2
817             assert items[1][0].subtype == "html"
818             assert items[1][1] == 2
819             assert items[2][0].subtype == "alternative"
820             assert items[2][1] == 1
821             assert items[3][0].subtype == "png"
822             assert items[3][1] == 1
823             assert items[4][0].subtype == "relative"
824             assert items[4][1] == 0
825
826         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
827             mimetree = MIMETreeDFWalker()
828             items = []
829
830             def visitor_fn(item, ancestry, debugprint):
831                 items.append(item)
832
833             p = Part("text", "plain", const1)
834             mimetree.walk([p], visitor_fn=visitor_fn)
835             assert items[-1].subtype == "plain"
836             mimetree.walk([p, p], visitor_fn=visitor_fn)
837             assert items[-1].subtype == "mixed"
838
839         def test_MIMETreeDFWalker_visitor_in_constructor(
840             self, mime_tree_related_to_alternative
841         ):
842             items = []
843
844             def visitor_fn(item, ancestry, debugprint):
845                 items.append(item)
846
847             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
848             mimetree.walk(mime_tree_related_to_alternative)
849             assert len(items) == 5
850
851         @pytest.fixture
852         def string_io(self, const1, text=None):
853             return StringIO(text or const1)
854
855         def test_do_massage_basic(self, const1, string_io, capsys):
856             def converter(
857                 drafttext,
858                 draftpath,
859                 cssfile,
860                 related_to_html_only,
861                 extensions,
862                 tempdir,
863             ):
864                 return Part("text", "plain", draftpath, orig=True)
865
866             do_massage(
867                 draft_f=string_io,
868                 draftpath=const1,
869                 cmd_f=sys.stdout,
870                 converter=converter,
871             )
872
873             captured = capsys.readouterr()
874             lines = captured.out.splitlines()
875             assert '="$my_editor"' in lines.pop(0)
876             assert '="$my_edit_headers"' in lines.pop(0)
877             assert "unset my_editor" == lines.pop(0)
878             assert "unset my_edit_headers" == lines.pop(0)
879             assert "send-message" in lines.pop(0)
880             assert "update-encoding" in lines.pop(0)
881             assert "first-entry" in lines.pop(0)
882             assert "source 'rm -f " in lines.pop(0)
883             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
884
885         def test_do_massage_fulltree(
886             self, string_io, const1, mime_tree_related_to_alternative, capsys
887         ):
888             def converter(
889                 drafttext,
890                 draftpath,
891                 cssfile,
892                 related_to_html_only,
893                 extensions,
894                 tempdir,
895             ):
896                 return mime_tree_related_to_alternative
897
898             do_massage(
899                 draft_f=string_io,
900                 draftpath=const1,
901                 cmd_f=sys.stdout,
902                 converter=converter,
903             )
904
905             captured = capsys.readouterr()
906             lines = captured.out.splitlines()[4:-2]
907             assert "first-entry" in lines.pop()
908             assert "update-encoding" in lines.pop()
909             assert "Plain" in lines.pop()
910             assert "part.html" in lines.pop()
911             assert "toggle-unlink" in lines.pop()
912             assert "HTML" in lines.pop()
913             assert "jump>1" in lines.pop()
914             assert "jump>2" in lines.pop()
915             assert "group-alternatives" in lines.pop()
916             assert "Alternative" in lines.pop()
917             assert "logo.png" in lines.pop()
918             assert "toggle-unlink" in lines.pop()
919             assert "content-id" in lines.pop()
920             assert "Logo" in lines.pop()
921             assert "jump>1" in lines.pop()
922             assert "jump>4" in lines.pop()
923             assert "group-related" in lines.pop()
924             assert "Related" in lines.pop()
925             assert "send-message" in lines.pop()
926             assert len(lines) == 0
927
928         @pytest.fixture
929         def fake_filewriter(self):
930             class FileWriter:
931                 def __init__(self):
932                     self._writes = []
933
934                 def __call__(self, path, content, mode="w", **kwargs):
935                     self._writes.append((path, content))
936
937                 def pop(self, index=-1):
938                     return self._writes.pop(index)
939
940             return FileWriter()
941
942         @pytest.fixture
943         def markdown_non_converter(self, const1, const2):
944             return lambda s, text: f"{const1}{text}{const2}"
945
946         def test_converter_tree_basic(self, const1, const2, fake_filewriter):
947             path = pathlib.Path(const2)
948             tree = convert_markdown_to_html(
949                 const1, path, filewriter_fn=fake_filewriter
950             )
951
952             assert tree.subtype == "alternative"
953             assert len(tree.children) == 2
954             assert tree.children[0].subtype == "plain"
955             assert tree.children[0].path == path
956             assert tree.children[0].orig
957             assert tree.children[1].subtype == "html"
958             assert tree.children[1].path == path.with_suffix(".html")
959
960         def test_converter_writes(
961             self,
962             const1,
963             const2,
964             fake_filewriter,
965             monkeypatch,
966             markdown_non_converter,
967         ):
968             path = pathlib.Path(const2)
969
970             with monkeypatch.context() as m:
971                 m.setattr(markdown.Markdown, "convert", markdown_non_converter)
972                 convert_markdown_to_html(
973                     const1, path, filewriter_fn=fake_filewriter
974                 )
975
976             assert (path, const1) == fake_filewriter.pop(0)
977             assert (
978                 path.with_suffix(".html"),
979                 markdown_non_converter(None, const1),
980             ) == fake_filewriter.pop(0)
981
982         def test_markdown_inline_image_processor(self):
983             imgpath1 = "file:/path/to/image.png"
984             imgpath2 = "file:///path/to/image.png?url=params"
985             imgpath3 = "/path/to/image.png"
986             text = f"""![inline local image]({imgpath1})
987                        ![image inlined
988                          with newline]({imgpath2})
989                        ![image local path]({imgpath3})"""
990             text, html, images = markdown_with_inline_image_support(text)
991
992             # local paths have been normalised to URLs:
993             imgpath3 = f"file://{imgpath3}"
994
995             assert 'src="cid:' in html
996             assert "](cid:" in text
997             assert len(images) == 3
998             assert imgpath1 in images
999             assert imgpath2 in images
1000             assert imgpath3 in images
1001             assert images[imgpath1].cid != images[imgpath2].cid
1002             assert images[imgpath1].cid != images[imgpath3].cid
1003             assert images[imgpath2].cid != images[imgpath3].cid
1004
1005         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1006             imgpath = "file:///path/to/image.png"
1007             text = f'![inline local image]({imgpath} "{const1}")'
1008             text, html, images = markdown_with_inline_image_support(text)
1009             assert images[imgpath].desc == const1
1010
1011         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1012             imgpath = "file:///path/to/image.png"
1013             text = f"![{const1}]({imgpath})"
1014             text, html, images = markdown_with_inline_image_support(text)
1015             assert images[imgpath].desc == const1
1016
1017         def test_markdown_inline_image_processor_title_over_alt_desc(
1018             self, const1, const2
1019         ):
1020             imgpath = "file:///path/to/image.png"
1021             text = f'![{const1}]({imgpath} "{const2}")'
1022             text, html, images = markdown_with_inline_image_support(text)
1023             assert images[imgpath].desc == const2
1024
1025         def test_markdown_inline_image_not_external(self):
1026             imgpath = "https://path/to/image.png"
1027             text = f"![inline image]({imgpath})"
1028             text, html, images = markdown_with_inline_image_support(text)
1029
1030             assert 'src="cid:' not in html
1031             assert "](cid:" not in text
1032             assert len(images) == 0
1033
1034         def test_markdown_inline_image_local_file(self):
1035             imgpath = "/path/to/image.png"
1036             text = f"![inline image]({imgpath})"
1037             text, html, images = markdown_with_inline_image_support(text)
1038
1039             for k, v in images.items():
1040                 assert k == f"file://{imgpath}"
1041                 break
1042
1043         @pytest.fixture
1044         def test_png(self):
1045             return (
1046                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1047                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1048             )
1049
1050         def test_markdown_inline_image_processor_base64(self, test_png):
1051             text = f"![1px white inlined]({test_png})"
1052             text, html, images = markdown_with_inline_image_support(text)
1053
1054             assert 'src="cid:' in html
1055             assert "](cid:" in text
1056             assert len(images) == 1
1057             assert test_png in images
1058
1059         def test_converter_tree_inline_image_base64(
1060             self, test_png, const1, fake_filewriter
1061         ):
1062             text = f"![inline base64 image]({test_png})"
1063             path = pathlib.Path(const1)
1064             tree = convert_markdown_to_html(
1065                 text,
1066                 path,
1067                 filewriter_fn=fake_filewriter,
1068                 related_to_html_only=False,
1069             )
1070             assert tree.subtype == "relative"
1071             assert tree.children[0].subtype == "alternative"
1072             assert tree.children[1].subtype == "png"
1073             written = fake_filewriter.pop()
1074             assert tree.children[1].path == written[0]
1075             assert written[1] == request.urlopen(test_png).read()
1076
1077         def test_converter_tree_inline_image_base64_related_to_html(
1078             self, test_png, const1, fake_filewriter
1079         ):
1080             text = f"![inline base64 image]({test_png})"
1081             path = pathlib.Path(const1)
1082             tree = convert_markdown_to_html(
1083                 text,
1084                 path,
1085                 filewriter_fn=fake_filewriter,
1086                 related_to_html_only=True,
1087             )
1088             assert tree.subtype == "alternative"
1089             assert tree.children[1].subtype == "relative"
1090             assert tree.children[1].children[1].subtype == "png"
1091             written = fake_filewriter.pop()
1092             assert tree.children[1].children[1].path == written[0]
1093             assert written[1] == request.urlopen(test_png).read()
1094
1095         def test_converter_tree_inline_image_cid(
1096             self, const1, fake_filewriter
1097         ):
1098             text = f"![inline base64 image](cid:{const1})"
1099             path = pathlib.Path(const1)
1100             tree = convert_markdown_to_html(
1101                 text,
1102                 path,
1103                 filewriter_fn=fake_filewriter,
1104                 related_to_html_only=False,
1105             )
1106             assert len(tree.children) == 2
1107             assert tree.children[0].cid != const1
1108             assert tree.children[0].type != "image"
1109             assert tree.children[1].cid != const1
1110             assert tree.children[1].type != "image"
1111
1112         def test_inline_image_collection(
1113             self, test_png, const1, const2, fake_filewriter
1114         ):
1115             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1116             relparts = collect_inline_images(
1117                 test_images, filewriter_fn=fake_filewriter
1118             )
1119
1120             written = fake_filewriter.pop()
1121             assert b"PNG" in written[1]
1122
1123             assert relparts[0].subtype == "png"
1124             assert relparts[0].path == written[0]
1125             assert relparts[0].cid == const1
1126             assert relparts[0].desc.endswith(const2)
1127
1128         def test_apply_stylesheet(self):
1129             if _PYNLINER:
1130                 html = "<p>Hello, world!</p>"
1131                 css = "p { color:red }"
1132                 out = apply_styling(html, css)
1133                 assert 'p style="color' in out
1134
1135         def test_apply_stylesheet_pygments(self):
1136             if _PYGMENTS_CSS:
1137                 html = (
1138                     f'<div class="{_CODEHILITE_CLASS}">'
1139                     "<pre>def foo():\n    return</pre></div>"
1140                 )
1141                 out = apply_styling(html, _PYGMENTS_CSS)
1142                 assert f'{_CODEHILITE_CLASS}" style="' in out
1143
1144         def test_mime_tree_relative_within_alternative(
1145             self, string_io, const1, capsys, mime_tree_related_to_html
1146         ):
1147             def converter(
1148                 drafttext,
1149                 draftpath,
1150                 cssfile,
1151                 related_to_html_only,
1152                 extensions,
1153                 tempdir,
1154             ):
1155                 return mime_tree_related_to_html
1156
1157             do_massage(
1158                 draft_f=string_io,
1159                 draftpath=const1,
1160                 cmd_f=sys.stdout,
1161                 converter=converter,
1162             )
1163
1164             captured = capsys.readouterr()
1165             lines = captured.out.splitlines()[4:-2]
1166             assert "first-entry" in lines.pop()
1167             assert "update-encoding" in lines.pop()
1168             assert "Plain" in lines.pop()
1169             assert "part.html" in lines.pop()
1170             assert "toggle-unlink" in lines.pop()
1171             assert "HTML" in lines.pop()
1172             assert "logo.png" in lines.pop()
1173             assert "toggle-unlink" in lines.pop()
1174             assert "content-id" in lines.pop()
1175             assert "Logo" in lines.pop()
1176             assert "jump>2" in lines.pop()
1177             assert "jump>3" in lines.pop()
1178             assert "group-related" in lines.pop()
1179             assert "Related" in lines.pop()
1180             assert "jump>1" in lines.pop()
1181             assert "jump>2" in lines.pop()
1182             assert "group-alternative" in lines.pop()
1183             assert "Alternative" in lines.pop()
1184             assert "send-message" in lines.pop()
1185             assert len(lines) == 0
1186
1187 except ImportError:
1188     pass