]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

buildmimetree.py: support modifying the original markdown source
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     macro compose B "\
10 #       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
11 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
12 #     " "Convert message into a modern MIME tree with inline images"
13 #
14 #     (Yes, we need to call source twice, as mutt only starts to process output
15 #     from a source command when the command exits, and since we need to react
16 #     to the output, we need to be invoked again, using a $my_ variable to pass
17 #     information)
18 #
19 # Requirements:
20 #   - python3
21 #   - python3-markdown
22 # Optional:
23 #   - pytest
24 #   - Pynliner
25 #   - Pygments, if installed, then syntax highlighting is enabled
26 #
27 # Latest version:
28 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
29 #
30 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
31 # Released under the GPL-2+ licence, just like Mutt itself.
32 #
33
34 import sys
35 import pathlib
36 import markdown
37 import tempfile
38 import argparse
39 from collections import namedtuple
40
41
42 def parse_cli_args(*args, **kwargs):
43     parser = argparse.ArgumentParser(
44         description=(
45             "NeoMutt helper to turn text/markdown email parts "
46             "into full-fledged MIME trees"
47         )
48     )
49     parser.epilog = (
50         "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
51         "Released under the MIT licence"
52     )
53
54     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
55     parser_setup = subp.add_parser("setup", help="Setup phase")
56     parser_massage = subp.add_parser("massage", help="Massaging phase")
57
58     parser_setup.add_argument(
59         "--debug-commands",
60         action="store_true",
61         help="Turn on debug logging of commands generated to stderr",
62     )
63
64     parser_setup.add_argument(
65         "--extension",
66         "-x",
67         metavar="EXTENSION",
68         dest="extensions",
69         nargs="?",
70         default=[],
71         action="append",
72         help="Markdown extension to add to the list of extensions use",
73     )
74
75     parser_setup.add_argument(
76         "--send-message",
77         action="store_true",
78         help="Generate command(s) to send the message after processing",
79     )
80
81     parser_massage.add_argument(
82         "--debug-commands",
83         action="store_true",
84         help="Turn on debug logging of commands generated to stderr",
85     )
86
87     parser_massage.add_argument(
88         "--debug-walk",
89         action="store_true",
90         help="Turn on debugging to stderr of the MIME tree walk",
91     )
92
93     parser_massage.add_argument(
94         "--extensions",
95         metavar="EXTENSIONS",
96         type=str,
97         default="",
98         help="Markdown extension to use (comma-separated list)",
99     )
100
101     parser_massage.add_argument(
102         "--write-commands-to",
103         metavar="PATH",
104         dest="cmdpath",
105         help="Temporary file path to write commands to",
106     )
107
108     parser_massage.add_argument(
109         "MAILDRAFT",
110         nargs="?",
111         help="If provided, the script is invoked as editor on the mail draft",
112     )
113
114     return parser.parse_args(*args, **kwargs)
115
116
117 # [ PARTS GENERATION ] ########################################################
118
119
120 class Part(
121     namedtuple(
122         "Part",
123         ["type", "subtype", "path", "desc", "cid", "orig"],
124         defaults=[None, None, False],
125     )
126 ):
127     def __str__(self):
128         ret = f"<{self.type}/{self.subtype}>"
129         if self.cid:
130             ret = f"{ret} cid:{self.cid}"
131         if self.orig:
132             ret = f"{ret} ORIGINAL"
133         return ret
134
135
136 class Multipart(
137     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
138 ):
139     def __str__(self):
140         return f"<multipart/{self.subtype}> children={len(self.children)}"
141
142
143 def convert_markdown_to_html(maildraft, *, extensions=None):
144     draftpath = pathlib.Path(maildraft)
145     with open(draftpath, "r", encoding="utf-8") as textmarkdown:
146         text = textmarkdown.read()
147
148     mdwn = markdown.Markdown(extensions=extensions)
149
150
151     with open(draftpath, "w", encoding="utf-8") as textplain:
152         textplain.write(text)
153     textpart = Part(
154         "text", "plain", draftpath, "Plain-text version", orig=True
155     )
156
157     html = mdwn.convert(text)
158     htmlpath = draftpath.with_suffix(".html")
159     with open(
160         htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
161     ) as texthtml:
162         texthtml.write(html)
163     htmlpart = Part("text", "html", htmlpath, "HTML version")
164
165     logopart = Part(
166         "image",
167         "png",
168         "/usr/share/doc/neomutt/logo/neomutt-256.png",
169         "Logo",
170         "neomutt-256.png",
171     )
172
173     return Multipart(
174         "relative",
175         [
176             Multipart(
177                 "alternative",
178                 [textpart, htmlpart],
179                 "Group of alternative content",
180             ),
181             logopart,
182         ],
183         "Group of related content",
184     )
185
186
187 class MIMETreeDFWalker:
188     def __init__(self, *, visitor_fn=None, debug=False):
189         self._visitor_fn = visitor_fn
190         self._debug = debug
191
192     def walk(self, root, *, visitor_fn=None):
193         """
194         Recursive function to implement a depth-dirst walk of the MIME-tree
195         rooted at `root`.
196         """
197
198         if isinstance(root, list):
199             root = Multipart("mixed", children=root)
200
201         self._walk(
202             root,
203             stack=[],
204             visitor_fn=visitor_fn or self._visitor_fn,
205         )
206
207     def _walk(self, node, *, stack, visitor_fn):
208         # Let's start by enumerating the parts at the current level. At the
209         # root level, stack will be the empty list, and we expect a multipart/*
210         # container at this level. Later, e.g. within a mutlipart/alternative
211         # container, the subtree will just be the alternative parts, while the
212         # top of the stack will be the multipart/alternative container, which
213         # we will process after the following loop.
214
215         lead = f"{'| '*len(stack)}|-"
216         if isinstance(node, Multipart):
217             self.debugprint(
218                 f"{lead}{node} parents={[s.subtype for s in stack]}"
219             )
220
221             # Depth-first, so push the current container onto the stack,
222             # then descend …
223             stack.append(node)
224             self.debugprint("| " * (len(stack) + 1))
225             for child in node.children:
226                 self._walk(
227                     child,
228                     stack=stack,
229                     visitor_fn=visitor_fn,
230                 )
231             self.debugprint("| " * len(stack))
232             assert stack.pop() == node
233
234         else:
235             self.debugprint(f"{lead}{node}")
236
237         if visitor_fn:
238             visitor_fn(node, stack, debugprint=self.debugprint)
239
240     def debugprint(self, s, **kwargs):
241         if self._debug:
242             print(s, file=sys.stderr, **kwargs)
243
244
245 # [ RUN MODES ] ###############################################################
246
247
248 class MuttCommands:
249     """
250     Stupid class to interface writing out Mutt commands. This is quite a hack
251     to deal with the fact that Mutt runs "push" commands in reverse order, so
252     all of a sudden, things become very complicated when mixing with "real"
253     commands.
254
255     Hence we keep two sets of commands, and one set of pushes. Commands are
256     added to the first until a push is added, after which commands are added to
257     the second set of commands.
258
259     On flush(), the first set is printed, followed by the pushes in reverse,
260     and then the second set is printed. All 3 sets are then cleared.
261     """
262
263     def __init__(self, out_f=sys.stdout, *, debug=False):
264         self._cmd1, self._push, self._cmd2 = [], [], []
265         self._out_f = out_f
266         self._debug = debug
267
268     def cmd(self, s):
269         self.debugprint(s)
270         if self._push:
271             self._cmd2.append(s)
272         else:
273             self._cmd1.append(s)
274
275     def push(self, s):
276         s = s.replace('"', '"')
277         s = f'push "{s}"'
278         self.debugprint(s)
279         self._push.insert(0, s)
280
281     def flush(self):
282         print(
283             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
284         )
285         self._cmd1, self._push, self._cmd2 = [], [], []
286
287     def debugprint(self, s, **kwargs):
288         if self._debug:
289             print(s, file=sys.stderr, **kwargs)
290
291
292 def do_setup(
293     extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
294 ):
295     extensions = extensions or []
296     temppath = temppath or pathlib.Path(
297         tempfile.mkstemp(prefix="muttmdwn-")[1]
298     )
299     cmds = MuttCommands(out_f, debug=debug_commands)
300
301     editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
302     if extensions:
303         editor = f'{editor} --extensions {",".join(extensions)}'
304
305     cmds.cmd('set my_editor="$editor"')
306     cmds.cmd('set my_edit_headers="$edit_headers"')
307     cmds.cmd(f'set editor="{editor}"')
308     cmds.cmd("unset edit_headers")
309     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
310     cmds.push("<first-entry><edit-file>")
311     cmds.flush()
312
313
314 def do_massage(
315     maildraft,
316     cmd_f,
317     *,
318     extensions=None,
319     converter=convert_markdown_to_html,
320     debug_commands=False,
321     debug_walk=False,
322 ):
323     # Here's the big picture: we're being invoked as the editor on the email
324     # draft, and whatever commands we write to the file given as cmdpath will
325     # be run by the second source command in the macro definition.
326
327     # Let's start by cleaning up what the setup did (see above), i.e. we
328     # restore the $editor and $edit_headers variables, and also unset the
329     # variable used to identify the command file we're currently writing
330     # to.
331     cmds = MuttCommands(cmd_f, debug=debug_commands)
332     cmds.cmd('set editor="$my_editor"')
333     cmds.cmd('set edit_headers="$my_edit_headers"')
334     cmds.cmd("unset my_editor")
335     cmds.cmd("unset my_edit_headers")
336
337     # let's flush those commands, as there'll be a lot of pushes from now
338     # on, which need to be run in reverse order
339     cmds.flush()
340
341     extensions = extensions.split(",") if extensions else []
342     tree = converter(maildraft, extensions=extensions)
343
344     mimetree = MIMETreeDFWalker(debug=debug_walk)
345
346     def visitor_fn(item, stack, *, debugprint=None):
347         """
348         Visitor function called for every node (part) of the MIME tree,
349         depth-first, and responsible for telling NeoMutt how to assemble
350         the tree.
351         """
352         if isinstance(item, Part):
353             # We've hit a leaf-node, i.e. an alternative or a related part
354             # with actual content.
355
356             # Let's add the part
357             if item.orig:
358                 # The original source already exists in the NeoMutt tree, but
359                 # the underlying file may have been modified, so we need to
360                 # update the encoding, but that's it:
361                 cmds.push("<update-encoding>")
362             else:
363                 # … whereas all other parts need to be added, and they're all
364                 # considered to be temporary and inline:
365                 cmds.push(f"<attach-file>{item.path}<enter>")
366                 cmds.push("<toggle-unlink><toggle-disposition>")
367
368             # If the item (including the original) comes with additional
369             # information, then we might just as well update the NeoMutt
370             # tree now:
371             if item.cid:
372                 cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
373             if item.desc:
374                 cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
375
376         elif isinstance(item, Multipart):
377             # This node has children, but we already visited them (see
378             # above), and so they have been tagged in NeoMutt's compose
379             # window. Now it's just a matter of telling NeoMutt to do the
380             # appropriate grouping:
381             if item.subtype == "alternative":
382                 cmds.push("<group-alternatives>")
383             elif item.subtype == "relative":
384                 cmds.push("<group-related>")
385             elif item.subtype == "multilingual":
386                 cmds.push("<group-multilingual>")
387
388             # Again, if there is a description, we might just as well:
389             if item.desc:
390                 cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
391
392         else:
393             # We should never get here
394             assert not "is valid part"
395
396         # Finally, if we're at non-root level, tag the new container,
397         # as it might itself be part of a container, to be processed
398         # one level up:
399         if stack:
400             cmds.push("<tag-entry>")
401
402     # -----------------
403     # End of visitor_fn
404
405     # Let's walk the tree and visit every node with our fancy visitor
406     # function
407     mimetree.walk(tree, visitor_fn=visitor_fn)
408
409     # Finally, cleanup. Since we're responsible for removing the temporary
410     # file, how's this for a little hack?
411     try:
412         filename = cmd_f.name
413     except AttributeError:
414         filename = "pytest_internal_file"
415     cmds.cmd(f"source 'rm -f {filename}|'")
416     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
417     cmds.flush()
418
419
420 # [ CLI ENTRY ] ###############################################################
421
422 if __name__ == "__main__":
423     args = parse_cli_args()
424
425     if args.mode == "setup":
426         if args.send_message:
427             raise NotImplementedError()
428
429         do_setup(args.extensions, debug_commands=args.debug_commands)
430
431     elif args.mode == "massage":
432         with open(args.cmdpath, "w") as cmd_f:
433             do_massage(
434                 args.MAILDRAFT,
435                 cmd_f,
436                 extensions=args.extensions,
437                 debug_commands=args.debug_commands,
438                 debug_walk=args.debug_walk,
439             )
440
441
442 # [ TESTS ] ###################################################################
443
444 try:
445     import pytest
446
447     class Tests:
448         @pytest.fixture
449         def const1(self):
450             return "CONSTANT STRING 1"
451
452         @pytest.fixture
453         def const2(self):
454             return "CONSTANT STRING 2"
455
456         # NOTE: tests using the capsys fixture must specify sys.stdout to the
457         # functions they call, else old stdout is used and not captured
458
459         def test_MuttCommands_cmd(self, const1, const2, capsys):
460             "Assert order of commands"
461             cmds = MuttCommands(out_f=sys.stdout)
462             cmds.cmd(const1)
463             cmds.cmd(const2)
464             cmds.flush()
465             captured = capsys.readouterr()
466             assert captured.out == "\n".join((const1, const2, ""))
467
468         def test_MuttCommands_push(self, const1, const2, capsys):
469             "Assert reverse order of pushes"
470             cmds = MuttCommands(out_f=sys.stdout)
471             cmds.push(const1)
472             cmds.push(const2)
473             cmds.flush()
474             captured = capsys.readouterr()
475             assert (
476                 captured.out
477                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
478             )
479
480         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
481             "Assert reverse order of pushes"
482             cmds = MuttCommands(out_f=sys.stdout)
483             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
484             for i in range(2):
485                 cmds.cmd(lines[4 * i + 0])
486                 cmds.cmd(lines[4 * i + 1])
487                 cmds.push(lines[4 * i + 2])
488                 cmds.push(lines[4 * i + 3])
489             cmds.flush()
490
491             captured = capsys.readouterr()
492             lines_out = captured.out.splitlines()
493             assert lines[0] in lines_out[0]
494             assert lines[1] in lines_out[1]
495             assert lines[7] in lines_out[2]
496             assert lines[6] in lines_out[3]
497             assert lines[3] in lines_out[4]
498             assert lines[2] in lines_out[5]
499             assert lines[4] in lines_out[6]
500             assert lines[5] in lines_out[7]
501
502         @pytest.fixture
503         def basic_mime_tree(self):
504             return Multipart(
505                 "relative",
506                 children=[
507                     Multipart(
508                         "alternative",
509                         children=[
510                             Part(
511                                 "text",
512                                 "plain",
513                                 "part.txt",
514                                 desc="Plain",
515                                 orig=True,
516                             ),
517                             Part("text", "html", "part.html", desc="HTML"),
518                         ],
519                         desc="Alternative",
520                     ),
521                     Part(
522                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
523                     ),
524                 ],
525                 desc="Related",
526             )
527
528         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
529             mimetree = MIMETreeDFWalker()
530
531             items = []
532
533             def visitor_fn(item, stack, debugprint):
534                 items.append((item, len(stack)))
535
536             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
537             assert len(items) == 5
538             assert items[0][0].subtype == "plain"
539             assert items[0][1] == 2
540             assert items[1][0].subtype == "html"
541             assert items[1][1] == 2
542             assert items[2][0].subtype == "alternative"
543             assert items[2][1] == 1
544             assert items[3][0].subtype == "png"
545             assert items[3][1] == 1
546             assert items[4][0].subtype == "relative"
547             assert items[4][1] == 0
548
549         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
550             mimetree = MIMETreeDFWalker()
551             items = []
552
553             def visitor_fn(item, stack, debugprint):
554                 items.append(item)
555
556             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
557             assert items[-1].subtype == "mixed"
558
559         def test_MIMETreeDFWalker_visitor_in_constructor(
560             self, basic_mime_tree
561         ):
562             items = []
563
564             def visitor_fn(item, stack, debugprint):
565                 items.append(item)
566
567             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
568             mimetree.walk(basic_mime_tree)
569             assert len(items) == 5
570
571         def test_do_setup_no_extensions(self, const1, capsys):
572             "Assert basics about the setup command output"
573             do_setup(temppath=const1, out_f=sys.stdout)
574             captout = capsys.readouterr()
575             lines = captout.out.splitlines()
576             assert lines[2].endswith(f'{const1}"')
577             assert lines[4].endswith(const1)
578             assert "first-entry" in lines[-1]
579             assert "edit-file" in lines[-1]
580
581         def test_do_setup_extensions(self, const1, const2, capsys):
582             "Assert that extensions are passed to editor"
583             do_setup(
584                 temppath=const1, extensions=[const2, const1], out_f=sys.stdout
585             )
586             captout = capsys.readouterr()
587             lines = captout.out.splitlines()
588             # assert comma-separated list of extensions passed
589             assert lines[2].endswith(f'{const2},{const1}"')
590             assert lines[4].endswith(const1)
591
592         def test_do_massage_basic(self, const1, capsys):
593             def converter(maildraft, extensions):
594                 return Part("text", "plain", "/dev/null", orig=True)
595
596             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
597             captured = capsys.readouterr()
598             assert (
599                 captured.out.strip()
600                 == """\
601             set editor="$my_editor"
602             set edit_headers="$my_edit_headers"
603             unset my_editor
604             unset my_edit_headers
605             source 'rm -f pytest_internal_file|'
606             unset my_mdwn_postprocess_cmd_file
607             """.replace(
608                     "            ", ""
609                 ).strip()
610             )
611
612         def test_do_massage_fulltree(self, const1, basic_mime_tree, capsys):
613             def converter(maildraft, extensions):
614                 return basic_mime_tree
615
616             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
617             captured = capsys.readouterr()
618             lines = captured.out.splitlines()[4:][::-1]
619             assert "Related" in lines.pop()
620             assert "group-related" in lines.pop()
621             assert "tag-entry" in lines.pop()
622             assert "Logo" in lines.pop()
623             assert "content-id" in lines.pop()
624             assert "toggle-unlink" in lines.pop()
625             assert "logo.png" in lines.pop()
626             assert "tag-entry" in lines.pop()
627             assert "Alternative" in lines.pop()
628             assert "group-alternatives" in lines.pop()
629             assert "tag-entry" in lines.pop()
630             assert "HTML" in lines.pop()
631             assert "toggle-unlink" in lines.pop()
632             assert "part.html" in lines.pop()
633             assert "tag-entry" in lines.pop()
634             assert "Plain" in lines.pop()
635
636 except ImportError:
637     pass