]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

buildmimetree.py: --send-message base work
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     macro compose B "\
10 #       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
11 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
12 #     " "Convert message into a modern MIME tree with inline images"
13 #
14 #     (Yes, we need to call source twice, as mutt only starts to process output
15 #     from a source command when the command exits, and since we need to react
16 #     to the output, we need to be invoked again, using a $my_ variable to pass
17 #     information)
18 #
19 # Requirements:
20 #   - python3
21 #   - python3-markdown
22 # Optional:
23 #   - pytest
24 #   - Pynliner
25 #   - Pygments, if installed, then syntax highlighting is enabled
26 #
27 # Latest version:
28 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
29 #
30 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
31 # Released under the GPL-2+ licence, just like Mutt itself.
32 #
33
34 import sys
35 import pathlib
36 import markdown
37 import tempfile
38 import argparse
39 from collections import namedtuple
40
41
42 def parse_cli_args(*args, **kwargs):
43     parser = argparse.ArgumentParser(
44         description=(
45             "NeoMutt helper to turn text/markdown email parts "
46             "into full-fledged MIME trees"
47         )
48     )
49     parser.epilog = (
50         "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
51         "Released under the MIT licence"
52     )
53
54     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
55     parser_setup = subp.add_parser("setup", help="Setup phase")
56     parser_massage = subp.add_parser("massage", help="Massaging phase")
57
58     parser_setup.add_argument(
59         "--debug-commands",
60         action="store_true",
61         help="Turn on debug logging of commands generated to stderr",
62     )
63
64     parser_setup.add_argument(
65         "--extension",
66         "-x",
67         metavar="EXTENSION",
68         dest="extensions",
69         nargs="?",
70         default=[],
71         action="append",
72         help="Markdown extension to add to the list of extensions use",
73     )
74
75     parser_setup.add_argument(
76         "--send-message",
77         action="store_true",
78         help="Generate command(s) to send the message after processing",
79     )
80
81     parser_massage.add_argument(
82         "--debug-commands",
83         action="store_true",
84         help="Turn on debug logging of commands generated to stderr",
85     )
86
87     parser_massage.add_argument(
88         "--debug-walk",
89         action="store_true",
90         help="Turn on debugging to stderr of the MIME tree walk",
91     )
92
93     parser_massage.add_argument(
94         "--extensions",
95         metavar="EXTENSIONS",
96         type=str,
97         default="",
98         help="Markdown extension to use (comma-separated list)",
99     )
100
101     parser_massage.add_argument(
102         "--write-commands-to",
103         metavar="PATH",
104         dest="cmdpath",
105         help="Temporary file path to write commands to",
106     )
107
108     parser_massage.add_argument(
109         "MAILDRAFT",
110         nargs="?",
111         help="If provided, the script is invoked as editor on the mail draft",
112     )
113
114     return parser.parse_args(*args, **kwargs)
115
116
117 # [ PARTS GENERATION ] ########################################################
118
119
120 class Part(
121     namedtuple(
122         "Part",
123         ["type", "subtype", "path", "desc", "cid", "orig"],
124         defaults=[None, None, False],
125     )
126 ):
127     def __str__(self):
128         ret = f"<{self.type}/{self.subtype}>"
129         if self.cid:
130             ret = f"{ret} cid:{self.cid}"
131         if self.orig:
132             ret = f"{ret} ORIGINAL"
133         return ret
134
135
136 class Multipart(
137     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
138 ):
139     def __str__(self):
140         return f"<multipart/{self.subtype}> children={len(self.children)}"
141
142
143 def convert_markdown_to_html(maildraft, *, extensions=None):
144     draftpath = pathlib.Path(maildraft)
145     textpart = Part(
146         "text", "plain", draftpath, "Plain-text version", orig=True
147     )
148
149     with open(draftpath, "r", encoding="utf-8") as textmarkdown:
150         text = textmarkdown.read()
151
152     mdwn = markdown.Markdown(extensions=extensions)
153     html = mdwn.convert(text)
154
155     htmlpath = draftpath.with_suffix(".html")
156     htmlpart = Part("text", "html", htmlpath, "HTML version")
157
158     with open(
159         htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
160     ) as texthtml:
161         texthtml.write(html)
162
163     logopart = Part(
164         "image",
165         "png",
166         "/usr/share/doc/neomutt/logo/neomutt-256.png",
167         "Logo",
168         "neomutt-256.png",
169     )
170
171     return Multipart(
172         "relative",
173         [
174             Multipart(
175                 "alternative",
176                 [textpart, htmlpart],
177                 "Group of alternative content",
178             ),
179             logopart,
180         ],
181         "Group of related content",
182     )
183
184
185 class MIMETreeDFWalker:
186     def __init__(self, *, visitor_fn=None, debug=False):
187         self._visitor_fn = visitor_fn
188         self._debug = debug
189
190     def walk(self, root, *, visitor_fn=None):
191         """
192         Recursive function to implement a depth-dirst walk of the MIME-tree
193         rooted at `root`.
194         """
195
196         if isinstance(root, list):
197             root = Multipart("mixed", children=root)
198
199         self._walk(
200             root,
201             stack=[],
202             visitor_fn=visitor_fn or self._visitor_fn,
203         )
204
205     def _walk(self, node, *, stack, visitor_fn):
206         # Let's start by enumerating the parts at the current level. At the
207         # root level, stack will be the empty list, and we expect a multipart/*
208         # container at this level. Later, e.g. within a mutlipart/alternative
209         # container, the subtree will just be the alternative parts, while the
210         # top of the stack will be the multipart/alternative container, which
211         # we will process after the following loop.
212
213         lead = f"{'| '*len(stack)}|-"
214         if isinstance(node, Multipart):
215             self.debugprint(
216                 f"{lead}{node} parents={[s.subtype for s in stack]}"
217             )
218
219             # Depth-first, so push the current container onto the stack,
220             # then descend …
221             stack.append(node)
222             self.debugprint("| " * (len(stack) + 1))
223             for child in node.children:
224                 self._walk(
225                     child,
226                     stack=stack,
227                     visitor_fn=visitor_fn,
228                 )
229             self.debugprint("| " * len(stack))
230             assert stack.pop() == node
231
232         else:
233             self.debugprint(f"{lead}{node}")
234
235         if visitor_fn:
236             visitor_fn(node, stack, debugprint=self.debugprint)
237
238     def debugprint(self, s, **kwargs):
239         if self._debug:
240             print(s, file=sys.stderr, **kwargs)
241
242
243 # [ RUN MODES ] ###############################################################
244
245
246 class MuttCommands:
247     """
248     Stupid class to interface writing out Mutt commands. This is quite a hack
249     to deal with the fact that Mutt runs "push" commands in reverse order, so
250     all of a sudden, things become very complicated when mixing with "real"
251     commands.
252
253     Hence we keep two sets of commands, and one set of pushes. Commands are
254     added to the first until a push is added, after which commands are added to
255     the second set of commands.
256
257     On flush(), the first set is printed, followed by the pushes in reverse,
258     and then the second set is printed. All 3 sets are then cleared.
259     """
260
261     def __init__(self, out_f=sys.stdout, *, debug=False):
262         self._cmd1, self._push, self._cmd2 = [], [], []
263         self._out_f = out_f
264         self._debug = debug
265
266     def cmd(self, s):
267         self.debugprint(s)
268         if self._push:
269             self._cmd2.append(s)
270         else:
271             self._cmd1.append(s)
272
273     def push(self, s):
274         s = s.replace('"', '"')
275         s = f'push "{s}"'
276         self.debugprint(s)
277         self._push.insert(0, s)
278
279     def flush(self):
280         print(
281             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
282         )
283         self._cmd1, self._push, self._cmd2 = [], [], []
284
285     def debugprint(self, s, **kwargs):
286         if self._debug:
287             print(s, file=sys.stderr, **kwargs)
288
289
290 def do_setup(
291     extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
292 ):
293     extensions = extensions or []
294     temppath = temppath or pathlib.Path(
295         tempfile.mkstemp(prefix="muttmdwn-")[1]
296     )
297     cmds = MuttCommands(out_f, debug=debug_commands)
298
299     editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
300     if extensions:
301         editor = f'{editor} --extensions {",".join(extensions)}'
302
303     cmds.cmd('set my_editor="$editor"')
304     cmds.cmd('set my_edit_headers="$edit_headers"')
305     cmds.cmd(f'set editor="{editor}"')
306     cmds.cmd("unset edit_headers")
307     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
308     cmds.push("<first-entry><edit-file>")
309     cmds.flush()
310
311
312 def do_massage(
313     maildraft,
314     cmd_f,
315     *,
316     extensions=None,
317     converter=convert_markdown_to_html,
318     debug_commands=False,
319     debug_walk=False,
320 ):
321     # Here's the big picture: we're being invoked as the editor on the email
322     # draft, and whatever commands we write to the file given as cmdpath will
323     # be run by the second source command in the macro definition.
324
325     # Let's start by cleaning up what the setup did (see above), i.e. we
326     # restore the $editor and $edit_headers variables, and also unset the
327     # variable used to identify the command file we're currently writing
328     # to.
329     cmds = MuttCommands(cmd_f, debug=debug_commands)
330     cmds.cmd('set editor="$my_editor"')
331     cmds.cmd('set edit_headers="$my_edit_headers"')
332     cmds.cmd("unset my_editor")
333     cmds.cmd("unset my_edit_headers")
334
335     # let's flush those commands, as there'll be a lot of pushes from now
336     # on, which need to be run in reverse order
337     cmds.flush()
338
339     extensions = extensions.split(",") if extensions else []
340     tree = converter(maildraft, extensions=extensions)
341
342     mimetree = MIMETreeDFWalker(debug=debug_walk)
343
344     def visitor_fn(item, stack, *, debugprint=None):
345         """
346         Visitor function called for every node (part) of the MIME tree,
347         depth-first, and responsible for telling NeoMutt how to assemble
348         the tree.
349         """
350         if isinstance(item, Part):
351             # We've hit a leaf-node, i.e. an alternative or a related part
352             # with actual content.
353
354             # If the part is not an original part, i.e. doesn't already
355             # exist, we must first add it.
356             if not item.orig:
357                 cmds.push(f"<attach-file>{item.path}<enter>")
358                 cmds.push("<toggle-unlink><toggle-disposition>")
359                 if item.cid:
360                     cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
361
362             # If the item (including the original) comes with a
363             # description, then we might just as well update the NeoMutt
364             # tree now:
365             if item.desc:
366                 cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
367
368         elif isinstance(item, Multipart):
369             # This node has children, but we already visited them (see
370             # above), and so they have been tagged in NeoMutt's compose
371             # window. Now it's just a matter of telling NeoMutt to do the
372             # appropriate grouping:
373             if item.subtype == "alternative":
374                 cmds.push("<group-alternatives>")
375             elif item.subtype == "relative":
376                 cmds.push("<group-related>")
377             elif item.subtype == "multilingual":
378                 cmds.push("<group-multilingual>")
379
380             # Again, if there is a description, we might just as well:
381             if item.desc:
382                 cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
383
384         else:
385             # We should never get here
386             assert not "is valid part"
387
388         # Finally, if we're at non-root level, tag the new container,
389         # as it might itself be part of a container, to be processed
390         # one level up:
391         if stack:
392             cmds.push("<tag-entry>")
393
394     # -----------------
395     # End of visitor_fn
396
397     # Let's walk the tree and visit every node with our fancy visitor
398     # function
399     mimetree.walk(tree, visitor_fn=visitor_fn)
400
401     # Finally, cleanup. Since we're responsible for removing the temporary
402     # file, how's this for a little hack?
403     try:
404         filename = cmd_f.name
405     except AttributeError:
406         filename = "pytest_internal_file"
407     cmds.cmd(f"source 'rm -f {filename}|'")
408     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
409     cmds.flush()
410
411
412 # [ CLI ENTRY ] ###############################################################
413
414 if __name__ == "__main__":
415     args = parse_cli_args()
416
417     if args.mode == "setup":
418         if args.send_message:
419             raise NotImplementedError()
420
421         do_setup(args.extensions, debug_commands=args.debug_commands)
422
423     elif args.mode == "massage":
424         with open(args.cmdpath, "w") as cmd_f:
425             do_massage(
426                 args.MAILDRAFT,
427                 cmd_f,
428                 extensions=args.extensions,
429                 debug_commands=args.debug_commands,
430                 debug_walk=args.debug_walk,
431             )
432
433
434 # [ TESTS ] ###################################################################
435
436 try:
437     import pytest
438
439     class Tests:
440         @pytest.fixture
441         def const1(self):
442             return "CONSTANT STRING 1"
443
444         @pytest.fixture
445         def const2(self):
446             return "CONSTANT STRING 2"
447
448         # NOTE: tests using the capsys fixture must specify sys.stdout to the
449         # functions they call, else old stdout is used and not captured
450
451         def test_MuttCommands_cmd(self, const1, const2, capsys):
452             "Assert order of commands"
453             cmds = MuttCommands(out_f=sys.stdout)
454             cmds.cmd(const1)
455             cmds.cmd(const2)
456             cmds.flush()
457             captured = capsys.readouterr()
458             assert captured.out == "\n".join((const1, const2, ""))
459
460         def test_MuttCommands_push(self, const1, const2, capsys):
461             "Assert reverse order of pushes"
462             cmds = MuttCommands(out_f=sys.stdout)
463             cmds.push(const1)
464             cmds.push(const2)
465             cmds.flush()
466             captured = capsys.readouterr()
467             assert (
468                 captured.out
469                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
470             )
471
472         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
473             "Assert reverse order of pushes"
474             cmds = MuttCommands(out_f=sys.stdout)
475             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
476             for i in range(2):
477                 cmds.cmd(lines[4 * i + 0])
478                 cmds.cmd(lines[4 * i + 1])
479                 cmds.push(lines[4 * i + 2])
480                 cmds.push(lines[4 * i + 3])
481             cmds.flush()
482
483             captured = capsys.readouterr()
484             lines_out = captured.out.splitlines()
485             assert lines[0] in lines_out[0]
486             assert lines[1] in lines_out[1]
487             assert lines[7] in lines_out[2]
488             assert lines[6] in lines_out[3]
489             assert lines[3] in lines_out[4]
490             assert lines[2] in lines_out[5]
491             assert lines[4] in lines_out[6]
492             assert lines[5] in lines_out[7]
493
494         @pytest.fixture
495         def basic_mime_tree(self):
496             return Multipart(
497                 "relative",
498                 children=[
499                     Multipart(
500                         "alternative",
501                         children=[
502                             Part(
503                                 "text",
504                                 "plain",
505                                 "part.txt",
506                                 desc="Plain",
507                                 orig=True,
508                             ),
509                             Part("text", "html", "part.html", desc="HTML"),
510                         ],
511                         desc="Alternative",
512                     ),
513                     Part(
514                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
515                     ),
516                 ],
517                 desc="Related",
518             )
519
520         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
521             mimetree = MIMETreeDFWalker()
522
523             items = []
524
525             def visitor_fn(item, stack, debugprint):
526                 items.append((item, len(stack)))
527
528             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
529             assert len(items) == 5
530             assert items[0][0].subtype == "plain"
531             assert items[0][1] == 2
532             assert items[1][0].subtype == "html"
533             assert items[1][1] == 2
534             assert items[2][0].subtype == "alternative"
535             assert items[2][1] == 1
536             assert items[3][0].subtype == "png"
537             assert items[3][1] == 1
538             assert items[4][0].subtype == "relative"
539             assert items[4][1] == 0
540
541         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
542             mimetree = MIMETreeDFWalker()
543             items = []
544
545             def visitor_fn(item, stack, debugprint):
546                 items.append(item)
547
548             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
549             assert items[-1].subtype == "mixed"
550
551         def test_MIMETreeDFWalker_visitor_in_constructor(
552             self, basic_mime_tree
553         ):
554             items = []
555
556             def visitor_fn(item, stack, debugprint):
557                 items.append(item)
558
559             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
560             mimetree.walk(basic_mime_tree)
561             assert len(items) == 5
562
563         def test_do_setup_no_extensions(self, const1, capsys):
564             "Assert basics about the setup command output"
565             do_setup(temppath=const1, out_f=sys.stdout)
566             captout = capsys.readouterr()
567             lines = captout.out.splitlines()
568             assert lines[2].endswith(f'{const1}"')
569             assert lines[4].endswith(const1)
570             assert "first-entry" in lines[-1]
571             assert "edit-file" in lines[-1]
572
573         def test_do_setup_extensions(self, const1, const2, capsys):
574             "Assert that extensions are passed to editor"
575             do_setup(
576                 temppath=const1, extensions=[const2, const1], out_f=sys.stdout
577             )
578             captout = capsys.readouterr()
579             lines = captout.out.splitlines()
580             # assert comma-separated list of extensions passed
581             assert lines[2].endswith(f'{const2},{const1}"')
582             assert lines[4].endswith(const1)
583
584         def test_do_massage_basic(self, const1, capsys):
585             def converter(maildraft, extensions):
586                 return Part("text", "plain", "/dev/null", orig=True)
587
588             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
589             captured = capsys.readouterr()
590             assert (
591                 captured.out.strip()
592                 == """\
593             set editor="$my_editor"
594             set edit_headers="$my_edit_headers"
595             unset my_editor
596             unset my_edit_headers
597             source 'rm -f pytest_internal_file|'
598             unset my_mdwn_postprocess_cmd_file
599             """.replace(
600                     "            ", ""
601                 ).strip()
602             )
603
604         def test_do_massage_fulltree(self, const1, basic_mime_tree, capsys):
605             def converter(maildraft, extensions):
606                 return basic_mime_tree
607
608             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
609             captured = capsys.readouterr()
610             lines = captured.out.splitlines()[4:][::-1]
611             assert "Related" in lines.pop()
612             assert "group-related" in lines.pop()
613             assert "tag-entry" in lines.pop()
614             assert "Logo" in lines.pop()
615             assert "content-id" in lines.pop()
616             assert "toggle-unlink" in lines.pop()
617             assert "logo.png" in lines.pop()
618             assert "tag-entry" in lines.pop()
619             assert "Alternative" in lines.pop()
620             assert "group-alternatives" in lines.pop()
621             assert "tag-entry" in lines.pop()
622             assert "HTML" in lines.pop()
623             assert "toggle-unlink" in lines.pop()
624             assert "part.html" in lines.pop()
625             assert "tag-entry" in lines.pop()
626             assert "Plain" in lines.pop()
627
628 except ImportError:
629     pass