]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

fe851676dba1718c668eec3a9a6a4d74e26ee35e
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     macro compose B "\
10 #       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
11 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
12 #     " "Convert message into a modern MIME tree with inline images"
13 #
14 #     (Yes, we need to call source twice, as mutt only starts to process output
15 #     from a source command when the command exits, and since we need to react
16 #     to the output, we need to be invoked again, using a $my_ variable to pass
17 #     information)
18 #
19 # Requirements:
20 #   - python3
21 #   - python3-markdown
22 # Optional:
23 #   - pytest
24 #   - Pynliner
25 #   - Pygments, if installed, then syntax highlighting is enabled
26 #
27 # Latest version:
28 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
29 #
30 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
31 # Released under the GPL-2+ licence, just like Mutt itself.
32 #
33
34 import sys
35 import pathlib
36 import markdown
37 import tempfile
38 import argparse
39 from collections import namedtuple
40
41
42 def parse_cli_args(*args, **kwargs):
43     parser = argparse.ArgumentParser(
44         description=(
45             "NeoMutt helper to turn text/markdown email parts "
46             "into full-fledged MIME trees"
47         )
48     )
49     parser.epilog = (
50         "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
51         "Released under the MIT licence"
52     )
53
54     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
55     parser_setup = subp.add_parser("setup", help="Setup phase")
56     parser_massage = subp.add_parser("massage", help="Massaging phase")
57
58     parser_setup.add_argument(
59         "--debug-commands",
60         action="store_true",
61         help="Turn on debug logging of commands generated to stderr",
62     )
63
64     parser_setup.add_argument(
65         "--extension",
66         "-x",
67         metavar="EXTENSION",
68         dest="extensions",
69         nargs="?",
70         default=[],
71         action="append",
72         help="Markdown extension to add to the list of extensions use",
73     )
74
75     parser_setup.add_argument(
76         "--send-message",
77         action="store_true",
78         help="Generate command(s) to send the message after processing",
79     )
80
81     parser_massage.add_argument(
82         "--debug-commands",
83         action="store_true",
84         help="Turn on debug logging of commands generated to stderr",
85     )
86
87     parser_massage.add_argument(
88         "--debug-walk",
89         action="store_true",
90         help="Turn on debugging to stderr of the MIME tree walk",
91     )
92
93     parser_massage.add_argument(
94         "--extensions",
95         metavar="EXTENSIONS",
96         type=str,
97         default="",
98         help="Markdown extension to use (comma-separated list)",
99     )
100
101     parser_massage.add_argument(
102         "--write-commands-to",
103         metavar="PATH",
104         dest="cmdpath",
105         help="Temporary file path to write commands to",
106     )
107
108     parser_massage.add_argument(
109         "MAILDRAFT",
110         nargs="?",
111         help="If provided, the script is invoked as editor on the mail draft",
112     )
113
114     return parser.parse_args(*args, **kwargs)
115
116
117 # [ PARTS GENERATION ] ########################################################
118
119
120 class Part(
121     namedtuple(
122         "Part",
123         ["type", "subtype", "path", "desc", "cid", "orig"],
124         defaults=[None, None, False],
125     )
126 ):
127     def __str__(self):
128         ret = f"<{self.type}/{self.subtype}>"
129         if self.cid:
130             ret = f"{ret} cid:{self.cid}"
131         if self.orig:
132             ret = f"{ret} ORIGINAL"
133         return ret
134
135
136 class Multipart(
137     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
138 ):
139     def __str__(self):
140         return f"<multipart/{self.subtype}> children={len(self.children)}"
141
142
143
144 def convert_markdown_to_html(
145     origtext, draftpath, *, filewriter_fn=None, extensions=None
146 ):
147     mdwn = markdown.Markdown(extensions=extensions)
148
149     if not filewriter_fn:
150
151         def filewriter_fn(path, content, mode="w", **kwargs):
152             with open(path, mode, **kwargs) as out_f:
153                 out_f.write(content)
154
155     filewriter_fn(draftpath, origtext, encoding="utf-8")
156     textpart = Part(
157         "text", "plain", draftpath, "Plain-text version", orig=True
158     )
159
160     htmltext = mdwn.convert(origtext)
161
162     htmlpath = draftpath.with_suffix(".html")
163     filewriter_fn(
164         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
165     )
166     htmlpart = Part("text", "html", htmlpath, "HTML version")
167
168     logopart = Part(
169         "image",
170         "png",
171         "/usr/share/doc/neomutt/logo/neomutt-256.png",
172         "Logo",
173         "neomutt-256.png",
174     )
175
176     return Multipart(
177         "relative",
178         [
179             Multipart(
180                 "alternative",
181                 [textpart, htmlpart],
182                 "Group of alternative content",
183             ),
184             logopart,
185         ],
186         "Group of related content",
187     )
188
189
190 class MIMETreeDFWalker:
191     def __init__(self, *, visitor_fn=None, debug=False):
192         self._visitor_fn = visitor_fn
193         self._debug = debug
194
195     def walk(self, root, *, visitor_fn=None):
196         """
197         Recursive function to implement a depth-dirst walk of the MIME-tree
198         rooted at `root`.
199         """
200
201         if isinstance(root, list):
202             root = Multipart("mixed", children=root)
203
204         self._walk(
205             root,
206             stack=[],
207             visitor_fn=visitor_fn or self._visitor_fn,
208         )
209
210     def _walk(self, node, *, stack, visitor_fn):
211         # Let's start by enumerating the parts at the current level. At the
212         # root level, stack will be the empty list, and we expect a multipart/*
213         # container at this level. Later, e.g. within a mutlipart/alternative
214         # container, the subtree will just be the alternative parts, while the
215         # top of the stack will be the multipart/alternative container, which
216         # we will process after the following loop.
217
218         lead = f"{'| '*len(stack)}|-"
219         if isinstance(node, Multipart):
220             self.debugprint(
221                 f"{lead}{node} parents={[s.subtype for s in stack]}"
222             )
223
224             # Depth-first, so push the current container onto the stack,
225             # then descend …
226             stack.append(node)
227             self.debugprint("| " * (len(stack) + 1))
228             for child in node.children:
229                 self._walk(
230                     child,
231                     stack=stack,
232                     visitor_fn=visitor_fn,
233                 )
234             self.debugprint("| " * len(stack))
235             assert stack.pop() == node
236
237         else:
238             self.debugprint(f"{lead}{node}")
239
240         if visitor_fn:
241             visitor_fn(node, stack, debugprint=self.debugprint)
242
243     def debugprint(self, s, **kwargs):
244         if self._debug:
245             print(s, file=sys.stderr, **kwargs)
246
247
248 # [ RUN MODES ] ###############################################################
249
250
251 class MuttCommands:
252     """
253     Stupid class to interface writing out Mutt commands. This is quite a hack
254     to deal with the fact that Mutt runs "push" commands in reverse order, so
255     all of a sudden, things become very complicated when mixing with "real"
256     commands.
257
258     Hence we keep two sets of commands, and one set of pushes. Commands are
259     added to the first until a push is added, after which commands are added to
260     the second set of commands.
261
262     On flush(), the first set is printed, followed by the pushes in reverse,
263     and then the second set is printed. All 3 sets are then cleared.
264     """
265
266     def __init__(self, out_f=sys.stdout, *, debug=False):
267         self._cmd1, self._push, self._cmd2 = [], [], []
268         self._out_f = out_f
269         self._debug = debug
270
271     def cmd(self, s):
272         self.debugprint(s)
273         if self._push:
274             self._cmd2.append(s)
275         else:
276             self._cmd1.append(s)
277
278     def push(self, s):
279         s = s.replace('"', '"')
280         s = f'push "{s}"'
281         self.debugprint(s)
282         self._push.insert(0, s)
283
284     def flush(self):
285         print(
286             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
287         )
288         self._cmd1, self._push, self._cmd2 = [], [], []
289
290     def debugprint(self, s, **kwargs):
291         if self._debug:
292             print(s, file=sys.stderr, **kwargs)
293
294
295 def do_setup(
296     extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
297 ):
298     extensions = extensions or []
299     temppath = temppath or pathlib.Path(
300         tempfile.mkstemp(prefix="muttmdwn-")[1]
301     )
302     cmds = MuttCommands(out_f, debug=debug_commands)
303
304     editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
305     if extensions:
306         editor = f'{editor} --extensions {",".join(extensions)}'
307     if debug_commands:
308         editor = f'{editor} --debug-commands'
309
310     cmds.cmd('set my_editor="$editor"')
311     cmds.cmd('set my_edit_headers="$edit_headers"')
312     cmds.cmd(f'set editor="{editor}"')
313     cmds.cmd("unset edit_headers")
314     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
315     cmds.push("<first-entry><edit-file>")
316     cmds.flush()
317
318
319 def do_massage(
320     maildraft,
321     cmd_f,
322     *,
323     extensions=None,
324     converter=convert_markdown_to_html,
325     debug_commands=False,
326     debug_walk=False,
327 ):
328     # Here's the big picture: we're being invoked as the editor on the email
329     # draft, and whatever commands we write to the file given as cmdpath will
330     # be run by the second source command in the macro definition.
331
332     # Let's start by cleaning up what the setup did (see above), i.e. we
333     # restore the $editor and $edit_headers variables, and also unset the
334     # variable used to identify the command file we're currently writing
335     # to.
336     cmds = MuttCommands(cmd_f, debug=debug_commands)
337     cmds.cmd('set editor="$my_editor"')
338     cmds.cmd('set edit_headers="$my_edit_headers"')
339     cmds.cmd("unset my_editor")
340     cmds.cmd("unset my_edit_headers")
341
342     # let's flush those commands, as there'll be a lot of pushes from now
343     # on, which need to be run in reverse order
344     cmds.flush()
345
346     extensions = extensions.split(",") if extensions else []
347     with open(maildraft, "r") as draft_f:
348         tree = converter(
349             draft_f.read(), pathlib.Path(maildraft), extensions=extensions
350         )
351
352     mimetree = MIMETreeDFWalker(debug=debug_walk)
353
354     def visitor_fn(item, stack, *, debugprint=None):
355         """
356         Visitor function called for every node (part) of the MIME tree,
357         depth-first, and responsible for telling NeoMutt how to assemble
358         the tree.
359         """
360         if isinstance(item, Part):
361             # We've hit a leaf-node, i.e. an alternative or a related part
362             # with actual content.
363
364             # Let's add the part
365             if item.orig:
366                 # The original source already exists in the NeoMutt tree, but
367                 # the underlying file may have been modified, so we need to
368                 # update the encoding, but that's it:
369                 cmds.push("<update-encoding>")
370             else:
371                 # … whereas all other parts need to be added, and they're all
372                 # considered to be temporary and inline:
373                 cmds.push(f"<attach-file>{item.path}<enter>")
374                 cmds.push("<toggle-unlink><toggle-disposition>")
375
376             # If the item (including the original) comes with additional
377             # information, then we might just as well update the NeoMutt
378             # tree now:
379             if item.cid:
380                 cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
381
382         elif isinstance(item, Multipart):
383             # This node has children, but we already visited them (see
384             # above), and so they have been tagged in NeoMutt's compose
385             # window. Now it's just a matter of telling NeoMutt to do the
386             # appropriate grouping:
387             if item.subtype == "alternative":
388                 cmds.push("<group-alternatives>")
389             elif item.subtype == "relative":
390                 cmds.push("<group-related>")
391             elif item.subtype == "multilingual":
392                 cmds.push("<group-multilingual>")
393
394         else:
395             # We should never get here
396             assert not "is valid part"
397
398         # If the item has a description, we might just as well add it
399         if item.desc:
400             cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
401
402         # Finally, if we're at non-root level, tag the new container,
403         # as it might itself be part of a container, to be processed
404         # one level up:
405         if stack:
406             cmds.push("<tag-entry>")
407
408     # -----------------
409     # End of visitor_fn
410
411     # Let's walk the tree and visit every node with our fancy visitor
412     # function
413     mimetree.walk(tree, visitor_fn=visitor_fn)
414
415     # Finally, cleanup. Since we're responsible for removing the temporary
416     # file, how's this for a little hack?
417     try:
418         filename = cmd_f.name
419     except AttributeError:
420         filename = "pytest_internal_file"
421     cmds.cmd(f"source 'rm -f {filename}|'")
422     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
423     cmds.flush()
424
425
426 # [ CLI ENTRY ] ###############################################################
427
428 if __name__ == "__main__":
429     args = parse_cli_args()
430
431     if args.mode == "setup":
432         if args.send_message:
433             raise NotImplementedError()
434
435         do_setup(args.extensions, debug_commands=args.debug_commands)
436
437     elif args.mode == "massage":
438         with open(args.cmdpath, "w") as cmd_f:
439             do_massage(
440                 args.MAILDRAFT,
441                 cmd_f,
442                 extensions=args.extensions,
443                 debug_commands=args.debug_commands,
444                 debug_walk=args.debug_walk,
445             )
446
447
448 # [ TESTS ] ###################################################################
449
450 try:
451     import pytest
452
453     class Tests:
454         @pytest.fixture
455         def const1(self):
456             return "CONSTANT STRING 1"
457
458         @pytest.fixture
459         def const2(self):
460             return "CONSTANT STRING 2"
461
462         # NOTE: tests using the capsys fixture must specify sys.stdout to the
463         # functions they call, else old stdout is used and not captured
464
465         def test_MuttCommands_cmd(self, const1, const2, capsys):
466             "Assert order of commands"
467             cmds = MuttCommands(out_f=sys.stdout)
468             cmds.cmd(const1)
469             cmds.cmd(const2)
470             cmds.flush()
471             captured = capsys.readouterr()
472             assert captured.out == "\n".join((const1, const2, ""))
473
474         def test_MuttCommands_push(self, const1, const2, capsys):
475             "Assert reverse order of pushes"
476             cmds = MuttCommands(out_f=sys.stdout)
477             cmds.push(const1)
478             cmds.push(const2)
479             cmds.flush()
480             captured = capsys.readouterr()
481             assert (
482                 captured.out
483                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
484             )
485
486         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
487             "Assert reverse order of pushes"
488             cmds = MuttCommands(out_f=sys.stdout)
489             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
490             for i in range(2):
491                 cmds.cmd(lines[4 * i + 0])
492                 cmds.cmd(lines[4 * i + 1])
493                 cmds.push(lines[4 * i + 2])
494                 cmds.push(lines[4 * i + 3])
495             cmds.flush()
496
497             captured = capsys.readouterr()
498             lines_out = captured.out.splitlines()
499             assert lines[0] in lines_out[0]
500             assert lines[1] in lines_out[1]
501             assert lines[7] in lines_out[2]
502             assert lines[6] in lines_out[3]
503             assert lines[3] in lines_out[4]
504             assert lines[2] in lines_out[5]
505             assert lines[4] in lines_out[6]
506             assert lines[5] in lines_out[7]
507
508         @pytest.fixture
509         def basic_mime_tree(self):
510             return Multipart(
511                 "relative",
512                 children=[
513                     Multipart(
514                         "alternative",
515                         children=[
516                             Part(
517                                 "text",
518                                 "plain",
519                                 "part.txt",
520                                 desc="Plain",
521                                 orig=True,
522                             ),
523                             Part("text", "html", "part.html", desc="HTML"),
524                         ],
525                         desc="Alternative",
526                     ),
527                     Part(
528                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
529                     ),
530                 ],
531                 desc="Related",
532             )
533
534         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
535             mimetree = MIMETreeDFWalker()
536
537             items = []
538
539             def visitor_fn(item, stack, debugprint):
540                 items.append((item, len(stack)))
541
542             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
543             assert len(items) == 5
544             assert items[0][0].subtype == "plain"
545             assert items[0][1] == 2
546             assert items[1][0].subtype == "html"
547             assert items[1][1] == 2
548             assert items[2][0].subtype == "alternative"
549             assert items[2][1] == 1
550             assert items[3][0].subtype == "png"
551             assert items[3][1] == 1
552             assert items[4][0].subtype == "relative"
553             assert items[4][1] == 0
554
555         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
556             mimetree = MIMETreeDFWalker()
557             items = []
558
559             def visitor_fn(item, stack, debugprint):
560                 items.append(item)
561
562             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
563             assert items[-1].subtype == "mixed"
564
565         def test_MIMETreeDFWalker_visitor_in_constructor(
566             self, basic_mime_tree
567         ):
568             items = []
569
570             def visitor_fn(item, stack, debugprint):
571                 items.append(item)
572
573             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
574             mimetree.walk(basic_mime_tree)
575             assert len(items) == 5
576
577         def test_do_setup_no_extensions(self, const1, capsys):
578             "Assert basics about the setup command output"
579             do_setup(temppath=const1, out_f=sys.stdout)
580             captout = capsys.readouterr()
581             lines = captout.out.splitlines()
582             assert lines[2].endswith(f'{const1}"')
583             assert lines[4].endswith(const1)
584             assert "first-entry" in lines[-1]
585             assert "edit-file" in lines[-1]
586
587         def test_do_setup_extensions(self, const1, const2, capsys):
588             "Assert that extensions are passed to editor"
589             do_setup(
590                 temppath=const1, extensions=[const2, const1], out_f=sys.stdout
591             )
592             captout = capsys.readouterr()
593             lines = captout.out.splitlines()
594             # assert comma-separated list of extensions passed
595             assert lines[2].endswith(f'{const2},{const1}"')
596             assert lines[4].endswith(const1)
597
598         def test_do_massage_basic(self, const1, capsys):
599             def converter(maildraft, extensions):
600                 return Part("text", "plain", "/dev/null", orig=True)
601
602             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
603             captured = capsys.readouterr()
604             assert (
605                 captured.out.strip()
606                 == """\
607             set editor="$my_editor"
608             set edit_headers="$my_edit_headers"
609             unset my_editor
610             unset my_edit_headers
611             source 'rm -f pytest_internal_file|'
612             unset my_mdwn_postprocess_cmd_file
613             """.replace(
614                     "            ", ""
615                 ).strip()
616             )
617
618         def test_do_massage_fulltree(self, const1, basic_mime_tree, capsys):
619             def converter(maildraft, extensions):
620                 return basic_mime_tree
621
622             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
623             captured = capsys.readouterr()
624             lines = captured.out.splitlines()[4:][::-1]
625             assert "Related" in lines.pop()
626             assert "group-related" in lines.pop()
627             assert "tag-entry" in lines.pop()
628             assert "Logo" in lines.pop()
629             assert "content-id" in lines.pop()
630             assert "toggle-unlink" in lines.pop()
631             assert "logo.png" in lines.pop()
632             assert "tag-entry" in lines.pop()
633             assert "Alternative" in lines.pop()
634             assert "group-alternatives" in lines.pop()
635             assert "tag-entry" in lines.pop()
636             assert "HTML" in lines.pop()
637             assert "toggle-unlink" in lines.pop()
638             assert "part.html" in lines.pop()
639             assert "tag-entry" in lines.pop()
640             assert "Plain" in lines.pop()
641
642 except ImportError:
643     pass