]> git.madduck.net Git - etc/neomutt.git/blob - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

WIP buildmimetree.py
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
1 #!/usr/bin/python3
2 #
3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
4 # alternative conversion, and handling of inline images, using NeoMutt's
5 # ability to manually craft MIME trees, but automating this process.
6 #
7 # Configuration:
8 #   neomuttrc (needs to be a single line):
9 #     macro compose B "\
10 #       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
11 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
12 #     " "Convert message into a modern MIME tree with inline images"
13 #
14 #     (Yes, we need to call source twice, as mutt only starts to process output
15 #     from a source command when the command exits, and since we need to react
16 #     to the output, we need to be invoked again, using a $my_ variable to pass
17 #     information)
18 #
19 # Requirements:
20 #   - python3
21 #   - python3-markdown
22 # Optional:
23 #   - pytest
24 #   - Pynliner
25 #   - Pygments, if installed, then syntax highlighting is enabled
26 #
27 # Latest version:
28 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
29 #
30 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
31 # Released under the GPL-2+ licence, just like Mutt itself.
32 #
33
34 import sys
35 import pathlib
36 import markdown
37 import tempfile
38 import argparse
39 from collections import namedtuple
40
41
42 def parse_cli_args(*args, **kwargs):
43     parser = argparse.ArgumentParser(
44         description=(
45             "NeoMutt helper to turn text/markdown email parts "
46             "into full-fledged MIME trees"
47         )
48     )
49     parser.epilog = (
50         "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
51         "Released under the MIT licence"
52     )
53
54     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
55     parser_setup = subp.add_parser("setup", help="Setup phase")
56     parser_massage = subp.add_parser("massage", help="Massaging phase")
57
58     parser_setup.add_argument(
59         "--debug-commands",
60         action="store_true",
61         help="Turn on debug logging of commands generated to stderr",
62     )
63
64     parser_setup.add_argument(
65         "--extension",
66         "-x",
67         metavar="EXTENSION",
68         dest="extensions",
69         nargs="?",
70         default=[],
71         action="append",
72         help="Markdown extension to add to the list of extensions use",
73     )
74
75     parser_massage.add_argument(
76         "--debug-commands",
77         action="store_true",
78         help="Turn on debug logging of commands generated to stderr",
79     )
80
81     parser_massage.add_argument(
82         "--debug-walk",
83         action="store_true",
84         help="Turn on debugging to stderr of the MIME tree walk",
85     )
86
87     parser_massage.add_argument(
88         "--extensions",
89         metavar="EXTENSIONS",
90         type=str,
91         default="",
92         help="Markdown extension to use (comma-separated list)",
93     )
94
95     parser_massage.add_argument(
96         "--write-commands-to",
97         metavar="PATH",
98         dest="cmdpath",
99         help="Temporary file path to write commands to",
100     )
101
102     parser_massage.add_argument(
103         "MAILDRAFT",
104         nargs="?",
105         help="If provided, the script is invoked as editor on the mail draft",
106     )
107
108     return parser.parse_args(*args, **kwargs)
109
110
111 # [ PARTS GENERATION ] ########################################################
112
113
114 class Part(
115     namedtuple(
116         "Part",
117         ["type", "subtype", "path", "desc", "cid", "orig"],
118         defaults=[None, None, False],
119     )
120 ):
121     def __str__(self):
122         ret = f"<{self.type}/{self.subtype}>"
123         if self.cid:
124             ret = f"{ret} cid:{self.cid}"
125         if self.orig:
126             ret = f"{ret} ORIGINAL"
127         return ret
128
129
130 class Multipart(
131     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
132 ):
133     def __str__(self):
134         return f"<multipart/{self.subtype}> children={len(self.children)}"
135
136
137 def convert_markdown_to_html(maildraft, *, extensions=None):
138     draftpath = pathlib.Path(maildraft)
139     textpart = Part(
140         "text", "plain", draftpath, "Plain-text version", orig=True
141     )
142
143     with open(draftpath, "r", encoding="utf-8") as textmarkdown:
144         text = textmarkdown.read()
145
146     mdwn = markdown.Markdown(extensions=extensions)
147     html = mdwn.convert(text)
148
149     htmlpath = draftpath.with_suffix(".html")
150     htmlpart = Part("text", "html", htmlpath, "HTML version")
151
152     with open(
153         htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
154     ) as texthtml:
155         texthtml.write(html)
156
157     logopart = Part(
158         "image",
159         "png",
160         "/usr/share/doc/neomutt/logo/neomutt-256.png",
161         "Logo",
162         "neomutt-256.png",
163     )
164
165     return Multipart(
166         "relative",
167         [
168             Multipart(
169                 "alternative",
170                 [textpart, htmlpart],
171                 "Group of alternative content",
172             ),
173             logopart,
174         ],
175         "Group of related content",
176     )
177
178
179 class MIMETreeDFWalker:
180     def __init__(self, *, visitor_fn=None, debug=False):
181         self._visitor_fn = visitor_fn
182         self._debug = debug
183
184     def walk(self, root, *, visitor_fn=None):
185         """
186         Recursive function to implement a depth-dirst walk of the MIME-tree
187         rooted at `root`.
188         """
189
190         if isinstance(root, list):
191             root = Multipart("mixed", children=root)
192
193         self._walk(
194             root,
195             stack=[],
196             visitor_fn=visitor_fn or self._visitor_fn,
197         )
198
199     def _walk(self, node, *, stack, visitor_fn):
200         # Let's start by enumerating the parts at the current level. At the
201         # root level, stack will be the empty list, and we expect a multipart/*
202         # container at this level. Later, e.g. within a mutlipart/alternative
203         # container, the subtree will just be the alternative parts, while the
204         # top of the stack will be the multipart/alternative container, which
205         # we will process after the following loop.
206
207         lead = f"{'| '*len(stack)}|-"
208         if isinstance(node, Multipart):
209             self.debugprint(
210                 f"{lead}{node} parents={[s.subtype for s in stack]}"
211             )
212
213             # Depth-first, so push the current container onto the stack,
214             # then descend …
215             stack.append(node)
216             self.debugprint("| " * (len(stack) + 1))
217             for child in node.children:
218                 self._walk(
219                     child,
220                     stack=stack,
221                     visitor_fn=visitor_fn,
222                 )
223             self.debugprint("| " * len(stack))
224             assert stack.pop() == node
225
226         else:
227             self.debugprint(f"{lead}{node}")
228
229         if visitor_fn:
230             visitor_fn(node, stack, debugprint=self.debugprint)
231
232     def debugprint(self, s, **kwargs):
233         if self._debug:
234             print(s, file=sys.stderr, **kwargs)
235
236
237 # [ RUN MODES ] ###############################################################
238
239
240 class MuttCommands:
241     """
242     Stupid class to interface writing out Mutt commands. This is quite a hack
243     to deal with the fact that Mutt runs "push" commands in reverse order, so
244     all of a sudden, things become very complicated when mixing with "real"
245     commands.
246
247     Hence we keep two sets of commands, and one set of pushes. Commands are
248     added to the first until a push is added, after which commands are added to
249     the second set of commands.
250
251     On flush(), the first set is printed, followed by the pushes in reverse,
252     and then the second set is printed. All 3 sets are then cleared.
253     """
254
255     def __init__(self, out_f=sys.stdout, *, debug=False):
256         self._cmd1, self._push, self._cmd2 = [], [], []
257         self._out_f = out_f
258         self._debug = debug
259
260     def cmd(self, s):
261         self.debugprint(s)
262         if self._push:
263             self._cmd2.append(s)
264         else:
265             self._cmd1.append(s)
266
267     def push(self, s):
268         s = s.replace('"', '"')
269         s = f'push "{s}"'
270         self.debugprint(s)
271         self._push.insert(0, s)
272
273     def flush(self):
274         print(
275             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
276         )
277         self._cmd1, self._push, self._cmd2 = [], [], []
278
279     def debugprint(self, s, **kwargs):
280         if self._debug:
281             print(s, file=sys.stderr, **kwargs)
282
283
284 def do_setup(
285     extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
286 ):
287     extensions = extensions or []
288     temppath = temppath or pathlib.Path(
289         tempfile.mkstemp(prefix="muttmdwn-")[1]
290     )
291     cmds = MuttCommands(out_f, debug=debug_commands)
292
293     editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
294     if extensions:
295         editor = f'{editor} --extensions {",".join(extensions)}'
296
297     cmds.cmd('set my_editor="$editor"')
298     cmds.cmd('set my_edit_headers="$edit_headers"')
299     cmds.cmd(f'set editor="{editor}"')
300     cmds.cmd("unset edit_headers")
301     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
302     cmds.push("<first-entry><edit-file>")
303     cmds.flush()
304
305
306 def do_massage(
307     maildraft,
308     cmdpath,
309     *,
310     extensions=None,
311     converter=convert_markdown_to_html,
312     debug_commands=False,
313     debug_walk=False,
314 ):
315     # Here's the big picture: we're being invoked as the editor on the email
316     # draft, and whatever commands we write to the file given as cmdpath will
317     # be run by the second source command in the macro definition.
318
319     with open(cmdpath, "w") as cmd_f:
320         # Let's start by cleaning up what the setup did (see above), i.e. we
321         # restore the $editor and $edit_headers variables, and also unset the
322         # variable used to identify the command file we're currently writing
323         # to.
324         cmds = MuttCommands(cmd_f, debug=debug_commands)
325         cmds.cmd('set editor="$my_editor"')
326         cmds.cmd('set edit_headers="$my_edit_headers"')
327         cmds.cmd("unset my_editor")
328         cmds.cmd("unset my_edit_headers")
329
330         # let's flush those commands, as there'll be a lot of pushes from now
331         # on, which need to be run in reverse order
332         cmds.flush()
333
334         extensions = extensions.split(",") if extensions else []
335         tree = converter(maildraft, extensions=extensions)
336
337         mimetree = MIMETreeDFWalker(debug=args.debug_walk)
338
339         def visitor_fn(item, stack, *, debugprint=None):
340             """
341             Visitor function called for every node (part) of the MIME tree,
342             depth-first, and responsible for telling NeoMutt how to assemble
343             the tree.
344             """
345             if isinstance(item, Part):
346                 # We've hit a leaf-node, i.e. an alternative or a related part
347                 # with actual content.
348
349                 # If the part is not an original part, i.e. doesn't already
350                 # exist, we must first add it.
351                 if not item.orig:
352                     cmds.push(f"<attach-file>{item.path}<enter>")
353                     cmds.push("<toggle-unlink><toggle-disposition>")
354                     if item.cid:
355                         cmds.push(
356                             f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>"
357                         )
358
359                 # If the item (including the original) comes with a
360                 # description, then we might just as well update the NeoMutt
361                 # tree now:
362                 if item.desc:
363                     cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
364
365                 # Finally, tag the entry that we just processed, so that when
366                 # we're done at this level, as we walk up the stack, the items
367                 # to be grouped will already be tagged and ready.
368                 cmds.push("<tag-entry>")
369
370             elif isinstance(item, Multipart):
371                 # This node has children, but we already visited them (see
372                 # above), and so they have been tagged in NeoMutt's compose
373                 # window. Now it's just a matter of telling NeoMutt to do the
374                 # appropriate grouping:
375                 if item.subtype == "alternative":
376                     cmds.push("<group-alternatives>")
377                 elif item.subtype == "relative":
378                     cmds.push("<group-related>")
379                 elif item.subtype == "multilingual":
380                     cmds.push("<group-multilingual>")
381
382                 # Again, if there is a description, we might just as well:
383                 if item.desc:
384                     cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
385
386                 # Finally, if we're at non-root level, tag the new container,
387                 # as it might itself be part of a container, to be processed
388                 # one level up:
389                 if stack:
390                     cmds.push("<tag-entry>")
391
392             else:
393                 # We should never get here
394                 assert not "is valid part"
395
396         # -----------------
397         # End of visitor_fn
398
399         # Let's walk the tree and visit every node with our fancy visitor
400         # function
401         mimetree.walk(tree, visitor_fn=visitor_fn)
402
403         # Finally, cleanup. Since we're responsible for removing the temporary
404         # file, how's this for a little hack?
405         cmds.cmd(f"source 'rm -f {args.cmdpath}|'")
406         cmds.cmd("unset my_mdwn_postprocess_cmd_file")
407         cmds.flush()
408
409
410 # [ CLI ENTRY ] ###############################################################
411
412 if __name__ == "__main__":
413     args = parse_cli_args()
414
415     if args.mode == "setup":
416         do_setup(args.extensions, debug_commands=args.debug_commands)
417
418     elif args.mode == "massage":
419         do_massage(
420             args.MAILDRAFT,
421             args.cmdpath,
422             extensions=args.extensions,
423             debug_commands=args.debug_commands,
424             debug_walk=args.debug_walk,
425         )
426
427
428 # [ TESTS ] ###################################################################
429
430 try:
431     import pytest
432
433     class Tests:
434         @pytest.fixture
435         def const1(self):
436             return "CONSTANT STRING 1"
437
438         @pytest.fixture
439         def const2(self):
440             return "CONSTANT STRING 2"
441
442         # NOTE: tests using the capsys fixture must specify sys.stdout to the
443         # functions they call, else old stdout is used and not captured
444
445         def test_MuttCommands_cmd(self, const1, const2, capsys):
446             "Assert order of commands"
447             cmds = MuttCommands(out_f=sys.stdout)
448             cmds.cmd(const1)
449             cmds.cmd(const2)
450             cmds.flush()
451             captured = capsys.readouterr()
452             assert captured.out == "\n".join((const1, const2, ""))
453
454         def test_MuttCommands_push(self, const1, const2, capsys):
455             "Assert reverse order of pushes"
456             cmds = MuttCommands(out_f=sys.stdout)
457             cmds.push(const1)
458             cmds.push(const2)
459             cmds.flush()
460             captured = capsys.readouterr()
461             assert (
462                 captured.out
463                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
464             )
465
466         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
467             "Assert reverse order of pushes"
468             cmds = MuttCommands(out_f=sys.stdout)
469             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
470             for i in range(2):
471                 cmds.cmd(lines[4 * i + 0])
472                 cmds.cmd(lines[4 * i + 1])
473                 cmds.push(lines[4 * i + 2])
474                 cmds.push(lines[4 * i + 3])
475             cmds.flush()
476
477             captured = capsys.readouterr()
478             lines_out = captured.out.splitlines()
479             assert lines[0] in lines_out[0]
480             assert lines[1] in lines_out[1]
481             assert lines[7] in lines_out[2]
482             assert lines[6] in lines_out[3]
483             assert lines[3] in lines_out[4]
484             assert lines[2] in lines_out[5]
485             assert lines[4] in lines_out[6]
486             assert lines[5] in lines_out[7]
487
488         @pytest.fixture
489         def basic_mime_tree(self):
490             return Multipart(
491                 "related",
492                 children=[
493                     Multipart(
494                         "alternative",
495                         children=[
496                             Part("text", "plain", "part.txt", desc="Plain"),
497                             Part("text", "html", "part.html", desc="HTML"),
498                         ],
499                         desc="Alternative",
500                     ),
501                     Part(
502                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
503                     ),
504                 ],
505                 desc="Related",
506             )
507
508         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
509             mimetree = MIMETreeDFWalker()
510
511             items = []
512
513             def visitor_fn(item, stack, debugprint):
514                 items.append((item, len(stack)))
515
516             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
517             assert len(items) == 5
518             assert items[0][0].subtype == "plain"
519             assert items[0][1] == 2
520             assert items[1][0].subtype == "html"
521             assert items[1][1] == 2
522             assert items[2][0].subtype == "alternative"
523             assert items[2][1] == 1
524             assert items[3][0].subtype == "png"
525             assert items[3][1] == 1
526             assert items[4][0].subtype == "related"
527             assert items[4][1] == 0
528
529         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
530             mimetree = MIMETreeDFWalker()
531
532             items = []
533
534             def visitor_fn(item, stack, debugprint):
535                 items.append(item)
536
537             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
538             assert items[-1].subtype == "mixed"
539
540         def test_MIMETreeDFWalker_visitor_in_constructor(
541             self, basic_mime_tree
542         ):
543             items = []
544
545             def visitor_fn(item, stack, debugprint):
546                 items.append(item)
547
548             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
549             mimetree.walk(basic_mime_tree)
550             assert len(items) == 5
551
552         def test_do_setup_no_extensions(self, const1, capsys):
553             "Assert basics about the setup command output"
554             do_setup(temppath=const1, out_f=sys.stdout)
555             captout = capsys.readouterr()
556             lines = captout.out.splitlines()
557             assert lines[2].endswith(f'{const1}"')
558             assert lines[4].endswith(const1)
559             assert "first-entry" in lines[-1]
560             assert "edit-file" in lines[-1]
561
562         def test_do_setup_extensions(self, const1, const2, capsys):
563             "Assert that extensions are passed to editor"
564             do_setup(
565                 temppath=const1, extensions=[const2, const1], out_f=sys.stdout
566             )
567             captout = capsys.readouterr()
568             lines = captout.out.splitlines()
569             # assert comma-separated list of extensions passed
570             assert lines[2].endswith(f'{const2},{const1}"')
571             assert lines[4].endswith(const1)
572
573 except ImportError:
574     pass