.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     macro compose B "\
  10 #       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
  11 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  12 #     " "Convert message into a modern MIME tree with inline images"
  13 #
  14 #     (Yes, we need to call source twice, as mutt only starts to process output
  15 #     from a source command when the command exits, and since we need to react
  16 #     to the output, we need to be invoked again, using a $my_ variable to pass
  17 #     information)
  18 #
  19 # Requirements:
  20 #   - python3
  21 #   - python3-markdown
  22 # Optional:
  23 #   - pytest
  24 #   - Pynliner
  25 #   - Pygments, if installed, then syntax highlighting is enabled
  26 #
  27 # Latest version:
  28 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  29 #
  30 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  31 # Released under the GPL-2+ licence, just like Mutt itself.
  32 #
  33
  34 import sys
  35 import pathlib
  36 import markdown
  37 import tempfile
  38 import argparse
  39 from collections import namedtuple
  40
  41
  42 def parse_cli_args(*args, **kwargs):
  43     parser = argparse.ArgumentParser(
  44         description=(
  45             "NeoMutt helper to turn text/markdown email parts "
  46             "into full-fledged MIME trees"
  47         )
  48     )
  49     parser.epilog = (
  50         "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
  51         "Released under the MIT licence"
  52     )
  53
  54     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  55     parser_setup = subp.add_parser("setup", help="Setup phase")
  56     parser_massage = subp.add_parser("massage", help="Massaging phase")
  57
  58     parser_setup.add_argument(
  59         "--debug-commands",
  60         action="store_true",
  61         help="Turn on debug logging of commands generated to stderr",
  62     )
  63
  64     parser_setup.add_argument(
  65         "--extension",
  66         "-x",
  67         metavar="EXTENSION",
  68         dest="extensions",
  69         nargs="?",
  70         default=[],
  71         action="append",
  72         help="Markdown extension to add to the list of extensions use",
  73     )
  74
  75     parser_setup.add_argument(
  76         "--send-message",
  77         action="store_true",
  78         help="Generate command(s) to send the message after processing",
  79     )
  80
  81     parser_massage.add_argument(
  82         "--debug-commands",
  83         action="store_true",
  84         help="Turn on debug logging of commands generated to stderr",
  85     )
  86
  87     parser_massage.add_argument(
  88         "--debug-walk",
  89         action="store_true",
  90         help="Turn on debugging to stderr of the MIME tree walk",
  91     )
  92
  93     parser_massage.add_argument(
  94         "--extensions",
  95         metavar="EXTENSIONS",
  96         type=str,
  97         default="",
  98         help="Markdown extension to use (comma-separated list)",
  99     )
 100
 101     parser_massage.add_argument(
 102         "--write-commands-to",
 103         metavar="PATH",
 104         dest="cmdpath",
 105         help="Temporary file path to write commands to",
 106     )
 107
 108     parser_massage.add_argument(
 109         "MAILDRAFT",
 110         nargs="?",
 111         help="If provided, the script is invoked as editor on the mail draft",
 112     )
 113
 114     return parser.parse_args(*args, **kwargs)
 115
 116
 117 # [ PARTS GENERATION ] ########################################################
 118
 119
 120 class Part(
 121     namedtuple(
 122         "Part",
 123         ["type", "subtype", "path", "desc", "cid", "orig"],
 124         defaults=[None, None, False],
 125     )
 126 ):
 127     def __str__(self):
 128         ret = f"<{self.type}/{self.subtype}>"
 129         if self.cid:
 130             ret = f"{ret} cid:{self.cid}"
 131         if self.orig:
 132             ret = f"{ret} ORIGINAL"
 133         return ret
 134
 135
 136 class Multipart(
 137     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 138 ):
 139     def __str__(self):
 140         return f"<multipart/{self.subtype}> children={len(self.children)}"
 141
 142
 143
 144 def convert_markdown_to_html(
 145     origtext, draftpath, *, filewriter_fn=None, extensions=None
 146 ):
 147     mdwn = markdown.Markdown(extensions=extensions)
 148
 149     if not filewriter_fn:
 150
 151         def filewriter_fn(path, content, mode="w", **kwargs):
 152             with open(path, mode, **kwargs) as out_f:
 153                 out_f.write(content)
 154
 155     filewriter_fn(draftpath, origtext, encoding="utf-8")
 156     textpart = Part(
 157         "text", "plain", draftpath, "Plain-text version", orig=True
 158     )
 159
 160     htmltext = mdwn.convert(origtext)
 161
 162     htmlpath = draftpath.with_suffix(".html")
 163     filewriter_fn(
 164         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
 165     )
 166     htmlpart = Part("text", "html", htmlpath, "HTML version")
 167
 168     logopart = Part(
 169         "image",
 170         "png",
 171         "/usr/share/doc/neomutt/logo/neomutt-256.png",
 172         "Logo",
 173         "neomutt-256.png",
 174     )
 175
 176     return Multipart(
 177         "relative",
 178         [
 179             Multipart(
 180                 "alternative",
 181                 [textpart, htmlpart],
 182                 "Group of alternative content",
 183             ),
 184             logopart,
 185         ],
 186         "Group of related content",
 187     )
 188
 189
 190 class MIMETreeDFWalker:
 191     def __init__(self, *, visitor_fn=None, debug=False):
 192         self._visitor_fn = visitor_fn
 193         self._debug = debug
 194
 195     def walk(self, root, *, visitor_fn=None):
 196         """
 197         Recursive function to implement a depth-dirst walk of the MIME-tree
 198         rooted at `root`.
 199         """
 200
 201         if isinstance(root, list):
 202             root = Multipart("mixed", children=root)
 203
 204         self._walk(
 205             root,
 206             stack=[],
 207             visitor_fn=visitor_fn or self._visitor_fn,
 208         )
 209
 210     def _walk(self, node, *, stack, visitor_fn):
 211         # Let's start by enumerating the parts at the current level. At the
 212         # root level, stack will be the empty list, and we expect a multipart/*
 213         # container at this level. Later, e.g. within a mutlipart/alternative
 214         # container, the subtree will just be the alternative parts, while the
 215         # top of the stack will be the multipart/alternative container, which
 216         # we will process after the following loop.
 217
 218         lead = f"{'| '*len(stack)}|-"
 219         if isinstance(node, Multipart):
 220             self.debugprint(
 221                 f"{lead}{node} parents={[s.subtype for s in stack]}"
 222             )
 223
 224             # Depth-first, so push the current container onto the stack,
 225             # then descend …
 226             stack.append(node)
 227             self.debugprint("| " * (len(stack) + 1))
 228             for child in node.children:
 229                 self._walk(
 230                     child,
 231                     stack=stack,
 232                     visitor_fn=visitor_fn,
 233                 )
 234             self.debugprint("| " * len(stack))
 235             assert stack.pop() == node
 236
 237         else:
 238             self.debugprint(f"{lead}{node}")
 239
 240         if visitor_fn:
 241             visitor_fn(node, stack, debugprint=self.debugprint)
 242
 243     def debugprint(self, s, **kwargs):
 244         if self._debug:
 245             print(s, file=sys.stderr, **kwargs)
 246
 247
 248 # [ RUN MODES ] ###############################################################
 249
 250
 251 class MuttCommands:
 252     """
 253     Stupid class to interface writing out Mutt commands. This is quite a hack
 254     to deal with the fact that Mutt runs "push" commands in reverse order, so
 255     all of a sudden, things become very complicated when mixing with "real"
 256     commands.
 257
 258     Hence we keep two sets of commands, and one set of pushes. Commands are
 259     added to the first until a push is added, after which commands are added to
 260     the second set of commands.
 261
 262     On flush(), the first set is printed, followed by the pushes in reverse,
 263     and then the second set is printed. All 3 sets are then cleared.
 264     """
 265
 266     def __init__(self, out_f=sys.stdout, *, debug=False):
 267         self._cmd1, self._push, self._cmd2 = [], [], []
 268         self._out_f = out_f
 269         self._debug = debug
 270
 271     def cmd(self, s):
 272         self.debugprint(s)
 273         if self._push:
 274             self._cmd2.append(s)
 275         else:
 276             self._cmd1.append(s)
 277
 278     def push(self, s):
 279         s = s.replace('"', '"')
 280         s = f'push "{s}"'
 281         self.debugprint(s)
 282         self._push.insert(0, s)
 283
 284     def flush(self):
 285         print(
 286             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 287         )
 288         self._cmd1, self._push, self._cmd2 = [], [], []
 289
 290     def debugprint(self, s, **kwargs):
 291         if self._debug:
 292             print(s, file=sys.stderr, **kwargs)
 293
 294
 295 def do_setup(
 296     extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
 297 ):
 298     extensions = extensions or []
 299     temppath = temppath or pathlib.Path(
 300         tempfile.mkstemp(prefix="muttmdwn-")[1]
 301     )
 302     cmds = MuttCommands(out_f, debug=debug_commands)
 303
 304     editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
 305     if extensions:
 306         editor = f'{editor} --extensions {",".join(extensions)}'
 307     if debug_commands:
 308         editor = f'{editor} --debug-commands'
 309
 310     cmds.cmd('set my_editor="$editor"')
 311     cmds.cmd('set my_edit_headers="$edit_headers"')
 312     cmds.cmd(f'set editor="{editor}"')
 313     cmds.cmd("unset edit_headers")
 314     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 315     cmds.push("<first-entry><edit-file>")
 316     cmds.flush()
 317
 318
 319 def do_massage(
 320     draft_f,
 321     draftpath,
 322     cmd_f,
 323     *,
 324     extensions=None,
 325     converter=convert_markdown_to_html,
 326     debug_commands=False,
 327     debug_walk=False,
 328 ):
 329     # Here's the big picture: we're being invoked as the editor on the email
 330     # draft, and whatever commands we write to the file given as cmdpath will
 331     # be run by the second source command in the macro definition.
 332
 333     # Let's start by cleaning up what the setup did (see above), i.e. we
 334     # restore the $editor and $edit_headers variables, and also unset the
 335     # variable used to identify the command file we're currently writing
 336     # to.
 337     cmds = MuttCommands(cmd_f, debug=debug_commands)
 338     cmds.cmd('set editor="$my_editor"')
 339     cmds.cmd('set edit_headers="$my_edit_headers"')
 340     cmds.cmd("unset my_editor")
 341     cmds.cmd("unset my_edit_headers")
 342
 343     # let's flush those commands, as there'll be a lot of pushes from now
 344     # on, which need to be run in reverse order
 345     cmds.flush()
 346
 347     extensions = extensions.split(",") if extensions else []
 348     tree = converter(draft_f.read(), draftpath, extensions=extensions)
 349
 350     mimetree = MIMETreeDFWalker(debug=debug_walk)
 351
 352     def visitor_fn(item, stack, *, debugprint=None):
 353         """
 354         Visitor function called for every node (part) of the MIME tree,
 355         depth-first, and responsible for telling NeoMutt how to assemble
 356         the tree.
 357         """
 358         if isinstance(item, Part):
 359             # We've hit a leaf-node, i.e. an alternative or a related part
 360             # with actual content.
 361
 362             # Let's add the part
 363             if item.orig:
 364                 # The original source already exists in the NeoMutt tree, but
 365                 # the underlying file may have been modified, so we need to
 366                 # update the encoding, but that's it:
 367                 cmds.push("<update-encoding>")
 368             else:
 369                 # … whereas all other parts need to be added, and they're all
 370                 # considered to be temporary and inline:
 371                 cmds.push(f"<attach-file>{item.path}<enter>")
 372                 cmds.push("<toggle-unlink><toggle-disposition>")
 373
 374             # If the item (including the original) comes with additional
 375             # information, then we might just as well update the NeoMutt
 376             # tree now:
 377             if item.cid:
 378                 cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
 379
 380         elif isinstance(item, Multipart):
 381             # This node has children, but we already visited them (see
 382             # above), and so they have been tagged in NeoMutt's compose
 383             # window. Now it's just a matter of telling NeoMutt to do the
 384             # appropriate grouping:
 385             if item.subtype == "alternative":
 386                 cmds.push("<group-alternatives>")
 387             elif item.subtype == "relative":
 388                 cmds.push("<group-related>")
 389             elif item.subtype == "multilingual":
 390                 cmds.push("<group-multilingual>")
 391
 392         else:
 393             # We should never get here
 394             assert not "is valid part"
 395
 396         # If the item has a description, we might just as well add it
 397         if item.desc:
 398             cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
 399
 400         # Finally, if we're at non-root level, tag the new container,
 401         # as it might itself be part of a container, to be processed
 402         # one level up:
 403         if stack:
 404             cmds.push("<tag-entry>")
 405
 406     # -----------------
 407     # End of visitor_fn
 408
 409     # Let's walk the tree and visit every node with our fancy visitor
 410     # function
 411     mimetree.walk(tree, visitor_fn=visitor_fn)
 412
 413     # Finally, cleanup. Since we're responsible for removing the temporary
 414     # file, how's this for a little hack?
 415     try:
 416         filename = cmd_f.name
 417     except AttributeError:
 418         filename = "pytest_internal_file"
 419     cmds.cmd(f"source 'rm -f {filename}|'")
 420     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 421     cmds.flush()
 422
 423
 424 # [ CLI ENTRY ] ###############################################################
 425
 426 if __name__ == "__main__":
 427     args = parse_cli_args()
 428
 429     if args.mode == "setup":
 430         if args.send_message:
 431             raise NotImplementedError()
 432
 433         do_setup(args.extensions, debug_commands=args.debug_commands)
 434
 435     elif args.mode == "massage":
 436         with open(args.MAILDRAFT, "r") as draft_f, open(
 437             args.cmdpath, "w"
 438         ) as cmd_f:
 439             do_massage(
 440                 draft_f,
 441                 pathlib.Path(args.MAILDRAFT),
 442                 cmd_f,
 443                 extensions=args.extensions,
 444                 debug_commands=args.debug_commands,
 445                 debug_walk=args.debug_walk,
 446             )
 447
 448
 449 # [ TESTS ] ###################################################################
 450
 451 try:
 452     import pytest
 453     from io import StringIO
 454
 455     class Tests:
 456         @pytest.fixture
 457         def const1(self):
 458             return "CONSTANT STRING 1"
 459
 460         @pytest.fixture
 461         def const2(self):
 462             return "CONSTANT STRING 2"
 463
 464         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 465         # functions they call, else old stdout is used and not captured
 466
 467         def test_MuttCommands_cmd(self, const1, const2, capsys):
 468             "Assert order of commands"
 469             cmds = MuttCommands(out_f=sys.stdout)
 470             cmds.cmd(const1)
 471             cmds.cmd(const2)
 472             cmds.flush()
 473             captured = capsys.readouterr()
 474             assert captured.out == "\n".join((const1, const2, ""))
 475
 476         def test_MuttCommands_push(self, const1, const2, capsys):
 477             "Assert reverse order of pushes"
 478             cmds = MuttCommands(out_f=sys.stdout)
 479             cmds.push(const1)
 480             cmds.push(const2)
 481             cmds.flush()
 482             captured = capsys.readouterr()
 483             assert (
 484                 captured.out
 485                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 486             )
 487
 488         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 489             "Assert reverse order of pushes"
 490             cmds = MuttCommands(out_f=sys.stdout)
 491             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 492             for i in range(2):
 493                 cmds.cmd(lines[4 * i + 0])
 494                 cmds.cmd(lines[4 * i + 1])
 495                 cmds.push(lines[4 * i + 2])
 496                 cmds.push(lines[4 * i + 3])
 497             cmds.flush()
 498
 499             captured = capsys.readouterr()
 500             lines_out = captured.out.splitlines()
 501             assert lines[0] in lines_out[0]
 502             assert lines[1] in lines_out[1]
 503             assert lines[7] in lines_out[2]
 504             assert lines[6] in lines_out[3]
 505             assert lines[3] in lines_out[4]
 506             assert lines[2] in lines_out[5]
 507             assert lines[4] in lines_out[6]
 508             assert lines[5] in lines_out[7]
 509
 510         @pytest.fixture
 511         def basic_mime_tree(self):
 512             return Multipart(
 513                 "relative",
 514                 children=[
 515                     Multipart(
 516                         "alternative",
 517                         children=[
 518                             Part(
 519                                 "text",
 520                                 "plain",
 521                                 "part.txt",
 522                                 desc="Plain",
 523                                 orig=True,
 524                             ),
 525                             Part("text", "html", "part.html", desc="HTML"),
 526                         ],
 527                         desc="Alternative",
 528                     ),
 529                     Part(
 530                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 531                     ),
 532                 ],
 533                 desc="Related",
 534             )
 535
 536         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 537             mimetree = MIMETreeDFWalker()
 538
 539             items = []
 540
 541             def visitor_fn(item, stack, debugprint):
 542                 items.append((item, len(stack)))
 543
 544             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 545             assert len(items) == 5
 546             assert items[0][0].subtype == "plain"
 547             assert items[0][1] == 2
 548             assert items[1][0].subtype == "html"
 549             assert items[1][1] == 2
 550             assert items[2][0].subtype == "alternative"
 551             assert items[2][1] == 1
 552             assert items[3][0].subtype == "png"
 553             assert items[3][1] == 1
 554             assert items[4][0].subtype == "relative"
 555             assert items[4][1] == 0
 556
 557         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
 558             mimetree = MIMETreeDFWalker()
 559             items = []
 560
 561             def visitor_fn(item, stack, debugprint):
 562                 items.append(item)
 563
 564             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
 565             assert items[-1].subtype == "mixed"
 566
 567         def test_MIMETreeDFWalker_visitor_in_constructor(
 568             self, basic_mime_tree
 569         ):
 570             items = []
 571
 572             def visitor_fn(item, stack, debugprint):
 573                 items.append(item)
 574
 575             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 576             mimetree.walk(basic_mime_tree)
 577             assert len(items) == 5
 578
 579         def test_do_setup_no_extensions(self, const1, capsys):
 580             "Assert basics about the setup command output"
 581             do_setup(temppath=const1, out_f=sys.stdout)
 582             captout = capsys.readouterr()
 583             lines = captout.out.splitlines()
 584             assert lines[2].endswith(f'{const1}"')
 585             assert lines[4].endswith(const1)
 586             assert "first-entry" in lines[-1]
 587             assert "edit-file" in lines[-1]
 588
 589         def test_do_setup_extensions(self, const1, const2, capsys):
 590             "Assert that extensions are passed to editor"
 591             do_setup(
 592                 temppath=const1, extensions=[const2, const1], out_f=sys.stdout
 593             )
 594             captout = capsys.readouterr()
 595             lines = captout.out.splitlines()
 596             # assert comma-separated list of extensions passed
 597             assert lines[2].endswith(f'{const2},{const1}"')
 598             assert lines[4].endswith(const1)
 599
 600         @pytest.fixture
 601         def string_io(self, const1, text=None):
 602             return StringIO(text or const1)
 603
 604         def test_do_massage_basic(self, const1, string_io, capsys):
 605             def converter(drafttext, draftpath, extensions):
 606                 return Part("text", "plain", draftpath, orig=True)
 607
 608             do_massage(
 609                 draft_f=string_io,
 610                 draftpath=const1,
 611                 cmd_f=sys.stdout,
 612                 converter=converter,
 613             )
 614
 615             captured = capsys.readouterr()
 616             lines = captured.out.splitlines()
 617             assert '="$my_editor"' in lines.pop(0)
 618             assert '="$my_edit_headers"' in lines.pop(0)
 619             assert "unset my_editor" == lines.pop(0)
 620             assert "unset my_edit_headers" == lines.pop(0)
 621             assert "update-encoding" in lines.pop(0)
 622             assert "source 'rm -f " in lines.pop(0)
 623             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
 624
 625         def test_do_massage_fulltree(
 626             self, string_io, const1, basic_mime_tree, capsys
 627         ):
 628             def converter(drafttext, draftpath, extensions):
 629                 return basic_mime_tree
 630
 631             do_massage(
 632                 draft_f=string_io,
 633                 draftpath=const1,
 634                 cmd_f=sys.stdout,
 635                 converter=converter,
 636             )
 637
 638             captured = capsys.readouterr()
 639             lines = captured.out.splitlines()[4:][::-1]
 640             assert "Related" in lines.pop()
 641             assert "group-related" in lines.pop()
 642             assert "tag-entry" in lines.pop()
 643             assert "Logo" in lines.pop()
 644             assert "content-id" in lines.pop()
 645             assert "toggle-unlink" in lines.pop()
 646             assert "logo.png" in lines.pop()
 647             assert "tag-entry" in lines.pop()
 648             assert "Alternative" in lines.pop()
 649             assert "group-alternatives" in lines.pop()
 650             assert "tag-entry" in lines.pop()
 651             assert "HTML" in lines.pop()
 652             assert "toggle-unlink" in lines.pop()
 653             assert "part.html" in lines.pop()
 654             assert "tag-entry" in lines.pop()
 655             assert "Plain" in lines.pop()
 656
 657 except ImportError:
 658     pass