.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     macro compose B "\
  10 #       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
  11 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
  12 #     " "Convert message into a modern MIME tree with inline images"
  13 #
  14 #     (Yes, we need to call source twice, as mutt only starts to process output
  15 #     from a source command when the command exits, and since we need to react
  16 #     to the output, we need to be invoked again, using a $my_ variable to pass
  17 #     information)
  18 #
  19 # Requirements:
  20 #   - python3
  21 #   - python3-markdown
  22 # Optional:
  23 #   - pytest
  24 #   - Pynliner
  25 #   - Pygments, if installed, then syntax highlighting is enabled
  26 #
  27 # Latest version:
  28 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  29 #
  30 # Copyright © 2023 martin f. krafft <madduck@madduck.net>
  31 # Released under the GPL-2+ licence, just like Mutt itself.
  32 #
  33
  34 import sys
  35 import pathlib
  36 import markdown
  37 import tempfile
  38 import argparse
  39 from collections import namedtuple
  40
  41
  42 def parse_cli_args(*args, **kwargs):
  43     parser = argparse.ArgumentParser(
  44         description=(
  45             "NeoMutt helper to turn text/markdown email parts "
  46             "into full-fledged MIME trees"
  47         )
  48     )
  49     parser.epilog = (
  50         "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
  51         "Released under the MIT licence"
  52     )
  53
  54     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
  55     parser_setup = subp.add_parser("setup", help="Setup phase")
  56     parser_massage = subp.add_parser("massage", help="Massaging phase")
  57
  58     parser_setup.add_argument(
  59         "--debug-commands",
  60         action="store_true",
  61         help="Turn on debug logging of commands generated to stderr",
  62     )
  63
  64     parser_setup.add_argument(
  65         "--extension",
  66         "-x",
  67         metavar="EXTENSION",
  68         dest="extensions",
  69         nargs="?",
  70         default=[],
  71         action="append",
  72         help="Markdown extension to add to the list of extensions use",
  73     )
  74
  75     parser_setup.add_argument(
  76         "--send-message",
  77         action="store_true",
  78         help="Generate command(s) to send the message after processing",
  79     )
  80
  81     parser_massage.add_argument(
  82         "--debug-commands",
  83         action="store_true",
  84         help="Turn on debug logging of commands generated to stderr",
  85     )
  86
  87     parser_massage.add_argument(
  88         "--debug-walk",
  89         action="store_true",
  90         help="Turn on debugging to stderr of the MIME tree walk",
  91     )
  92
  93     parser_massage.add_argument(
  94         "--extensions",
  95         metavar="EXTENSIONS",
  96         type=str,
  97         default="",
  98         help="Markdown extension to use (comma-separated list)",
  99     )
 100
 101     parser_massage.add_argument(
 102         "--write-commands-to",
 103         metavar="PATH",
 104         dest="cmdpath",
 105         help="Temporary file path to write commands to",
 106     )
 107
 108     parser_massage.add_argument(
 109         "MAILDRAFT",
 110         nargs="?",
 111         help="If provided, the script is invoked as editor on the mail draft",
 112     )
 113
 114     return parser.parse_args(*args, **kwargs)
 115
 116
 117 # [ PARTS GENERATION ] ########################################################
 118
 119
 120 class Part(
 121     namedtuple(
 122         "Part",
 123         ["type", "subtype", "path", "desc", "cid", "orig"],
 124         defaults=[None, None, False],
 125     )
 126 ):
 127     def __str__(self):
 128         ret = f"<{self.type}/{self.subtype}>"
 129         if self.cid:
 130             ret = f"{ret} cid:{self.cid}"
 131         if self.orig:
 132             ret = f"{ret} ORIGINAL"
 133         return ret
 134
 135
 136 class Multipart(
 137     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 138 ):
 139     def __str__(self):
 140         return f"<multipart/{self.subtype}> children={len(self.children)}"
 141
 142
 143 def convert_markdown_to_html(maildraft, *, extensions=None):
 144     draftpath = pathlib.Path(maildraft)
 145     with open(draftpath, "r", encoding="utf-8") as textmarkdown:
 146         text = textmarkdown.read()
 147
 148     mdwn = markdown.Markdown(extensions=extensions)
 149
 150
 151     with open(draftpath, "w", encoding="utf-8") as textplain:
 152         textplain.write(text)
 153     textpart = Part(
 154         "text", "plain", draftpath, "Plain-text version", orig=True
 155     )
 156
 157     html = mdwn.convert(text)
 158     htmlpath = draftpath.with_suffix(".html")
 159     with open(
 160         htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
 161     ) as texthtml:
 162         texthtml.write(html)
 163     htmlpart = Part("text", "html", htmlpath, "HTML version")
 164
 165     logopart = Part(
 166         "image",
 167         "png",
 168         "/usr/share/doc/neomutt/logo/neomutt-256.png",
 169         "Logo",
 170         "neomutt-256.png",
 171     )
 172
 173     return Multipart(
 174         "relative",
 175         [
 176             Multipart(
 177                 "alternative",
 178                 [textpart, htmlpart],
 179                 "Group of alternative content",
 180             ),
 181             logopart,
 182         ],
 183         "Group of related content",
 184     )
 185
 186
 187 class MIMETreeDFWalker:
 188     def __init__(self, *, visitor_fn=None, debug=False):
 189         self._visitor_fn = visitor_fn
 190         self._debug = debug
 191
 192     def walk(self, root, *, visitor_fn=None):
 193         """
 194         Recursive function to implement a depth-dirst walk of the MIME-tree
 195         rooted at `root`.
 196         """
 197
 198         if isinstance(root, list):
 199             root = Multipart("mixed", children=root)
 200
 201         self._walk(
 202             root,
 203             stack=[],
 204             visitor_fn=visitor_fn or self._visitor_fn,
 205         )
 206
 207     def _walk(self, node, *, stack, visitor_fn):
 208         # Let's start by enumerating the parts at the current level. At the
 209         # root level, stack will be the empty list, and we expect a multipart/*
 210         # container at this level. Later, e.g. within a mutlipart/alternative
 211         # container, the subtree will just be the alternative parts, while the
 212         # top of the stack will be the multipart/alternative container, which
 213         # we will process after the following loop.
 214
 215         lead = f"{'| '*len(stack)}|-"
 216         if isinstance(node, Multipart):
 217             self.debugprint(
 218                 f"{lead}{node} parents={[s.subtype for s in stack]}"
 219             )
 220
 221             # Depth-first, so push the current container onto the stack,
 222             # then descend …
 223             stack.append(node)
 224             self.debugprint("| " * (len(stack) + 1))
 225             for child in node.children:
 226                 self._walk(
 227                     child,
 228                     stack=stack,
 229                     visitor_fn=visitor_fn,
 230                 )
 231             self.debugprint("| " * len(stack))
 232             assert stack.pop() == node
 233
 234         else:
 235             self.debugprint(f"{lead}{node}")
 236
 237         if visitor_fn:
 238             visitor_fn(node, stack, debugprint=self.debugprint)
 239
 240     def debugprint(self, s, **kwargs):
 241         if self._debug:
 242             print(s, file=sys.stderr, **kwargs)
 243
 244
 245 # [ RUN MODES ] ###############################################################
 246
 247
 248 class MuttCommands:
 249     """
 250     Stupid class to interface writing out Mutt commands. This is quite a hack
 251     to deal with the fact that Mutt runs "push" commands in reverse order, so
 252     all of a sudden, things become very complicated when mixing with "real"
 253     commands.
 254
 255     Hence we keep two sets of commands, and one set of pushes. Commands are
 256     added to the first until a push is added, after which commands are added to
 257     the second set of commands.
 258
 259     On flush(), the first set is printed, followed by the pushes in reverse,
 260     and then the second set is printed. All 3 sets are then cleared.
 261     """
 262
 263     def __init__(self, out_f=sys.stdout, *, debug=False):
 264         self._cmd1, self._push, self._cmd2 = [], [], []
 265         self._out_f = out_f
 266         self._debug = debug
 267
 268     def cmd(self, s):
 269         self.debugprint(s)
 270         if self._push:
 271             self._cmd2.append(s)
 272         else:
 273             self._cmd1.append(s)
 274
 275     def push(self, s):
 276         s = s.replace('"', '"')
 277         s = f'push "{s}"'
 278         self.debugprint(s)
 279         self._push.insert(0, s)
 280
 281     def flush(self):
 282         print(
 283             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 284         )
 285         self._cmd1, self._push, self._cmd2 = [], [], []
 286
 287     def debugprint(self, s, **kwargs):
 288         if self._debug:
 289             print(s, file=sys.stderr, **kwargs)
 290
 291
 292 def do_setup(
 293     extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
 294 ):
 295     extensions = extensions or []
 296     temppath = temppath or pathlib.Path(
 297         tempfile.mkstemp(prefix="muttmdwn-")[1]
 298     )
 299     cmds = MuttCommands(out_f, debug=debug_commands)
 300
 301     editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
 302     if extensions:
 303         editor = f'{editor} --extensions {",".join(extensions)}'
 304
 305     cmds.cmd('set my_editor="$editor"')
 306     cmds.cmd('set my_edit_headers="$edit_headers"')
 307     cmds.cmd(f'set editor="{editor}"')
 308     cmds.cmd("unset edit_headers")
 309     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 310     cmds.push("<first-entry><edit-file>")
 311     cmds.flush()
 312
 313
 314 def do_massage(
 315     maildraft,
 316     cmd_f,
 317     *,
 318     extensions=None,
 319     converter=convert_markdown_to_html,
 320     debug_commands=False,
 321     debug_walk=False,
 322 ):
 323     # Here's the big picture: we're being invoked as the editor on the email
 324     # draft, and whatever commands we write to the file given as cmdpath will
 325     # be run by the second source command in the macro definition.
 326
 327     # Let's start by cleaning up what the setup did (see above), i.e. we
 328     # restore the $editor and $edit_headers variables, and also unset the
 329     # variable used to identify the command file we're currently writing
 330     # to.
 331     cmds = MuttCommands(cmd_f, debug=debug_commands)
 332     cmds.cmd('set editor="$my_editor"')
 333     cmds.cmd('set edit_headers="$my_edit_headers"')
 334     cmds.cmd("unset my_editor")
 335     cmds.cmd("unset my_edit_headers")
 336
 337     # let's flush those commands, as there'll be a lot of pushes from now
 338     # on, which need to be run in reverse order
 339     cmds.flush()
 340
 341     extensions = extensions.split(",") if extensions else []
 342     tree = converter(maildraft, extensions=extensions)
 343
 344     mimetree = MIMETreeDFWalker(debug=debug_walk)
 345
 346     def visitor_fn(item, stack, *, debugprint=None):
 347         """
 348         Visitor function called for every node (part) of the MIME tree,
 349         depth-first, and responsible for telling NeoMutt how to assemble
 350         the tree.
 351         """
 352         if isinstance(item, Part):
 353             # We've hit a leaf-node, i.e. an alternative or a related part
 354             # with actual content.
 355
 356             # Let's add the part
 357             if item.orig:
 358                 # The original source already exists in the NeoMutt tree, but
 359                 # the underlying file may have been modified, so we need to
 360                 # update the encoding, but that's it:
 361                 cmds.push("<update-encoding>")
 362             else:
 363                 # … whereas all other parts need to be added, and they're all
 364                 # considered to be temporary and inline:
 365                 cmds.push(f"<attach-file>{item.path}<enter>")
 366                 cmds.push("<toggle-unlink><toggle-disposition>")
 367
 368             # If the item (including the original) comes with additional
 369             # information, then we might just as well update the NeoMutt
 370             # tree now:
 371             if item.cid:
 372                 cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
 373             if item.desc:
 374                 cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
 375
 376         elif isinstance(item, Multipart):
 377             # This node has children, but we already visited them (see
 378             # above), and so they have been tagged in NeoMutt's compose
 379             # window. Now it's just a matter of telling NeoMutt to do the
 380             # appropriate grouping:
 381             if item.subtype == "alternative":
 382                 cmds.push("<group-alternatives>")
 383             elif item.subtype == "relative":
 384                 cmds.push("<group-related>")
 385             elif item.subtype == "multilingual":
 386                 cmds.push("<group-multilingual>")
 387
 388             # Again, if there is a description, we might just as well:
 389             if item.desc:
 390                 cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
 391
 392         else:
 393             # We should never get here
 394             assert not "is valid part"
 395
 396         # Finally, if we're at non-root level, tag the new container,
 397         # as it might itself be part of a container, to be processed
 398         # one level up:
 399         if stack:
 400             cmds.push("<tag-entry>")
 401
 402     # -----------------
 403     # End of visitor_fn
 404
 405     # Let's walk the tree and visit every node with our fancy visitor
 406     # function
 407     mimetree.walk(tree, visitor_fn=visitor_fn)
 408
 409     # Finally, cleanup. Since we're responsible for removing the temporary
 410     # file, how's this for a little hack?
 411     try:
 412         filename = cmd_f.name
 413     except AttributeError:
 414         filename = "pytest_internal_file"
 415     cmds.cmd(f"source 'rm -f {filename}|'")
 416     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
 417     cmds.flush()
 418
 419
 420 # [ CLI ENTRY ] ###############################################################
 421
 422 if __name__ == "__main__":
 423     args = parse_cli_args()
 424
 425     if args.mode == "setup":
 426         if args.send_message:
 427             raise NotImplementedError()
 428
 429         do_setup(args.extensions, debug_commands=args.debug_commands)
 430
 431     elif args.mode == "massage":
 432         with open(args.cmdpath, "w") as cmd_f:
 433             do_massage(
 434                 args.MAILDRAFT,
 435                 cmd_f,
 436                 extensions=args.extensions,
 437                 debug_commands=args.debug_commands,
 438                 debug_walk=args.debug_walk,
 439             )
 440
 441
 442 # [ TESTS ] ###################################################################
 443
 444 try:
 445     import pytest
 446
 447     class Tests:
 448         @pytest.fixture
 449         def const1(self):
 450             return "CONSTANT STRING 1"
 451
 452         @pytest.fixture
 453         def const2(self):
 454             return "CONSTANT STRING 2"
 455
 456         # NOTE: tests using the capsys fixture must specify sys.stdout to the
 457         # functions they call, else old stdout is used and not captured
 458
 459         def test_MuttCommands_cmd(self, const1, const2, capsys):
 460             "Assert order of commands"
 461             cmds = MuttCommands(out_f=sys.stdout)
 462             cmds.cmd(const1)
 463             cmds.cmd(const2)
 464             cmds.flush()
 465             captured = capsys.readouterr()
 466             assert captured.out == "\n".join((const1, const2, ""))
 467
 468         def test_MuttCommands_push(self, const1, const2, capsys):
 469             "Assert reverse order of pushes"
 470             cmds = MuttCommands(out_f=sys.stdout)
 471             cmds.push(const1)
 472             cmds.push(const2)
 473             cmds.flush()
 474             captured = capsys.readouterr()
 475             assert (
 476                 captured.out
 477                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
 478             )
 479
 480         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
 481             "Assert reverse order of pushes"
 482             cmds = MuttCommands(out_f=sys.stdout)
 483             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
 484             for i in range(2):
 485                 cmds.cmd(lines[4 * i + 0])
 486                 cmds.cmd(lines[4 * i + 1])
 487                 cmds.push(lines[4 * i + 2])
 488                 cmds.push(lines[4 * i + 3])
 489             cmds.flush()
 490
 491             captured = capsys.readouterr()
 492             lines_out = captured.out.splitlines()
 493             assert lines[0] in lines_out[0]
 494             assert lines[1] in lines_out[1]
 495             assert lines[7] in lines_out[2]
 496             assert lines[6] in lines_out[3]
 497             assert lines[3] in lines_out[4]
 498             assert lines[2] in lines_out[5]
 499             assert lines[4] in lines_out[6]
 500             assert lines[5] in lines_out[7]
 501
 502         @pytest.fixture
 503         def basic_mime_tree(self):
 504             return Multipart(
 505                 "relative",
 506                 children=[
 507                     Multipart(
 508                         "alternative",
 509                         children=[
 510                             Part(
 511                                 "text",
 512                                 "plain",
 513                                 "part.txt",
 514                                 desc="Plain",
 515                                 orig=True,
 516                             ),
 517                             Part("text", "html", "part.html", desc="HTML"),
 518                         ],
 519                         desc="Alternative",
 520                     ),
 521                     Part(
 522                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
 523                     ),
 524                 ],
 525                 desc="Related",
 526             )
 527
 528         def test_MIMETreeDFWalker_depth_first_walk(self, basic_mime_tree):
 529             mimetree = MIMETreeDFWalker()
 530
 531             items = []
 532
 533             def visitor_fn(item, stack, debugprint):
 534                 items.append((item, len(stack)))
 535
 536             mimetree.walk(basic_mime_tree, visitor_fn=visitor_fn)
 537             assert len(items) == 5
 538             assert items[0][0].subtype == "plain"
 539             assert items[0][1] == 2
 540             assert items[1][0].subtype == "html"
 541             assert items[1][1] == 2
 542             assert items[2][0].subtype == "alternative"
 543             assert items[2][1] == 1
 544             assert items[3][0].subtype == "png"
 545             assert items[3][1] == 1
 546             assert items[4][0].subtype == "relative"
 547             assert items[4][1] == 0
 548
 549         def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
 550             mimetree = MIMETreeDFWalker()
 551             items = []
 552
 553             def visitor_fn(item, stack, debugprint):
 554                 items.append(item)
 555
 556             mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
 557             assert items[-1].subtype == "mixed"
 558
 559         def test_MIMETreeDFWalker_visitor_in_constructor(
 560             self, basic_mime_tree
 561         ):
 562             items = []
 563
 564             def visitor_fn(item, stack, debugprint):
 565                 items.append(item)
 566
 567             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
 568             mimetree.walk(basic_mime_tree)
 569             assert len(items) == 5
 570
 571         def test_do_setup_no_extensions(self, const1, capsys):
 572             "Assert basics about the setup command output"
 573             do_setup(temppath=const1, out_f=sys.stdout)
 574             captout = capsys.readouterr()
 575             lines = captout.out.splitlines()
 576             assert lines[2].endswith(f'{const1}"')
 577             assert lines[4].endswith(const1)
 578             assert "first-entry" in lines[-1]
 579             assert "edit-file" in lines[-1]
 580
 581         def test_do_setup_extensions(self, const1, const2, capsys):
 582             "Assert that extensions are passed to editor"
 583             do_setup(
 584                 temppath=const1, extensions=[const2, const1], out_f=sys.stdout
 585             )
 586             captout = capsys.readouterr()
 587             lines = captout.out.splitlines()
 588             # assert comma-separated list of extensions passed
 589             assert lines[2].endswith(f'{const2},{const1}"')
 590             assert lines[4].endswith(const1)
 591
 592         def test_do_massage_basic(self, const1, capsys):
 593             def converter(maildraft, extensions):
 594                 return Part("text", "plain", "/dev/null", orig=True)
 595
 596             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
 597             captured = capsys.readouterr()
 598             assert (
 599                 captured.out.strip()
 600                 == """\
 601             set editor="$my_editor"
 602             set edit_headers="$my_edit_headers"
 603             unset my_editor
 604             unset my_edit_headers
 605             source 'rm -f pytest_internal_file|'
 606             unset my_mdwn_postprocess_cmd_file
 607             """.replace(
 608                     "            ", ""
 609                 ).strip()
 610             )
 611
 612         def test_do_massage_fulltree(self, const1, basic_mime_tree, capsys):
 613             def converter(maildraft, extensions):
 614                 return basic_mime_tree
 615
 616             do_massage(maildraft=const1, cmd_f=sys.stdout, converter=converter)
 617             captured = capsys.readouterr()
 618             lines = captured.out.splitlines()[4:][::-1]
 619             assert "Related" in lines.pop()
 620             assert "group-related" in lines.pop()
 621             assert "tag-entry" in lines.pop()
 622             assert "Logo" in lines.pop()
 623             assert "content-id" in lines.pop()
 624             assert "toggle-unlink" in lines.pop()
 625             assert "logo.png" in lines.pop()
 626             assert "tag-entry" in lines.pop()
 627             assert "Alternative" in lines.pop()
 628             assert "group-alternatives" in lines.pop()
 629             assert "tag-entry" in lines.pop()
 630             assert "HTML" in lines.pop()
 631             assert "toggle-unlink" in lines.pop()
 632             assert "part.html" in lines.pop()
 633             assert "tag-entry" in lines.pop()
 634             assert "Plain" in lines.pop()
 635
 636 except ImportError:
 637     pass