]> git.madduck.net Git - etc/neomutt.git/blobdiff - .config/neomutt/buildmimetree.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

buildmimetree.py: list only becomes mp/mixed if len>1
[etc/neomutt.git] / .config / neomutt / buildmimetree.py
index bd446bf1b196a8662d6a74349c73c5655583e5ed..779f60b9f449edc56efb473c90b058768efa4bad 100755 (executable)
@@ -6,8 +6,10 @@
 #
 # Configuration:
 #   neomuttrc (needs to be a single line):
+#     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
 #     macro compose B "\
-#       <enter-command> source '$my_confdir/buildmimetree.py setup|'<enter>\
+#       <enter-command> source '$my_confdir/buildmimetree.py \
+#       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
 #     " "Convert message into a modern MIME tree with inline images"
 #
@@ -21,8 +23,8 @@
 #   - python3-markdown
 # Optional:
 #   - pytest
-#   - Pynliner
-#   - Pygments, if installed, then syntax highlighting is enabled
+#   - Pynliner, provides --css-file and thus inline styling of HTML output
+#   - Pygments, then syntax highlighting for fenced code is enabled
 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
@@ -53,65 +55,66 @@ def parse_cli_args(*args, **kwargs):
         )
     )
     parser.epilog = (
-        "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
+        "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
         "Released under the MIT licence"
     )
 
-    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
-    parser_setup = subp.add_parser("setup", help="Setup phase")
-    parser_massage = subp.add_parser("massage", help="Massaging phase")
-
-    parser_setup.add_argument(
-        "--debug-commands",
-        action="store_true",
-        help="Turn on debug logging of commands generated to stderr",
+    parser.add_argument(
+        "--extensions",
+        type=str,
+        default="",
+        help="Markdown extension to use (comma-separated list)",
     )
 
-    parser_setup.add_argument(
-        "--extension",
-        "-x",
-        metavar="EXTENSION",
-        dest="extensions",
-        nargs="?",
-        default=[],
-        action="append",
-        help="Markdown extension to add to the list of extensions use",
-    )
+    if _PYNLINER:
+        parser.add_argument(
+            "--css-file",
+            type=str,
+            default="",
+            help="CSS file to merge with the final HTML",
+        )
+    else:
+        parser.set_defaults(css_file=None)
 
-    parser_setup.add_argument(
-        "--send-message",
+    parser.add_argument(
+        "--only-build",
         action="store_true",
-        help="Generate command(s) to send the message after processing",
+        help="Only build, don't send the message",
     )
 
-    parser_massage.add_argument(
-        "--debug-commands",
-        action="store_true",
-        help="Turn on debug logging of commands generated to stderr",
+    parser.add_argument(
+        "--tempdir",
+        default=None,
+        help="Specify temporary directory to use for attachments",
     )
 
-    parser_massage.add_argument(
-        "--debug-walk",
+    parser.add_argument(
+        "--debug-commands",
         action="store_true",
-        help="Turn on debugging to stderr of the MIME tree walk",
+        help="Turn on debug logging of commands generated to stderr",
     )
 
-    parser_massage.add_argument(
-        "--extensions",
-        metavar="EXTENSIONS",
-        type=str,
-        default="",
-        help="Markdown extension to use (comma-separated list)",
+    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
+    massage_p = subp.add_parser(
+        "massage", help="Massaging phase (internal use)"
     )
 
-    parser_massage.add_argument(
+    massage_p.add_argument(
         "--write-commands-to",
+        "-o",
         metavar="PATH",
         dest="cmdpath",
+        required=True,
         help="Temporary file path to write commands to",
     )
 
-    parser_massage.add_argument(
+    massage_p.add_argument(
+        "--debug-walk",
+        action="store_true",
+        help="Turn on debugging to stderr of the MIME tree walk",
+    )
+
+    massage_p.add_argument(
         "MAILDRAFT",
         nargs="?",
         help="If provided, the script is invoked as editor on the mail draft",
@@ -169,11 +172,15 @@ class InlineImageExtension(Extension):
         return self._images
 
 
-def markdown_with_inline_image_support(text, *, extensions=None):
+def markdown_with_inline_image_support(
+    text, *, extensions=None, extension_configs=None
+):
     inline_image_handler = InlineImageExtension()
     extensions = extensions or []
     extensions.append(inline_image_handler)
-    mdwn = markdown.Markdown(extensions=extensions)
+    mdwn = markdown.Markdown(
+        extensions=extensions, extension_configs=extension_configs
+    )
     htmltext = mdwn.convert(text)
 
     images = inline_image_handler.get_images()
@@ -188,6 +195,38 @@ def markdown_with_inline_image_support(text, *, extensions=None):
     return text, htmltext, images
 
 
+# [ CSS STYLING ] #############################################################
+
+try:
+    import pynliner
+
+    _PYNLINER = True
+
+except ImportError:
+    _PYNLINER = False
+
+try:
+    from pygments.formatters import get_formatter_by_name
+
+    _CODEHILITE_CLASS = "codehilite"
+
+    _PYGMENTS_CSS = get_formatter_by_name(
+        "html", style="default"
+    ).get_style_defs(f".{_CODEHILITE_CLASS}")
+
+except ImportError:
+    _PYGMENTS_CSS = None
+
+
+def apply_styling(html, css):
+    return (
+        pynliner.Pynliner()
+        .from_string(html)
+        .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
+        .run()
+    )
+
+
 # [ PARTS GENERATION ] ########################################################
 
 
@@ -213,6 +252,9 @@ class Multipart(
     def __str__(self):
         return f"<multipart/{self.subtype}> children={len(self.children)}"
 
+    def __hash__(self):
+        return hash(str(self.subtype) + "".join(str(self.children)))
+
 
 def filewriter_fn(path, content, mode="w", **kwargs):
     with open(path, mode, **kwargs) as out_f:
@@ -224,6 +266,9 @@ def collect_inline_images(
 ):
     relparts = []
     for path, info in images.items():
+        if path.startswith("cid:"):
+            continue
+
         data = request.urlopen(path)
 
         mimetype = data.headers["Content-Type"]
@@ -234,17 +279,34 @@ def collect_inline_images(
         filewriter_fn(path, data.read(), "w+b")
 
         relparts.append(
-            Part(*mimetype.split("/"), path, cid=info.cid, desc=info.desc)
+            Part(
+                *mimetype.split("/"),
+                path,
+                cid=info.cid,
+                desc=f"Image: {info.desc}",
+            )
         )
 
     return relparts
 
 
 def convert_markdown_to_html(
-    origtext, draftpath, *, filewriter_fn=filewriter_fn, extensions=None
+    origtext,
+    draftpath,
+    *,
+    cssfile=None,
+    filewriter_fn=filewriter_fn,
+    tempdir=None,
+    extensions=None,
+    extension_configs=None,
 ):
+    # TODO extension_configs need to be handled differently
+    extension_configs = extension_configs or {}
+    extension_configs.setdefault("pymdownx.highlight", {})
+    extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
+
     origtext, htmltext, images = markdown_with_inline_image_support(
-        origtext, extensions=extensions
+        origtext, extensions=extensions, extension_configs=extension_configs
     )
 
     filewriter_fn(draftpath, origtext, encoding="utf-8")
@@ -252,6 +314,8 @@ def convert_markdown_to_html(
         "text", "plain", draftpath, "Plain-text version", orig=True
     )
 
+    htmltext = apply_styling(htmltext, cssfile)
+
     htmlpath = draftpath.with_suffix(".html")
     filewriter_fn(
         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
@@ -262,7 +326,9 @@ def convert_markdown_to_html(
         "alternative", [textpart, htmlpart], "Group of alternative content"
     )
 
-    imgparts = collect_inline_images(images, filewriter_fn=filewriter_fn)
+    imgparts = collect_inline_images(
+        images, tempdir=tempdir, filewriter_fn=filewriter_fn
+    )
     if imgparts:
         return Multipart(
             "relative", [altpart] + imgparts, "Group of related content"
@@ -273,17 +339,22 @@ def convert_markdown_to_html(
 
 class MIMETreeDFWalker:
     def __init__(self, *, visitor_fn=None, debug=False):
-        self._visitor_fn = visitor_fn
+        self._visitor_fn = visitor_fn or self._echovisit
         self._debug = debug
 
+    def _echovisit(self, node, ancestry, debugprint):
+        debugprint(f"node={node} ancestry={ancestry}")
+
     def walk(self, root, *, visitor_fn=None):
         """
         Recursive function to implement a depth-dirst walk of the MIME-tree
         rooted at `root`.
         """
-
         if isinstance(root, list):
-            root = Multipart("mixed", children=root)
+            if len(root) > 1:
+                root = Multipart("mixed", children=root)
+            else:
+                root = root[0]
 
         self._walk(
             root,
@@ -377,19 +448,18 @@ class MuttCommands:
 
 
 def do_setup(
-    extensions=None, *, out_f=sys.stdout, temppath=None, debug_commands=False
+    *,
+    out_f=sys.stdout,
+    temppath=None,
+    tempdir=None,
+    debug_commands=False,
 ):
-    extensions = extensions or []
     temppath = temppath or pathlib.Path(
-        tempfile.mkstemp(prefix="muttmdwn-")[1]
+        tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
     )
     cmds = MuttCommands(out_f, debug=debug_commands)
 
-    editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
-    if extensions:
-        editor = f'{editor} --extensions {",".join(extensions)}'
-    if debug_commands:
-        editor = f"{editor} --debug-commands"
+    editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 
     cmds.cmd('set my_editor="$editor"')
     cmds.cmd('set my_edit_headers="$edit_headers"')
@@ -406,7 +476,10 @@ def do_massage(
     cmd_f,
     *,
     extensions=None,
+    cssfile=None,
     converter=convert_markdown_to_html,
+    only_build=False,
+    tempdir=None,
     debug_commands=False,
     debug_walk=False,
 ):
@@ -429,7 +502,13 @@ def do_massage(
     cmds.flush()
 
     extensions = extensions.split(",") if extensions else []
-    tree = converter(draft_f.read(), draftpath, extensions=extensions)
+    tree = converter(
+        draft_f.read(),
+        draftpath,
+        cssfile=cssfile,
+        tempdir=tempdir,
+        extensions=extensions,
+    )
 
     mimetree = MIMETreeDFWalker(debug=debug_walk)
 
@@ -439,6 +518,8 @@ def do_massage(
         depth-first, and responsible for telling NeoMutt how to assemble
         the tree.
         """
+        KILL_LINE = r"\Ca\Ck"
+
         if isinstance(item, Part):
             # We've hit a leaf-node, i.e. an alternative or a related part
             # with actual content.
@@ -459,7 +540,7 @@ def do_massage(
             # information, then we might just as well update the NeoMutt
             # tree now:
             if item.cid:
-                cmds.push(f"<edit-content-id>\\Ca\\Ck{item.cid}<enter>")
+                cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
 
         elif isinstance(item, Multipart):
             # This node has children, but we already visited them (see
@@ -468,7 +549,7 @@ def do_massage(
             # appropriate grouping:
             if item.subtype == "alternative":
                 cmds.push("<group-alternatives>")
-            elif item.subtype == "relative":
+            elif item.subtype in ("relative", "related"):
                 cmds.push("<group-related>")
             elif item.subtype == "multilingual":
                 cmds.push("<group-multilingual>")
@@ -479,7 +560,7 @@ def do_massage(
 
         # If the item has a description, we might just as well add it
         if item.desc:
-            cmds.push(f"<edit-description>\\Ca\\Ck{item.desc}<enter>")
+            cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
 
         # Finally, if we're at non-root level, tag the new container,
         # as it might itself be part of a container, to be processed
@@ -494,6 +575,9 @@ def do_massage(
     # function
     mimetree.walk(tree, visitor_fn=visitor_fn)
 
+    if not only_build:
+        cmds.push("<send-message>")
+
     # Finally, cleanup. Since we're responsible for removing the temporary
     # file, how's this for a little hack?
     try:
@@ -510,11 +594,11 @@ def do_massage(
 if __name__ == "__main__":
     args = parse_cli_args()
 
-    if args.mode == "setup":
-        if args.send_message:
-            raise NotImplementedError()
-
-        do_setup(args.extensions, debug_commands=args.debug_commands)
+    if args.mode is None:
+        do_setup(
+            tempdir=args.tempdir,
+            debug_commands=args.debug_commands,
+        )
 
     elif args.mode == "massage":
         with open(args.MAILDRAFT, "r") as draft_f, open(
@@ -525,6 +609,9 @@ if __name__ == "__main__":
                 pathlib.Path(args.MAILDRAFT),
                 cmd_f,
                 extensions=args.extensions,
+                cssfile=args.css_file,
+                only_build=args.only_build,
+                tempdir=args.tempdir,
                 debug_commands=args.debug_commands,
                 debug_walk=args.debug_walk,
             )
@@ -638,14 +725,17 @@ try:
             assert items[4][0].subtype == "relative"
             assert items[4][1] == 0
 
-        def test_MIMETreeDFWalker_list_to_mixed(self, basic_mime_tree):
+        def test_MIMETreeDFWalker_list_to_mixed(self, const1):
             mimetree = MIMETreeDFWalker()
             items = []
 
             def visitor_fn(item, stack, debugprint):
                 items.append(item)
 
-            mimetree.walk([basic_mime_tree], visitor_fn=visitor_fn)
+            p = Part("text", "plain", const1)
+            mimetree.walk([p], visitor_fn=visitor_fn)
+            assert items[-1].subtype == "plain"
+            mimetree.walk([p, p], visitor_fn=visitor_fn)
             assert items[-1].subtype == "mixed"
 
         def test_MIMETreeDFWalker_visitor_in_constructor(
@@ -660,33 +750,12 @@ try:
             mimetree.walk(basic_mime_tree)
             assert len(items) == 5
 
-        def test_do_setup_no_extensions(self, const1, capsys):
-            "Assert basics about the setup command output"
-            do_setup(temppath=const1, out_f=sys.stdout)
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            assert lines[2].endswith(f'{const1}"')
-            assert lines[4].endswith(const1)
-            assert "first-entry" in lines[-1]
-            assert "edit-file" in lines[-1]
-
-        def test_do_setup_extensions(self, const1, const2, capsys):
-            "Assert that extensions are passed to editor"
-            do_setup(
-                temppath=const1, extensions=[const2, const1], out_f=sys.stdout
-            )
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            # assert comma-separated list of extensions passed
-            assert lines[2].endswith(f'{const2},{const1}"')
-            assert lines[4].endswith(const1)
-
         @pytest.fixture
         def string_io(self, const1, text=None):
             return StringIO(text or const1)
 
         def test_do_massage_basic(self, const1, string_io, capsys):
-            def converter(drafttext, draftpath, extensions):
+            def converter(drafttext, draftpath, cssfile, extensions, tempdir):
                 return Part("text", "plain", draftpath, orig=True)
 
             do_massage(
@@ -702,6 +771,7 @@ try:
             assert '="$my_edit_headers"' in lines.pop(0)
             assert "unset my_editor" == lines.pop(0)
             assert "unset my_edit_headers" == lines.pop(0)
+            assert "send-message" in lines.pop(0)
             assert "update-encoding" in lines.pop(0)
             assert "source 'rm -f " in lines.pop(0)
             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
@@ -709,7 +779,7 @@ try:
         def test_do_massage_fulltree(
             self, string_io, const1, basic_mime_tree, capsys
         ):
-            def converter(drafttext, draftpath, extensions):
+            def converter(drafttext, draftpath, cssfile, extensions, tempdir):
                 return basic_mime_tree
 
             do_massage(
@@ -721,6 +791,7 @@ try:
 
             captured = capsys.readouterr()
             lines = captured.out.splitlines()[4:]
+            assert "send-message" in lines.pop(0)
             assert "Related" in lines.pop(0)
             assert "group-related" in lines.pop(0)
             assert "tag-entry" in lines.pop(0)
@@ -864,7 +935,6 @@ try:
                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
             )
 
-
         def test_markdown_inline_image_processor_base64(self, test_png):
             text = f"![1px white inlined]({test_png})"
             text, html, images = markdown_with_inline_image_support(text)
@@ -889,5 +959,55 @@ try:
             assert tree.children[1].path == written[0]
             assert written[1] == request.urlopen(test_png).read()
 
+        def test_converter_tree_inline_image_cid(
+            self, const1, fake_filewriter
+        ):
+            text = f"![inline base64 image](cid:{const1})"
+            path = pathlib.Path(const1)
+            tree = convert_markdown_to_html(
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=False,
+            )
+            assert len(tree.children) == 2
+            assert tree.children[0].cid != const1
+            assert tree.children[0].type != "image"
+            assert tree.children[1].cid != const1
+            assert tree.children[1].type != "image"
+
+        def test_inline_image_collection(
+            self, test_png, const1, const2, fake_filewriter
+        ):
+            test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
+            relparts = collect_inline_images(
+                test_images, filewriter_fn=fake_filewriter
+            )
+
+            written = fake_filewriter.pop()
+            assert b"PNG" in written[1]
+
+            assert relparts[0].subtype == "png"
+            assert relparts[0].path == written[0]
+            assert relparts[0].cid == const1
+            assert relparts[0].desc.endswith(const2)
+
+        def test_apply_stylesheet(self):
+            if _PYNLINER:
+                html = "<p>Hello, world!</p>"
+                css = "p { color:red }"
+                out = apply_styling(html, css)
+                assert 'p style="color' in out
+
+        def test_apply_stylesheet_pygments(self):
+            if _PYGMENTS_CSS:
+                html = (
+                    f'<div class="{_CODEHILITE_CLASS}">'
+                    "<pre>def foo():\n    return</pre></div>"
+                )
+                out = apply_styling(html, _PYGMENTS_CSS)
+                assert f'{_CODEHILITE_CLASS}" style="' in out
+
+
 except ImportError:
     pass