buildmimetree.py: handle image URLs starting with cid:

[etc/neomutt.git] / .config / neomutt / buildmimetree.py

diff --git a/.config/neomutt/buildmimetree.py b/.config/neomutt/buildmimetree.py
index 0b62aa4f25b14dda3316db4c4a58c3dcf6794fdb..e2a23a741bbee012d1fd42c6eecf3fbc5fcef7e6 100755 (executable)
--- a/.config/neomutt/buildmimetree.py
+++ b/.config/neomutt/buildmimetree.py
@@ -8,7 +8,7 @@
 #   neomuttrc (needs to be a single line):
 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
 #     macro compose B "\
 #   neomuttrc (needs to be a single line):
 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
 #     macro compose B "\

-#       <enter-command> source '$my_confdir/buildmimetree.py setup \
+#       <enter-command> source '$my_confdir/buildmimetree.py \

 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
 #     " "Convert message into a modern MIME tree with inline images"
 #       --tempdir $tempdir --extensions $my_mdwn_extensions|'<enter>\
 #       <enter-command> sourc e \$my_mdwn_postprocess_cmd_file<enter>\
 #     " "Convert message into a modern MIME tree with inline images"


@@ -23,8 +23,8 @@
 #   - python3-markdown
 # Optional:
 #   - pytest
 #   - python3-markdown
 # Optional:
 #   - pytest

-#   - Pynliner
-#   - Pygments, if installed, then syntax highlighting is enabled
+#   - Pynliner, provides --css-file and thus inline styling of HTML output
+#   - Pygments, then syntax highlighting for fenced code is enabled

 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
 #
 # Latest version:
 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py


@@ -55,79 +55,66 @@ def parse_cli_args(*args, **kwargs):
         )
     )
     parser.epilog = (
         )
     )
     parser.epilog = (

-        "Copyright © 2022 martin f. krafft <madduck@madduck.net>.\n"
+        "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"

         "Released under the MIT licence"
     )
 
         "Released under the MIT licence"
     )
 

-    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
-    parser_setup = subp.add_parser("setup", help="Setup phase")
-    parser_massage = subp.add_parser("massage", help="Massaging phase")
-
-    parser_setup.add_argument(
+    parser.add_argument(

         "--extensions",
         type=str,
         default="",
         "--extensions",
         type=str,
         default="",

-        help="Markdown extension to use (comma-separated list)"
+        help="Markdown extension to use (comma-separated list)",

     )
 
     )
 

-    parser_setup.add_argument(
+    if _PYNLINER:
+        parser.add_argument(
+            "--css-file",
+            type=str,
+            default="",
+            help="CSS file to merge with the final HTML",
+        )
+    else:
+        parser.set_defaults(css_file=None)
+
+    parser.add_argument(

         "--only-build",
         action="store_true",
         help="Only build, don't send the message",
     )
 
         "--only-build",
         action="store_true",
         help="Only build, don't send the message",
     )
 

-    parser_setup.add_argument(
+    parser.add_argument(

         "--tempdir",
         default=None,
         help="Specify temporary directory to use for attachments",
     )
 
         "--tempdir",
         default=None,
         help="Specify temporary directory to use for attachments",
     )
 

-    parser_setup.add_argument(
+    parser.add_argument(

         "--debug-commands",
         action="store_true",
         help="Turn on debug logging of commands generated to stderr",
     )
 
         "--debug-commands",
         action="store_true",
         help="Turn on debug logging of commands generated to stderr",
     )
 

-    parser_massage.add_argument(
+    subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
+    massage_p = subp.add_parser(
+        "massage", help="Massaging phase (internal use)"
+    )
+
+    massage_p.add_argument(

         "--write-commands-to",
         "--write-commands-to",

+        "-o",

         metavar="PATH",
         dest="cmdpath",
         metavar="PATH",
         dest="cmdpath",

+        required=True,

         help="Temporary file path to write commands to",
     )
 
         help="Temporary file path to write commands to",
     )
 

-    parser_massage.add_argument(
-        "--extensions",
-        metavar="EXTENSIONS",
-        type=str,
-        default="",
-        help="Markdown extension to use (comma-separated list)",
-    )
-
-    parser_massage.add_argument(
-        "--only-build",
-        action="store_true",
-        help="Only build, don't send the message",
-    )
-
-    parser_massage.add_argument(
-        "--tempdir",
-        default=None,
-        help="Specify temporary directory to use for attachments",
-    )
-
-    parser_massage.add_argument(
-        "--debug-commands",
-        action="store_true",
-        help="Turn on debug logging of commands generated to stderr",
-    )
-
-    parser_massage.add_argument(
+    massage_p.add_argument(

         "--debug-walk",
         action="store_true",
         help="Turn on debugging to stderr of the MIME tree walk",
     )
 
         "--debug-walk",
         action="store_true",
         help="Turn on debugging to stderr of the MIME tree walk",
     )
 

-    parser_massage.add_argument(
+    massage_p.add_argument(

         "MAILDRAFT",
         nargs="?",
         help="If provided, the script is invoked as editor on the mail draft",
         "MAILDRAFT",
         nargs="?",
         help="If provided, the script is invoked as editor on the mail draft",


@@ -185,11 +172,15 @@ class InlineImageExtension(Extension):
         return self._images
 
 
         return self._images
 
 

-def markdown_with_inline_image_support(text, *, extensions=None):
+def markdown_with_inline_image_support(
+    text, *, extensions=None, extension_configs=None
+):

     inline_image_handler = InlineImageExtension()
     extensions = extensions or []
     extensions.append(inline_image_handler)
     inline_image_handler = InlineImageExtension()
     extensions = extensions or []
     extensions.append(inline_image_handler)

-    mdwn = markdown.Markdown(extensions=extensions)
+    mdwn = markdown.Markdown(
+        extensions=extensions, extension_configs=extension_configs
+    )

     htmltext = mdwn.convert(text)
 
     images = inline_image_handler.get_images()
     htmltext = mdwn.convert(text)
 
     images = inline_image_handler.get_images()


@@ -204,6 +195,38 @@ def markdown_with_inline_image_support(text, *, extensions=None):
     return text, htmltext, images
 
 
     return text, htmltext, images
 
 

+# [ CSS STYLING ] #############################################################
+
+try:
+    import pynliner
+
+    _PYNLINER = True
+
+except ImportError:
+    _PYNLINER = False
+
+try:
+    from pygments.formatters import get_formatter_by_name
+
+    _CODEHILITE_CLASS = "codehilite"
+
+    _PYGMENTS_CSS = get_formatter_by_name(
+        "html", style="default"
+    ).get_style_defs(f".{_CODEHILITE_CLASS}")
+
+except ImportError:
+    _PYGMENTS_CSS = None
+
+
+def apply_styling(html, css):
+    return (
+        pynliner.Pynliner()
+        .from_string(html)
+        .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
+        .run()
+    )
+
+

 # [ PARTS GENERATION ] ########################################################
 
 
 # [ PARTS GENERATION ] ########################################################
 
 


@@ -229,6 +252,9 @@ class Multipart(
     def __str__(self):
         return f"<multipart/{self.subtype}> children={len(self.children)}"
 
     def __str__(self):
         return f"<multipart/{self.subtype}> children={len(self.children)}"
 

+    def __hash__(self):
+        return hash(str(self.subtype) + "".join(str(self.children)))
+

 
 def filewriter_fn(path, content, mode="w", **kwargs):
     with open(path, mode, **kwargs) as out_f:
 
 def filewriter_fn(path, content, mode="w", **kwargs):
     with open(path, mode, **kwargs) as out_f:


@@ -240,6 +266,9 @@ def collect_inline_images(
 ):
     relparts = []
     for path, info in images.items():
 ):
     relparts = []
     for path, info in images.items():

+        if path.startswith("cid:"):
+            continue
+

         data = request.urlopen(path)
 
         mimetype = data.headers["Content-Type"]
         data = request.urlopen(path)
 
         mimetype = data.headers["Content-Type"]


@@ -250,7 +279,12 @@ def collect_inline_images(
         filewriter_fn(path, data.read(), "w+b")
 
         relparts.append(
         filewriter_fn(path, data.read(), "w+b")
 
         relparts.append(

-            Part(*mimetype.split("/"), path, cid=info.cid, desc=f"Image: {info.desc}")
+            Part(
+                *mimetype.split("/"),
+                path,
+                cid=info.cid,
+                desc=f"Image: {info.desc}",
+            )

         )
 
     return relparts
         )
 
     return relparts


@@ -260,12 +294,19 @@ def convert_markdown_to_html(
     origtext,
     draftpath,
     *,
     origtext,
     draftpath,
     *,

+    cssfile=None,

     filewriter_fn=filewriter_fn,
     tempdir=None,
     extensions=None,
     filewriter_fn=filewriter_fn,
     tempdir=None,
     extensions=None,

+    extension_configs=None,

 ):
 ):

+    # TODO extension_configs need to be handled differently
+    extension_configs = extension_configs or {}
+    extension_configs.setdefault("pymdownx.highlight", {})
+    extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
+

     origtext, htmltext, images = markdown_with_inline_image_support(
     origtext, htmltext, images = markdown_with_inline_image_support(

-        origtext, extensions=extensions
+        origtext, extensions=extensions, extension_configs=extension_configs

     )
 
     filewriter_fn(draftpath, origtext, encoding="utf-8")
     )
 
     filewriter_fn(draftpath, origtext, encoding="utf-8")


@@ -273,6 +314,8 @@ def convert_markdown_to_html(
         "text", "plain", draftpath, "Plain-text version", orig=True
     )
 
         "text", "plain", draftpath, "Plain-text version", orig=True
     )
 

+    htmltext = apply_styling(htmltext, cssfile)
+

     htmlpath = draftpath.with_suffix(".html")
     filewriter_fn(
         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
     htmlpath = draftpath.with_suffix(".html")
     filewriter_fn(
         htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"


@@ -400,10 +443,8 @@ class MuttCommands:
 
 
 def do_setup(
 
 
 def do_setup(

-    extensions=None,

     *,
     out_f=sys.stdout,
     *,
     out_f=sys.stdout,

-    only_build=False,

     temppath=None,
     tempdir=None,
     debug_commands=False,
     temppath=None,
     tempdir=None,
     debug_commands=False,


@@ -413,15 +454,7 @@ def do_setup(
     )
     cmds = MuttCommands(out_f, debug=debug_commands)
 
     )
     cmds = MuttCommands(out_f, debug=debug_commands)
 

-    editor = f"{sys.argv[0]} massage --write-commands-to {temppath}"
-    if extensions:
-        editor = f'{editor} --extensions {extensions}'
-    if only_build:
-        editor = f'{editor} --only-build'
-    if tempdir:
-        editor = f"{editor} --tempdir {tempdir}"
-    if debug_commands:
-        editor = f"{editor} --debug-commands"
+    editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"

 
     cmds.cmd('set my_editor="$editor"')
     cmds.cmd('set my_edit_headers="$edit_headers"')
 
     cmds.cmd('set my_editor="$editor"')
     cmds.cmd('set my_edit_headers="$edit_headers"')


@@ -438,6 +471,7 @@ def do_massage(
     cmd_f,
     *,
     extensions=None,
     cmd_f,
     *,
     extensions=None,

+    cssfile=None,

     converter=convert_markdown_to_html,
     only_build=False,
     tempdir=None,
     converter=convert_markdown_to_html,
     only_build=False,
     tempdir=None,


@@ -463,7 +497,13 @@ def do_massage(
     cmds.flush()
 
     extensions = extensions.split(",") if extensions else []
     cmds.flush()
 
     extensions = extensions.split(",") if extensions else []

-    tree = converter(draft_f.read(), draftpath, tempdir=tempdir, extensions=extensions)
+    tree = converter(
+        draft_f.read(),
+        draftpath,
+        cssfile=cssfile,
+        tempdir=tempdir,
+        extensions=extensions,
+    )

 
     mimetree = MIMETreeDFWalker(debug=debug_walk)
 
 
     mimetree = MIMETreeDFWalker(debug=debug_walk)
 


@@ -473,7 +513,7 @@ def do_massage(
         depth-first, and responsible for telling NeoMutt how to assemble
         the tree.
         """
         depth-first, and responsible for telling NeoMutt how to assemble
         the tree.
         """

-        KILL_LINE=r'\Ca\Ck'
+        KILL_LINE = r"\Ca\Ck"

 
         if isinstance(item, Part):
             # We've hit a leaf-node, i.e. an alternative or a related part
 
         if isinstance(item, Part):
             # We've hit a leaf-node, i.e. an alternative or a related part


@@ -549,10 +589,8 @@ def do_massage(
 if __name__ == "__main__":
     args = parse_cli_args()
 
 if __name__ == "__main__":
     args = parse_cli_args()
 

-    if args.mode == "setup":
+    if args.mode is None:

         do_setup(
         do_setup(

-            args.extensions,
-            only_build=args.only_build,

             tempdir=args.tempdir,
             debug_commands=args.debug_commands,
         )
             tempdir=args.tempdir,
             debug_commands=args.debug_commands,
         )


@@ -566,6 +604,7 @@ if __name__ == "__main__":
                 pathlib.Path(args.MAILDRAFT),
                 cmd_f,
                 extensions=args.extensions,
                 pathlib.Path(args.MAILDRAFT),
                 cmd_f,
                 extensions=args.extensions,

+                cssfile=args.css_file,

                 only_build=args.only_build,
                 tempdir=args.tempdir,
                 debug_commands=args.debug_commands,
                 only_build=args.only_build,
                 tempdir=args.tempdir,
                 debug_commands=args.debug_commands,


@@ -703,33 +742,12 @@ try:
             mimetree.walk(basic_mime_tree)
             assert len(items) == 5
 
             mimetree.walk(basic_mime_tree)
             assert len(items) == 5
 

-        def test_do_setup_no_extensions(self, const1, capsys):
-            "Assert basics about the setup command output"
-            do_setup(temppath=const1, out_f=sys.stdout)
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            assert lines[2].endswith(f'{const1}"')
-            assert lines[4].endswith(const1)
-            assert "first-entry" in lines[-1]
-            assert "edit-file" in lines[-1]
-
-        def test_do_setup_extensions(self, const1, const2, capsys):
-            "Assert that extensions are passed to editor"
-            do_setup(
-                temppath=const1, extensions=[const2, const1], out_f=sys.stdout
-            )
-            captout = capsys.readouterr()
-            lines = captout.out.splitlines()
-            # assert comma-separated list of extensions passed
-            assert lines[2].endswith(f'{const2},{const1}"')
-            assert lines[4].endswith(const1)
-

         @pytest.fixture
         def string_io(self, const1, text=None):
             return StringIO(text or const1)
 
         def test_do_massage_basic(self, const1, string_io, capsys):
         @pytest.fixture
         def string_io(self, const1, text=None):
             return StringIO(text or const1)
 
         def test_do_massage_basic(self, const1, string_io, capsys):

-            def converter(drafttext, draftpath, extensions, tempdir):
+            def converter(drafttext, draftpath, cssfile, extensions, tempdir):

                 return Part("text", "plain", draftpath, orig=True)
 
             do_massage(
                 return Part("text", "plain", draftpath, orig=True)
 
             do_massage(


@@ -753,7 +771,7 @@ try:
         def test_do_massage_fulltree(
             self, string_io, const1, basic_mime_tree, capsys
         ):
         def test_do_massage_fulltree(
             self, string_io, const1, basic_mime_tree, capsys
         ):

-            def converter(drafttext, draftpath, extensions, tempdir):
+            def converter(drafttext, draftpath, cssfile, extensions, tempdir):

                 return basic_mime_tree
 
             do_massage(
                 return basic_mime_tree
 
             do_massage(


@@ -933,6 +951,23 @@ try:
             assert tree.children[1].path == written[0]
             assert written[1] == request.urlopen(test_png).read()
 
             assert tree.children[1].path == written[0]
             assert written[1] == request.urlopen(test_png).read()
 

+        def test_converter_tree_inline_image_cid(
+            self, const1, fake_filewriter
+        ):
+            text = f"![inline base64 image](cid:{const1})"
+            path = pathlib.Path(const1)
+            tree = convert_markdown_to_html(
+                text,
+                path,
+                filewriter_fn=fake_filewriter,
+                related_to_html_only=False,
+            )
+            assert len(tree.children) == 2
+            assert tree.children[0].cid != const1
+            assert tree.children[0].type != "image"
+            assert tree.children[1].cid != const1
+            assert tree.children[1].type != "image"
+

         def test_inline_image_collection(
             self, test_png, const1, const2, fake_filewriter
         ):
         def test_inline_image_collection(
             self, test_png, const1, const2, fake_filewriter
         ):


@@ -949,5 +984,22 @@ try:
             assert relparts[0].cid == const1
             assert relparts[0].desc.endswith(const2)
 
             assert relparts[0].cid == const1
             assert relparts[0].desc.endswith(const2)
 

+        def test_apply_stylesheet(self):
+            if _PYNLINER:
+                html = "<p>Hello, world!</p>"
+                css = "p { color:red }"
+                out = apply_styling(html, css)
+                assert 'p style="color' in out
+
+        def test_apply_stylesheet_pygments(self):
+            if _PYGMENTS_CSS:
+                html = (
+                    f'<div class="{_CODEHILITE_CLASS}">'
+                    "<pre>def foo():\n    return</pre></div>"
+                )
+                out = apply_styling(html, _PYGMENTS_CSS)
+                assert f'{_CODEHILITE_CLASS}" style="' in out
+
+

 except ImportError:
     pass
 except ImportError:
     pass