.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions \
  13 #       --css-file $my_confdir/htmlmail.css |'<enter>\
  14 #       <enter-command> source \$my_mdwn_postprocess_cmd_file<enter>\
  15 #     " "Convert message into a modern MIME tree with inline images"
  16 #
  17 #     (Yes, we need to call source twice, as mutt only starts to process output
  18 #     from a source command when the command exits, and since we need to react
  19 #     to the output, we need to be invoked again, using a $my_ variable to pass
  20 #     information)
  21 #
  22 # Requirements:
  23 #   - python3
  24 #   - python3-markdown
  25 #   - python3-beautifulsoup4
  26 # Optional:
  27 #   - pytest
  28 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  29 #   - Pygments, then syntax highlighting for fenced code is enabled
  30 #
  31 # Running tests:
  32 #   pytest -x buildmimetree.py
  33 #
  34 # Latest version:
  35 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  36 #
  37 # Copyright © 2023–2024 martin f. krafft <madduck@madduck.net>
  38 # Released under the GPL-2+ licence, just like NeoMutt itself.
  39 #
  40
  41 import sys
  42 import os.path
  43 import pathlib
  44 import markdown
  45 import tempfile
  46 import argparse
  47 import re
  48 import mimetypes
  49 import bs4
  50 import xml.etree.ElementTree as etree
  51 import io
  52 import enum
  53 from contextlib import contextmanager
  54 from collections import namedtuple, OrderedDict
  55 from markdown.extensions import Extension
  56 from markdown.blockprocessors import BlockProcessor
  57 from markdown.inlinepatterns import (
  58     SimpleTextInlineProcessor,
  59     ImageInlineProcessor,
  60     IMAGE_LINK_RE,
  61 )
  62 from email.utils import make_msgid
  63 from urllib import request
  64
  65
  66 def parse_cli_args(*args, **kwargs):
  67     parser = argparse.ArgumentParser(
  68         description=(
  69             "NeoMutt helper to turn text/markdown email parts "
  70             "into full-fledged MIME trees"
  71         )
  72     )
  73     parser.epilog = (
  74         "Copyright © 2023 martin f. krafft <madduck@madduck.net>.\n"
  75         "Released under the MIT licence"
  76     )
  77
  78     parser.add_argument(
  79         "--extensions",
  80         metavar="EXT[,EXT[,EXT]]",
  81         type=str,
  82         default="",
  83         help="Markdown extension to use (comma-separated list)",
  84     )
  85
  86     if _PYNLINER:
  87         parser.add_argument(
  88             "--css-file",
  89             metavar="FILE",
  90             type=pathlib.Path,
  91             default=os.devnull,
  92             help="CSS file to merge with the final HTML",
  93         )
  94     else:
  95         parser.set_defaults(css_file=None)
  96
  97     parser.add_argument(
  98         "--related-to-html-only",
  99         action="store_true",
 100         help="Make related content be sibling to HTML parts only",
 101     )
 102
 103     def positive_integer(value):
 104         try:
 105             if int(value) > 0:
 106                 return int(value)
 107
 108         except ValueError:
 109             pass
 110
 111         raise ValueError("Must be a positive integer")
 112
 113     parser.add_argument(
 114         "--max-number-other-attachments",
 115         metavar="INTEGER",
 116         type=positive_integer,
 117         default=20,
 118         help="Maximum number of other attachments to expect",
 119     )
 120
 121     parser.add_argument(
 122         "--only-build",
 123         "--just-build",
 124         action="store_true",
 125         help="Only build, don't send the message",
 126     )
 127
 128     parser.add_argument(
 129         "--domain",
 130         help="Domain to use in content IDs",
 131     )
 132
 133     parser.add_argument(
 134         "--tempdir",
 135         metavar="DIR",
 136         type=pathlib.Path,
 137         help="Specify temporary directory to use for attachments",
 138     )
 139
 140     parser.add_argument(
 141         "--debug-commands",
 142         action="store_true",
 143         help="Turn on debug logging of commands generated to stderr",
 144     )
 145
 146     parser.add_argument(
 147         "--debug-walk",
 148         action="store_true",
 149         help="Turn on debugging to stderr of the MIME tree walk",
 150     )
 151
 152     parser.add_argument(
 153         "--dump-html",
 154         metavar="FILE",
 155         type=pathlib.Path,
 156         help="Write the generated HTML to the file",
 157     )
 158
 159     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
 160     massage_p = subp.add_parser(
 161         "massage", help="Massaging phase (internal use)"
 162     )
 163
 164     massage_p.add_argument(
 165         "--write-commands-to",
 166         "-o",
 167         metavar="FILE",
 168         dest="cmdpath",
 169         type=pathlib.Path,
 170         required=True,
 171         help="Temporary file path to write commands to",
 172     )
 173
 174     massage_p.add_argument(
 175         "MAILDRAFT",
 176         nargs="?",
 177         type=pathlib.Path,
 178         help="If provided, the script is invoked as editor on the mail draft",
 179     )
 180
 181     return parser.parse_args(*args, **kwargs)
 182
 183
 184 # [ FILE I/O HANDLING ] #######################################################
 185
 186
 187 class File:
 188     class Op(enum.Enum):
 189         R = enum.auto()
 190         W = enum.auto()
 191
 192     def __init__(self, path=None, mode="r", content=None, **kwargs):
 193         if path:
 194             if content:
 195                 raise RuntimeError("Cannot specify path and content for File")
 196
 197             self._path = (
 198                 path if isinstance(path, pathlib.Path) else pathlib.Path(path)
 199             )
 200         else:
 201             self._path = None
 202
 203         if content and not re.search(r"[r+]", mode):
 204             raise RuntimeError("Cannot specify content without read mode")
 205
 206         self._cache = {File.Op.R: [content] if content else [], File.Op.W: []}
 207         self._lastop = None
 208         self._mode = mode
 209         self._kwargs = kwargs
 210         self._file = None
 211
 212     def open(self):
 213         if self._path:
 214             self._file = open(self._path, self._mode, **self._kwargs)
 215         elif "b" in self._mode:
 216             self._file = io.BytesIO()
 217         else:
 218             self._file = io.StringIO()
 219
 220     def __enter__(self):
 221         self.open()
 222         return self
 223
 224     def __exit__(self, exc_type, exc_val, exc_tb):
 225         self.close()
 226
 227     def close(self):
 228         self._file.close()
 229         self._file = None
 230         self._cache[File.Op.R] = self._cache[File.Op.W]
 231         self._lastop = None
 232
 233     def _get_cache(self, op):
 234         return (b"" if "b" in self._mode else "").join(self._cache[op])
 235
 236     def _add_to_cache(self, op, s):
 237         self._cache[op].append(s)
 238
 239     def read(self, *, cache=True):
 240         if cache and self._cache[File.Op.R]:
 241             return self._get_cache(File.Op.R)
 242
 243         if self._lastop == File.Op.W:
 244             try:
 245                 self._file.seek(0)
 246             except io.UnsupportedOperation:
 247                 pass
 248
 249         self._lastop = File.Op.R
 250
 251         if cache:
 252             self._add_to_cache(File.Op.R, self._file.read())
 253             return self._get_cache(File.Op.R)
 254         else:
 255             return self._file.read()
 256
 257     def write(self, s, *, cache=True):
 258         if self._lastop == File.Op.R:
 259             try:
 260                 self._file.seek(0)
 261             except io.UnsupportedOperation:
 262                 pass
 263
 264         if cache:
 265             self._add_to_cache(File.Op.W, s)
 266
 267         self._cache[File.Op.R] = self._cache[File.Op.W]
 268
 269         written = self._file.write(s)
 270         self._file.flush()
 271         self._lastop = File.Op.W
 272         return written
 273
 274     path = property(lambda s: s._path)
 275
 276     def __repr__(self):
 277         return (
 278             f'<File path={self._path or "(buffered)"} open={bool(self._file)} '
 279             f"rcache={sum(len(c) for c in self._rcache) if self._rcache is not None else False} "
 280             f"wcache={sum(len(c) for c in self._wcache) if self._wcache is not None else False}>"
 281         )
 282
 283
 284 class FileFactory:
 285     def __init__(self):
 286         self._files = []
 287
 288     def __call__(self, path=None, mode="r", content=None, **kwargs):
 289         f = File(path, mode, content, **kwargs)
 290         self._files.append(f)
 291         return f
 292
 293     def __len__(self):
 294         return self._files.__len__()
 295
 296     def pop(self, idx=-1):
 297         return self._files.pop(idx)
 298
 299     def __getitem__(self, idx):
 300         return self._files.__getitem__(idx)
 301
 302     def __contains__(self, f):
 303         return self._files.__contains__(f)
 304
 305
 306 class FakeFileFactory(FileFactory):
 307     def __init__(self):
 308         super().__init__()
 309         self._paths2files = OrderedDict()
 310
 311     def __call__(self, path=None, mode="r", content=None, **kwargs):
 312         if path in self._paths2files:
 313             return self._paths2files[path]
 314
 315         f = super().__call__(None, mode, content, **kwargs)
 316         self._paths2files[path] = f
 317
 318         mypath = path
 319
 320         class FakeFile(File):
 321             path = mypath
 322
 323         # this is quality Python! We do this so that the fake file, which has
 324         # no path, fake-pretends to have a path for testing purposes.
 325
 326         f.__class__ = FakeFile
 327         return f
 328
 329     def __getitem__(self, path):
 330         return self._paths2files.__getitem__(path)
 331
 332     def get(self, path, default):
 333         return self._paths2files.get(path, default)
 334
 335     def pop(self, last=True):
 336         return self._paths2files.popitem(last)
 337
 338     def __repr__(self):
 339         return (
 340             f"<FakeFileFactory nfiles={len(self._files)} "
 341             f"paths={len(self._paths2files)}>"
 342         )
 343
 344
 345 # [ IMAGE HANDLING ] ##########################################################
 346
 347
 348 InlineImageInfo = namedtuple(
 349     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 350 )
 351
 352
 353 class ImageRegistry:
 354     def __init__(self):
 355         self._images = OrderedDict()
 356
 357     def register(self, path, description=None, *, domain=None):
 358         # path = str(pathlib.Path(path).expanduser())
 359         path = os.path.expanduser(path)
 360         if path.startswith("/"):
 361             path = f"file://{path}"
 362         cid = make_msgid(domain=domain)[1:-1]
 363         self._images[path] = InlineImageInfo(cid, description)
 364         return cid
 365
 366     def __iter__(self):
 367         return self._images.__iter__()
 368
 369     def __getitem__(self, idx):
 370         return self._images.__getitem__(idx)
 371
 372     def __len__(self):
 373         return self._images.__len__()
 374
 375     def items(self):
 376         return self._images.items()
 377
 378     def __repr__(self):
 379         return f"<ImageRegistry(items={len(self._images)})>"
 380
 381     def __str__(self):
 382         return self._images.__str__()
 383
 384
 385 class InlineImageExtension(Extension):
 386     class RelatedImageInlineProcessor(ImageInlineProcessor):
 387         def __init__(self, re, md, registry):
 388             super().__init__(re, md)
 389             self._registry = registry
 390
 391         def handleMatch(self, m, data):
 392             el, start, end = super().handleMatch(m, data)
 393             if "src" in el.attrib:
 394                 src = el.attrib["src"]
 395                 if "://" not in src or src.startswith("file://"):
 396                     # We only inline local content
 397                     cid = self._registry.register(
 398                         el.attrib["src"],
 399                         el.attrib.get("title", el.attrib.get("alt")),
 400                     )
 401                     el.attrib["src"] = f"cid:{cid}"
 402             return el, start, end
 403
 404     def __init__(self, registry):
 405         super().__init__()
 406         self._image_registry = registry
 407
 408     INLINE_PATTERN_NAME = "image_link"
 409
 410     def extendMarkdown(self, md):
 411         md.registerExtension(self)
 412         inline_image_proc = self.RelatedImageInlineProcessor(
 413             IMAGE_LINK_RE, md, self._image_registry
 414         )
 415         md.inlinePatterns.register(
 416             inline_image_proc, InlineImageExtension.INLINE_PATTERN_NAME, 150
 417         )
 418
 419
 420 def markdown_with_inline_image_support(
 421     text,
 422     *,
 423     mdwn=None,
 424     image_registry=None,
 425     extensions=None,
 426     extension_configs=None,
 427 ):
 428     registry = (
 429         image_registry if image_registry is not None else ImageRegistry()
 430     )
 431     inline_image_handler = InlineImageExtension(registry=registry)
 432     extensions = extensions or []
 433     extensions.append(inline_image_handler)
 434     mdwn = markdown.Markdown(
 435         extensions=extensions, extension_configs=extension_configs
 436     )
 437
 438     htmltext = mdwn.convert(text)
 439
 440     def replace_image_with_cid(matchobj):
 441         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 442             if m in registry:
 443                 return f"(cid:{registry[m].cid}"
 444         return matchobj.group(0)
 445
 446     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 447     return text, htmltext, registry, mdwn
 448
 449
 450 # [ CSS STYLING ] #############################################################
 451
 452
 453 try:
 454     import pynliner
 455
 456     _PYNLINER = True
 457
 458 except ImportError:
 459     _PYNLINER = False
 460
 461 try:
 462     from pygments.formatters import get_formatter_by_name
 463
 464     _CODEHILITE_CLASS = "codehilite"
 465
 466     _PYGMENTS_CSS = get_formatter_by_name(
 467         "html", style="default"
 468     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 469
 470 except ImportError:
 471     _PYGMENTS_CSS = None
 472
 473
 474 def apply_styling(html, css):
 475     return (
 476         pynliner.Pynliner()
 477         .from_string(html)
 478         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 479         .run()
 480     )
 481
 482
 483 # [ FORMAT=FLOWED HANDLING ] ##################################################
 484
 485
 486 class FormatFlowedNewlineExtension(Extension):
 487     FFNL_RE = r"(?!\S)(\s)\n"
 488
 489     def extendMarkdown(self, md):
 490         ffnl = SimpleTextInlineProcessor(self.FFNL_RE)
 491         md.inlinePatterns.register(ffnl, "ffnl", 125)
 492
 493
 494 # [ QUOTE HANDLING ] ##########################################################
 495
 496
 497 class QuoteToAdmonitionExtension(Extension):
 498     class BlockProcessor(BlockProcessor):
 499         RE = re.compile(r"(?:^|\n)>\s*(.*)")
 500
 501         def __init__(self, parser):
 502             super().__init__(parser)
 503             self._title = None
 504             self._disable = False
 505
 506         def test(self, parent, blocks):
 507             if self._disable:
 508                 return False
 509
 510             if markdown.util.nearing_recursion_limit():
 511                 return False
 512
 513             lines = blocks.splitlines()
 514             if len(lines) < 2:
 515                 if not self._title:
 516                     return False
 517
 518                 elif not self.RE.search(lines[0]):
 519                     return False
 520
 521                 return len(lines) > 0
 522
 523             elif not self.RE.search(lines[0]) and self.RE.search(lines[1]):
 524                 return True
 525
 526             elif self._title and self.RE.search(lines[1]):
 527                 return True
 528
 529             return False
 530
 531         def run(self, parent, blocks):
 532             quotelines = blocks.pop(0).splitlines()
 533
 534             cont = bool(self._title)
 535             if not self.RE.search(quotelines[0]):
 536                 self._title = quotelines.pop(0)
 537
 538             admonition = etree.SubElement(parent, "div")
 539             admonition.set(
 540                 "class", f"admonition quote{' continued' if cont else ''}"
 541             )
 542             self.parser.parseChunk(admonition, self._title)
 543
 544             admonition[0].set("class", "admonition-title")
 545             with self.disable():
 546                 self.parser.parseChunk(admonition, "\n".join(quotelines))
 547
 548         @contextmanager
 549         def disable(self):
 550             self._disable = True
 551             yield True
 552             self._disable = False
 553
 554         @classmethod
 555         def clean(klass, line):
 556             m = klass.RE.match(line)
 557             return m.group(1) if m else line
 558
 559     def extendMarkdown(self, md):
 560         md.registerExtension(self)
 561         email_quote_proc = self.BlockProcessor(md.parser)
 562         md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25)
 563
 564
 565 # [ PARTS GENERATION ] ########################################################
 566
 567
 568 class Part(
 569     namedtuple(
 570         "Part",
 571         ["type", "subtype", "path", "desc", "cid", "orig"],
 572         defaults=[None, None, False],
 573     )
 574 ):
 575     def __str__(self):
 576         ret = f"<{self.type}/{self.subtype}>"
 577         if self.cid:
 578             ret = f"{ret} cid:{self.cid}"
 579         if self.orig:
 580             ret = f"{ret} ORIGINAL"
 581         return ret
 582
 583
 584 class Multipart(
 585     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 586 ):
 587     def __str__(self):
 588         return f"<multipart/{self.subtype}> children={len(self.children)}"
 589
 590     def __hash__(self):
 591         return hash(str(self.subtype) + "".join(str(self.children)))
 592
 593
 594 def collect_inline_images(
 595     image_registry, *, tempdir=None, filefactory=FileFactory()
 596 ):
 597     relparts = []
 598     for path, info in image_registry.items():
 599         if path.startswith("cid:"):
 600             continue
 601
 602         data = request.urlopen(path)
 603
 604         mimetype = data.headers["Content-Type"]
 605         ext = mimetypes.guess_extension(mimetype)
 606         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 607         path = pathlib.Path(tempfilename[1])
 608
 609         with filefactory(path, "w+b") as out_f:
 610             out_f.write(data.read())
 611
 612         # filewriter_fn(path, data.read(), "w+b")
 613
 614         desc = (
 615             f'Inline image: "{info.desc}"'
 616             if info.desc
 617             else f"Inline image {str(len(relparts)+1)}"
 618         )
 619         relparts.append(
 620             Part(*mimetype.split("/"), path, cid=info.cid, desc=desc)
 621         )
 622
 623     return relparts
 624
 625
 626 EMAIL_SIG_SEP = "\n-- \n"
 627 HTML_SIG_MARKER = "=htmlsig "
 628
 629
 630 def make_html_doc(body, sig=None):
 631     ret = (
 632         "<!DOCTYPE html>\n"
 633         "<html>\n"
 634         "<head>\n"
 635         '<meta http-equiv="content-type" content="text/html; charset=UTF-8">\n'  # noqa: E501
 636         '<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'  # noqa: E501
 637         "</head>\n"
 638         "<body>\n"
 639         f"{body}\n"
 640     )
 641
 642     if sig:
 643         nl = "\n"
 644         ret = (
 645             f'{ret}<div id="signature"><span class="sig_separator">{EMAIL_SIG_SEP.strip(nl)}</span>\n'  # noqa: E501
 646             f"{sig}\n"
 647             "</div>"
 648         )
 649
 650     return f"{ret}\n  </body>\n</html>"
 651
 652
 653 def make_text_mail(text, sig=None):
 654     return EMAIL_SIG_SEP.join((text, sig)) if sig else text
 655
 656
 657 def extract_signature(text, *, filefactory=FileFactory()):
 658     parts = text.split(EMAIL_SIG_SEP, 1)
 659     if len(parts) == 1:
 660         return text, None, None
 661
 662     lines = parts[1].splitlines()
 663     if lines[0].startswith(HTML_SIG_MARKER):
 664         path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1])
 665         textsig = "\n".join(lines)
 666
 667         with filefactory(path.expanduser()) as sig_f:
 668             sig_input = sig_f.read()
 669
 670         soup = bs4.BeautifulSoup(sig_input, "html.parser")
 671
 672         style = str(soup.style.extract()) if soup.style else ""
 673         for sig_selector in (
 674             "#signature",
 675             "#signatur",
 676             "#emailsig",
 677             ".signature",
 678             ".signatur",
 679             ".emailsig",
 680             "body",
 681             "div",
 682         ):
 683             sig = soup.select_one(sig_selector)
 684             if sig:
 685                 break
 686
 687         if not sig:
 688             return parts[0], textsig, style + sig_input
 689
 690         if sig.attrs.get("id") == "signature":
 691             sig = "".join(str(c) for c in sig.children)
 692
 693         return parts[0], textsig, style + str(sig)
 694
 695     return parts[0], parts[1], None
 696
 697
 698 def convert_markdown_to_html(
 699     draft_f,
 700     *,
 701     related_to_html_only=False,
 702     css_f=None,
 703     htmldump_f=None,
 704     filefactory=FileFactory(),
 705     tempdir=None,
 706     extensions=None,
 707     extension_configs=None,
 708     domain=None,
 709 ):
 710     # TODO extension_configs need to be handled differently
 711     extension_configs = extension_configs or {}
 712     extension_configs.setdefault("pymdownx.highlight", {})[
 713         "css_class"
 714     ] = _CODEHILITE_CLASS
 715
 716     extensions = extensions or []
 717     extensions.append(FormatFlowedNewlineExtension())
 718     extensions.append(QuoteToAdmonitionExtension())
 719
 720     draft = draft_f.read()
 721     origtext, textsig, htmlsig = extract_signature(
 722         draft, filefactory=filefactory
 723     )
 724
 725     (
 726         origtext,
 727         htmltext,
 728         image_registry,
 729         mdwn,
 730     ) = markdown_with_inline_image_support(
 731         origtext, extensions=extensions, extension_configs=extension_configs
 732     )
 733
 734     if htmlsig:
 735         if not textsig:
 736             # TODO: decide what to do if there is no plain-text version
 737             raise NotImplementedError("HTML signature but no text alternative")
 738
 739         soup = bs4.BeautifulSoup(htmlsig, "html.parser")
 740         for img in soup.find_all("img"):
 741             uri = img.attrs["src"]
 742             desc = img.attrs.get("title", img.attrs.get("alt"))
 743             cid = image_registry.register(uri, desc, domain=domain)
 744             img.attrs["src"] = f"cid:{cid}"
 745
 746         htmlsig = str(soup)
 747
 748     elif textsig:
 749         (
 750             textsig,
 751             htmlsig,
 752             image_registry,
 753             mdwn,
 754         ) = markdown_with_inline_image_support(
 755             textsig,
 756             extensions=extensions,
 757             extension_configs=extension_configs,
 758             image_registry=image_registry,
 759             mdwn=mdwn,
 760         )
 761
 762     origtext = make_text_mail(origtext, textsig)
 763     draft_f.write(origtext)
 764     textpart = Part(
 765         "text", "plain", draft_f.path, "Plain-text version", orig=True
 766     )
 767
 768     htmltext = make_html_doc(htmltext, htmlsig)
 769     htmltext = apply_styling(htmltext, css_f.read() if css_f else None)
 770
 771     if draft_f.path:
 772         htmlpath = draft_f.path.with_suffix(".html")
 773     else:
 774         htmlpath = pathlib.Path(
 775             tempfile.mkstemp(suffix=".html", dir=tempdir)[1]
 776         )
 777     with filefactory(
 778         htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
 779     ) as out_f:
 780         out_f.write(htmltext)
 781     htmlpart = Part("text", "html", htmlpath, "HTML version")
 782
 783     if htmldump_f:
 784         htmldump_f.write(htmltext)
 785
 786     imgparts = collect_inline_images(
 787         image_registry, tempdir=tempdir, filefactory=filefactory
 788     )
 789
 790     if related_to_html_only:
 791         # If there are inline image part, they will be contained within a
 792         # multipart/related part along with the HTML part only
 793         if imgparts:
 794             # replace htmlpart with a multipart/related container of the HTML
 795             # parts and the images
 796             htmlpart = Multipart(
 797                 "relative", [htmlpart] + imgparts, "Group of related content"
 798             )
 799
 800         return Multipart(
 801             "alternative", [textpart, htmlpart], "Group of alternative content"
 802         )
 803
 804     else:
 805         # If there are inline image part, they will be siblings to the
 806         # multipart/alternative tree within a multipart/related part
 807         altpart = Multipart(
 808             "alternative", [textpart, htmlpart], "Group of alternative content"
 809         )
 810         if imgparts:
 811             return Multipart(
 812                 "relative", [altpart] + imgparts, "Group of related content"
 813             )
 814         else:
 815             return altpart
 816
 817
 818 class MIMETreeDFWalker:
 819     def __init__(self, *, visitor_fn=None, debug=False):
 820         self._visitor_fn = visitor_fn or self._echovisit
 821         self._debug = debug
 822
 823     def _echovisit(self, node, ancestry, debugprint):
 824         debugprint(f"node={node} ancestry={ancestry}")
 825
 826     def walk(self, root, *, visitor_fn=None):
 827         """
 828         Recursive function to implement a depth-dirst walk of the MIME-tree
 829         rooted at `root`.
 830         """
 831         if isinstance(root, list):
 832             if len(root) > 1:
 833                 root = Multipart("mixed", children=root)
 834             else:
 835                 root = root[0]
 836
 837         self._walk(
 838             root,
 839             ancestry=[],
 840             descendents=[],
 841             visitor_fn=visitor_fn or self._visitor_fn,
 842         )
 843
 844     def _walk(self, node, *, ancestry, descendents, visitor_fn):
 845         # Let's start by enumerating the parts at the current level. At the
 846         # root level, ancestry will be the empty list, and we expect a
 847         # multipart/* container at this level. Later, e.g. within a
 848         # mutlipart/alternative container, the subtree will just be the
 849         # alternative parts, while the top of the ancestry will be the
 850         # multipart/alternative container, which we will process after the
 851         # following loop.
 852
 853         lead = f"{'│ '*len(ancestry)}"
 854         if isinstance(node, Multipart):
 855             self.debugprint(
 856                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
 857             )
 858
 859             # Depth-first, so push the current container onto the ancestry
 860             # stack, then descend …
 861             ancestry.append(node)
 862             self.debugprint(lead + "│ " * 2)
 863             for child in node.children:
 864                 self._walk(
 865                     child,
 866                     ancestry=ancestry,
 867                     descendents=descendents,
 868                     visitor_fn=visitor_fn,
 869                 )
 870             assert ancestry.pop() == node
 871             sibling_descendents = descendents
 872             descendents.extend(node.children)
 873
 874         else:
 875             self.debugprint(f"{lead}├{node}")
 876             sibling_descendents = descendents
 877
 878         if False and ancestry:
 879             self.debugprint(lead[:-1] + " │")
 880
 881         if visitor_fn:
 882             visitor_fn(
 883                 node, ancestry, sibling_descendents, debugprint=self.debugprint
 884             )
 885
 886     def debugprint(self, s, **kwargs):
 887         if self._debug:
 888             print(s, file=sys.stderr, **kwargs)
 889
 890
 891 # [ RUN MODES ] ###############################################################
 892
 893
 894 class MuttCommands:
 895     """
 896     Stupid class to interface writing out Mutt commands. This is quite a hack
 897     to deal with the fact that Mutt runs "push" commands in reverse order, so
 898     all of a sudden, things become very complicated when mixing with "real"
 899     commands.
 900
 901     Hence we keep two sets of commands, and one set of pushes. Commands are
 902     added to the first until a push is added, after which commands are added to
 903     the second set of commands.
 904
 905     On flush(), the first set is printed, followed by the pushes in reverse,
 906     and then the second set is printed. All 3 sets are then cleared.
 907     """
 908
 909     def __init__(self, out_f=sys.stdout, *, debug=False):
 910         self._cmd1, self._push, self._cmd2 = [], [], []
 911         self._out_f = out_f
 912         self._debug = debug
 913
 914     def cmd(self, s):
 915         self.debugprint(s)
 916         if self._push:
 917             self._cmd2.append(s)
 918         else:
 919             self._cmd1.append(s)
 920
 921     def push(self, s):
 922         s = s.replace('"', r"\"")
 923         s = f'push "{s}"'
 924         self.debugprint(s)
 925         self._push.insert(0, s)
 926
 927     def flush(self):
 928         print(
 929             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 930         )
 931         self._cmd1, self._push, self._cmd2 = [], [], []
 932
 933     def debugprint(self, s, **kwargs):
 934         if self._debug:
 935             print(s, file=sys.stderr, **kwargs)
 936
 937
 938 def do_setup(
 939     *,
 940     out_f=sys.stdout,
 941     temppath=None,
 942     tempdir=None,
 943     debug_commands=False,
 944 ):
 945     temppath = temppath or pathlib.Path(
 946         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 947     )
 948     cmds = MuttCommands(out_f, debug=debug_commands)
 949
 950     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 951
 952     cmds.cmd('set my_editor="$editor"')
 953     cmds.cmd('set my_edit_headers="$edit_headers"')
 954     cmds.cmd(f'set editor="{editor}"')
 955     cmds.cmd("unset edit_headers")
 956     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 957     cmds.push("<first-entry><edit-file>")
 958     cmds.flush()
 959
 960
 961 def do_massage(
 962     draft_f,
 963     cmd_f,
 964     *,
 965     extensions=None,
 966     css_f=None,
 967     htmldump_f=None,
 968     converter=convert_markdown_to_html,
 969     related_to_html_only=True,
 970     only_build=False,
 971     max_other_attachments=20,
 972     tempdir=None,
 973     domain=None,
 974     debug_commands=False,
 975     debug_walk=False,
 976 ):
 977     # Here's the big picture: we're being invoked as the editor on the email
 978     # draft, and whatever commands we write to the file given as cmdpath will
 979     # be run by the second source command in the macro definition.
 980
 981     # Let's start by cleaning up what the setup did (see above), i.e. we
 982     # restore the $editor and $edit_headers variables, and also unset the
 983     # variable used to identify the command file we're currently writing
 984     # to.
 985     cmds = MuttCommands(cmd_f, debug=debug_commands)
 986
 987     extensions = extensions.split(",") if extensions else []
 988     tree = converter(
 989         draft_f,
 990         css_f=css_f,
 991         htmldump_f=htmldump_f,
 992         related_to_html_only=related_to_html_only,
 993         tempdir=tempdir,
 994         extensions=extensions,
 995         domain=domain,
 996     )
 997
 998     mimetree = MIMETreeDFWalker(debug=debug_walk)
 999
1000     state = dict(pos=1, tags={}, parts=1)
1001
1002     def visitor_fn(item, ancestry, descendents, *, debugprint=None):
1003         """
1004         Visitor function called for every node (part) of the MIME tree,
1005         depth-first, and responsible for telling NeoMutt how to assemble
1006         the tree.
1007         """
1008         KILL_LINE = r"\Ca\Ck"
1009
1010         if isinstance(item, Part):
1011             # We've hit a leaf-node, i.e. an alternative or a related part
1012             # with actual content.
1013
1014             # Let's add the part
1015             if item.orig:
1016                 # The original source already exists in the NeoMutt tree, but
1017                 # the underlying file may have been modified, so we need to
1018                 # update the encoding, but that's it:
1019                 cmds.push("<first-entry>")
1020                 cmds.push("<update-encoding>")
1021
1022                 # We really just need to be able to assume that at this point,
1023                 # NeoMutt is at position 1, and that we've processed only this
1024                 # part so far. Nevermind about actual attachments, we can
1025                 # safely ignore those as they stay at the end.
1026                 assert state["pos"] == 1
1027                 assert state["parts"] == 1
1028             else:
1029                 # … whereas all other parts need to be added, and they're all
1030                 # considered to be temporary and inline:
1031                 cmds.push(f"<attach-file>{item.path}<enter>")
1032                 cmds.push("<toggle-unlink><toggle-disposition>")
1033
1034                 # This added a part at the end of the list of parts, and that's
1035                 # just how many parts we've seen so far, so it's position in
1036                 # the NeoMutt compose list is the count of parts
1037                 state["parts"] += 1
1038                 state["pos"] = state["parts"]
1039
1040             # If the item (including the original) comes with additional
1041             # information, then we might just as well update the NeoMutt
1042             # tree now:
1043             if item.cid:
1044                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
1045
1046             # Now for the biggest hack in this script, which is to handle
1047             # attachments, such as PDFs, that aren't related or alternatives.
1048             # The problem is that when we add an inline image, it always gets
1049             # appended to the list, i.e. inserted *after* other attachments.
1050             # Since we don't know the number of attachments, we also cannot
1051             # infer the postition of the new attachment. Therefore, we bubble
1052             # it all the way to the top, only to then move it down again:
1053             if state["pos"] > 1:  # skip for the first part
1054                 for i in range(max_other_attachments):
1055                     # could use any number here, but has to be larger than the
1056                     # number of possible attachments. The performance
1057                     # difference of using a high number is negligible.
1058                     # Bubble up the new part
1059                     cmds.push("<move-up>")
1060
1061                 # As we push the part to the right position in the list (i.e.
1062                 # the last of the subset of attachments this script added), we
1063                 # must handle the situation that subtrees are skipped by
1064                 # NeoMutt. Hence, the actual number of positions to move down
1065                 # is decremented by the number of descendents so far
1066                 # encountered.
1067                 for i in range(1, state["pos"] - len(descendents)):
1068                     cmds.push("<move-down>")
1069
1070         elif isinstance(item, Multipart):
1071             # This node has children, but we already visited them (see
1072             # above). The tags dictionary of State should contain a list of
1073             # their positions in the NeoMutt compose window, so iterate those
1074             # and tag the parts there:
1075             n_tags = len(state["tags"][item])
1076             for tag in state["tags"][item]:
1077                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
1078
1079             if item.subtype == "alternative":
1080                 cmds.push("<group-alternatives>")
1081             elif item.subtype in ("relative", "related"):
1082                 cmds.push("<group-related>")
1083             elif item.subtype == "multilingual":
1084                 cmds.push("<group-multilingual>")
1085             else:
1086                 raise NotImplementedError(
1087                     f"Handling of multipart/{item.subtype} is not implemented"
1088                 )
1089
1090             state["pos"] -= n_tags - 1
1091             state["parts"] += 1
1092
1093         else:
1094             # We should never get here
1095             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
1096
1097         # If the item has a description, we might just as well add it
1098         if item.desc:
1099             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
1100
1101         if ancestry:
1102             # If there's an ancestry, record the current (assumed) position in
1103             # the NeoMutt compose window as needed-to-tag by our direct parent
1104             # (i.e. the last item of the ancestry)
1105             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
1106
1107             lead = "│ " * (len(ancestry) + 1) + "* "
1108             debugprint(
1109                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
1110                 f"{lead}descendents={[d.subtype for d in descendents]}\n"
1111                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
1112                 f"{lead}pos={state['pos']}, parts={state['parts']}"
1113             )
1114
1115     # -----------------
1116     # End of visitor_fn
1117
1118     # Let's walk the tree and visit every node with our fancy visitor
1119     # function
1120     mimetree.walk(tree, visitor_fn=visitor_fn)
1121
1122     if not only_build:
1123         cmds.push("<send-message>")
1124
1125     # Finally, cleanup. Since we're responsible for removing the temporary
1126     # file, how's this for a little hack?
1127     try:
1128         filename = cmd_f.name
1129     except AttributeError:
1130         filename = "pytest_internal_file"
1131     cmds.cmd(f"source 'rm -f {filename}|'")
1132     cmds.cmd('set editor="$my_editor"')
1133     cmds.cmd('set edit_headers="$my_edit_headers"')
1134     cmds.cmd("unset my_editor")
1135     cmds.cmd("unset my_edit_headers")
1136     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
1137     cmds.flush()
1138
1139
1140 # [ CLI ENTRY ] ###############################################################
1141
1142 if __name__ == "__main__":
1143     args = parse_cli_args()
1144
1145     if args.mode is None:
1146         do_setup(
1147             tempdir=args.tempdir,
1148             debug_commands=args.debug_commands,
1149         )
1150
1151     elif args.mode == "massage":
1152         with (
1153             File(args.MAILDRAFT, "r+") as draft_f,
1154             File(args.cmdpath, "w") as cmd_f,
1155             File(args.css_file, "r") as css_f,
1156             File(args.dump_html, "w") as htmldump_f,
1157         ):
1158             do_massage(
1159                 draft_f,
1160                 cmd_f,
1161                 extensions=args.extensions,
1162                 css_f=css_f,
1163                 htmldump_f=htmldump_f,
1164                 related_to_html_only=args.related_to_html_only,
1165                 max_other_attachments=args.max_number_other_attachments,
1166                 only_build=args.only_build,
1167                 tempdir=args.tempdir,
1168                 domain=args.domain,
1169                 debug_commands=args.debug_commands,
1170                 debug_walk=args.debug_walk,
1171             )
1172
1173
1174 # [ TESTS ] ###################################################################
1175
1176 try:
1177     import pytest
1178
1179     class Tests:
1180         @pytest.fixture
1181         def const1(self):
1182             return "Curvature Vest Usher Dividing+T#iceps Senior"
1183
1184         @pytest.fixture
1185         def const2(self):
1186             return "Habitant Celestial 2litzy Resurf/ce Headpiece Harmonics"
1187
1188         @pytest.fixture
1189         def fakepath(self):
1190             return pathlib.Path("/does/not/exist")
1191
1192         @pytest.fixture
1193         def fakepath2(self):
1194             return pathlib.Path("/does/not/exist/either")
1195
1196         # NOTE: tests using the capsys fixture must specify sys.stdout to the
1197         # functions they call, else old stdout is used and not captured
1198
1199         @pytest.mark.muttctrl
1200         def test_MuttCommands_cmd(self, const1, const2, capsys):
1201             "Assert order of commands"
1202             cmds = MuttCommands(out_f=sys.stdout)
1203             cmds.cmd(const1)
1204             cmds.cmd(const2)
1205             cmds.flush()
1206             captured = capsys.readouterr()
1207             assert captured.out == "\n".join((const1, const2, ""))
1208
1209         @pytest.mark.muttctrl
1210         def test_MuttCommands_push(self, const1, const2, capsys):
1211             "Assert reverse order of pushes"
1212             cmds = MuttCommands(out_f=sys.stdout)
1213             cmds.push(const1)
1214             cmds.push(const2)
1215             cmds.flush()
1216             captured = capsys.readouterr()
1217             assert (
1218                 captured.out
1219                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
1220             )
1221
1222         @pytest.mark.muttctrl
1223         def test_MuttCommands_push_escape(self, const1, const2, capsys):
1224             cmds = MuttCommands(out_f=sys.stdout)
1225             cmds.push(f'"{const1}"')
1226             cmds.flush()
1227             captured = capsys.readouterr()
1228             assert f'"\\"{const1}\\""' in captured.out
1229
1230         @pytest.mark.muttctrl
1231         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
1232             "Assert reverse order of pushes"
1233             cmds = MuttCommands(out_f=sys.stdout)
1234             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
1235             for i in range(2):
1236                 cmds.cmd(lines[4 * i + 0])
1237                 cmds.cmd(lines[4 * i + 1])
1238                 cmds.push(lines[4 * i + 2])
1239                 cmds.push(lines[4 * i + 3])
1240             cmds.flush()
1241
1242             captured = capsys.readouterr()
1243             lines_out = captured.out.splitlines()
1244             assert lines[0] in lines_out[0]
1245             assert lines[1] in lines_out[1]
1246             assert lines[7] in lines_out[2]
1247             assert lines[6] in lines_out[3]
1248             assert lines[3] in lines_out[4]
1249             assert lines[2] in lines_out[5]
1250             assert lines[4] in lines_out[6]
1251             assert lines[5] in lines_out[7]
1252
1253         @pytest.fixture
1254         def mime_tree_related_to_alternative(self):
1255             return Multipart(
1256                 "relative",
1257                 children=[
1258                     Multipart(
1259                         "alternative",
1260                         children=[
1261                             Part(
1262                                 "text",
1263                                 "plain",
1264                                 "part.txt",
1265                                 desc="Plain",
1266                                 orig=True,
1267                             ),
1268                             Part("text", "html", "part.html", desc="HTML"),
1269                         ],
1270                         desc="Alternative",
1271                     ),
1272                     Part(
1273                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
1274                     ),
1275                 ],
1276                 desc="Related",
1277             )
1278
1279         @pytest.fixture
1280         def mime_tree_related_to_html(self):
1281             return Multipart(
1282                 "alternative",
1283                 children=[
1284                     Part(
1285                         "text",
1286                         "plain",
1287                         "part.txt",
1288                         desc="Plain",
1289                         orig=True,
1290                     ),
1291                     Multipart(
1292                         "relative",
1293                         children=[
1294                             Part("text", "html", "part.html", desc="HTML"),
1295                             Part(
1296                                 "text",
1297                                 "png",
1298                                 "logo.png",
1299                                 cid="logo.png",
1300                                 desc="Logo",
1301                             ),
1302                         ],
1303                         desc="Related",
1304                     ),
1305                 ],
1306                 desc="Alternative",
1307             )
1308
1309         @pytest.fixture
1310         def mime_tree_nested(self):
1311             return Multipart(
1312                 "relative",
1313                 children=[
1314                     Multipart(
1315                         "alternative",
1316                         children=[
1317                             Part(
1318                                 "text",
1319                                 "plain",
1320                                 "part.txt",
1321                                 desc="Plain",
1322                                 orig=True,
1323                             ),
1324                             Multipart(
1325                                 "alternative",
1326                                 children=[
1327                                     Part(
1328                                         "text",
1329                                         "plain",
1330                                         "part.txt",
1331                                         desc="Nested plain",
1332                                     ),
1333                                     Part(
1334                                         "text",
1335                                         "html",
1336                                         "part.html",
1337                                         desc="Nested HTML",
1338                                     ),
1339                                 ],
1340                                 desc="Nested alternative",
1341                             ),
1342                         ],
1343                         desc="Alternative",
1344                     ),
1345                     Part(
1346                         "text",
1347                         "png",
1348                         "logo.png",
1349                         cid="logo.png",
1350                         desc="Logo",
1351                     ),
1352                 ],
1353                 desc="Related",
1354             )
1355
1356         @pytest.mark.treewalk
1357         def test_MIMETreeDFWalker_depth_first_walk(
1358             self, mime_tree_related_to_alternative
1359         ):
1360             mimetree = MIMETreeDFWalker()
1361
1362             items = []
1363
1364             def visitor_fn(item, ancestry, descendents, debugprint):
1365                 items.append((item, len(ancestry), len(descendents)))
1366
1367             mimetree.walk(
1368                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
1369             )
1370             assert len(items) == 5
1371             assert items[0][0].subtype == "plain"
1372             assert items[0][1] == 2
1373             assert items[0][2] == 0
1374             assert items[1][0].subtype == "html"
1375             assert items[1][1] == 2
1376             assert items[1][2] == 0
1377             assert items[2][0].subtype == "alternative"
1378             assert items[2][1] == 1
1379             assert items[2][2] == 2
1380             assert items[3][0].subtype == "png"
1381             assert items[3][1] == 1
1382             assert items[3][2] == 2
1383             assert items[4][0].subtype == "relative"
1384             assert items[4][1] == 0
1385             assert items[4][2] == 4
1386
1387         @pytest.mark.treewalk
1388         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
1389             mimetree = MIMETreeDFWalker()
1390             items = []
1391
1392             def visitor_fn(item, ancestry, descendents, debugprint):
1393                 items.append(item)
1394
1395             p = Part("text", "plain", const1)
1396             mimetree.walk([p], visitor_fn=visitor_fn)
1397             assert items[-1].subtype == "plain"
1398             mimetree.walk([p, p], visitor_fn=visitor_fn)
1399             assert items[-1].subtype == "mixed"
1400
1401         @pytest.mark.treewalk
1402         def test_MIMETreeDFWalker_visitor_in_constructor(
1403             self, mime_tree_related_to_alternative
1404         ):
1405             items = []
1406
1407             def visitor_fn(item, ancestry, descendents, debugprint):
1408                 items.append(item)
1409
1410             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
1411             mimetree.walk(mime_tree_related_to_alternative)
1412             assert len(items) == 5
1413
1414         @pytest.fixture
1415         def string_io(self, const1, text=None):
1416             return StringIO(text or const1)
1417
1418         @pytest.mark.massage
1419         def test_do_massage_basic(self):
1420             def converter(draft_f, **kwargs):
1421                 return Part("text", "plain", draft_f.path, orig=True)
1422
1423             with File() as draft_f, File() as cmd_f:
1424                 do_massage(
1425                     draft_f=draft_f,
1426                     cmd_f=cmd_f,
1427                     converter=converter,
1428                 )
1429                 lines = cmd_f.read().splitlines()
1430
1431             assert "send-message" in lines.pop(0)
1432             assert "update-encoding" in lines.pop(0)
1433             assert "first-entry" in lines.pop(0)
1434             assert "source 'rm -f " in lines.pop(0)
1435             assert '="$my_editor"' in lines.pop(0)
1436             assert '="$my_edit_headers"' in lines.pop(0)
1437             assert "unset my_editor" == lines.pop(0)
1438             assert "unset my_edit_headers" == lines.pop(0)
1439             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
1440
1441         @pytest.mark.massage
1442         def test_do_massage_fulltree(self, mime_tree_related_to_alternative):
1443             def converter(draft_f, **kwargs):
1444                 return mime_tree_related_to_alternative
1445
1446             max_attachments = 5
1447
1448             with File() as draft_f, File() as cmd_f:
1449                 do_massage(
1450                     draft_f=draft_f,
1451                     cmd_f=cmd_f,
1452                     max_other_attachments=max_attachments,
1453                     converter=converter,
1454                 )
1455                 lines = cmd_f.read().splitlines()[:-6]
1456
1457             assert "first-entry" in lines.pop()
1458             assert "update-encoding" in lines.pop()
1459             assert "Plain" in lines.pop()
1460             assert "part.html" in lines.pop()
1461             assert "toggle-unlink" in lines.pop()
1462             for i in range(max_attachments):
1463                 assert "move-up" in lines.pop()
1464             assert "move-down" in lines.pop()
1465             assert "HTML" in lines.pop()
1466             assert "jump>1" in lines.pop()
1467             assert "jump>2" in lines.pop()
1468             assert "group-alternatives" in lines.pop()
1469             assert "Alternative" in lines.pop()
1470             assert "logo.png" in lines.pop()
1471             assert "toggle-unlink" in lines.pop()
1472             assert "content-id" in lines.pop()
1473             for i in range(max_attachments):
1474                 assert "move-up" in lines.pop()
1475             assert "move-down" in lines.pop()
1476             assert "Logo" in lines.pop()
1477             assert "jump>1" in lines.pop()
1478             assert "jump>4" in lines.pop()
1479             assert "group-related" in lines.pop()
1480             assert "Related" in lines.pop()
1481             assert "send-message" in lines.pop()
1482             assert len(lines) == 0
1483
1484         @pytest.mark.massage
1485         def test_mime_tree_relative_within_alternative(
1486             self, mime_tree_related_to_html
1487         ):
1488             def converter(draft_f, **kwargs):
1489                 return mime_tree_related_to_html
1490
1491             with File() as draft_f, File() as cmd_f:
1492                 do_massage(
1493                     draft_f=draft_f,
1494                     cmd_f=cmd_f,
1495                     converter=converter,
1496                 )
1497                 lines = cmd_f.read().splitlines()[:-6]
1498
1499             assert "first-entry" in lines.pop()
1500             assert "update-encoding" in lines.pop()
1501             assert "Plain" in lines.pop()
1502             assert "part.html" in lines.pop()
1503             assert "toggle-unlink" in lines.pop()
1504             assert "move-up" in lines.pop()
1505             while True:
1506                 top = lines.pop()
1507                 if "move-up" not in top:
1508                     break
1509             assert "move-down" in top
1510             assert "HTML" in lines.pop()
1511             assert "logo.png" in lines.pop()
1512             assert "toggle-unlink" in lines.pop()
1513             assert "content-id" in lines.pop()
1514             assert "move-up" in lines.pop()
1515             while True:
1516                 top = lines.pop()
1517                 if "move-up" not in top:
1518                     break
1519             assert "move-down" in top
1520             assert "move-down" in lines.pop()
1521             assert "Logo" in lines.pop()
1522             assert "jump>2" in lines.pop()
1523             assert "jump>3" in lines.pop()
1524             assert "group-related" in lines.pop()
1525             assert "Related" in lines.pop()
1526             assert "jump>1" in lines.pop()
1527             assert "jump>2" in lines.pop()
1528             assert "group-alternative" in lines.pop()
1529             assert "Alternative" in lines.pop()
1530             assert "send-message" in lines.pop()
1531             assert len(lines) == 0
1532
1533         @pytest.mark.massage
1534         def test_mime_tree_nested_trees_does_not_break_positioning(
1535             self, mime_tree_nested
1536         ):
1537             def converter(draft_f, **kwargs):
1538                 return mime_tree_nested
1539
1540             with File() as draft_f, File() as cmd_f:
1541                 do_massage(
1542                     draft_f=draft_f,
1543                     cmd_f=cmd_f,
1544                     converter=converter,
1545                 )
1546                 lines = cmd_f.read().splitlines()
1547
1548             while "logo.png" not in lines.pop():
1549                 pass
1550             lines.pop()
1551             assert "content-id" in lines.pop()
1552             assert "move-up" in lines.pop()
1553             while True:
1554                 top = lines.pop()
1555                 if "move-up" not in top:
1556                     break
1557             assert "move-down" in top
1558             # Due to the nested trees, the number of descendents of the sibling
1559             # actually needs to be considered, not just the nieces. So to move
1560             # from position 1 to position 6, it only needs one <move-down>
1561             # because that jumps over the entire sibling tree. Thus what
1562             # follows next must not be another <move-down>
1563             assert "Logo" in lines.pop()
1564
1565         @pytest.mark.converter
1566         def test_converter_tree_basic(self, fakepath, const1, fakefilefactory):
1567             with fakefilefactory(fakepath, content=const1) as draft_f:
1568                 tree = convert_markdown_to_html(
1569                     draft_f, filefactory=fakefilefactory
1570                 )
1571
1572             assert tree.subtype == "alternative"
1573             assert len(tree.children) == 2
1574             assert tree.children[0].subtype == "plain"
1575             assert tree.children[0].path == draft_f.path
1576             assert tree.children[0].orig
1577             assert tree.children[1].subtype == "html"
1578             assert tree.children[1].path == fakepath.with_suffix(".html")
1579
1580         @pytest.mark.converter
1581         def test_converter_writes(
1582             self, fakepath, fakefilefactory, const1, monkeypatch
1583         ):
1584             with fakefilefactory(fakepath, content=const1) as draft_f:
1585                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
1586
1587             html = fakefilefactory.pop()
1588             assert fakepath.with_suffix(".html") == html[0]
1589             assert const1 in html[1].read()
1590             text = fakefilefactory.pop()
1591             assert fakepath == text[0]
1592             assert const1 == text[1].read()
1593
1594         @pytest.mark.imgproc
1595         def test_markdown_inline_image_processor(self):
1596             imgpath1 = "file:/path/to/image.png"
1597             imgpath2 = "file:///path/to/image.png?url=params"
1598             imgpath3 = "/path/to/image.png"
1599             text = f"""![inline local image]({imgpath1})
1600                        ![image inlined
1601                          with newline]({imgpath2})
1602                        ![image local path]({imgpath3})"""
1603             text, html, images, mdwn = markdown_with_inline_image_support(text)
1604
1605             # local paths have been normalised to URLs:
1606             imgpath3 = f"file://{imgpath3}"
1607
1608             assert 'src="cid:' in html
1609             assert "](cid:" in text
1610             assert len(images) == 3
1611             assert imgpath1 in images
1612             assert imgpath2 in images
1613             assert imgpath3 in images
1614             assert images[imgpath1].cid != images[imgpath2].cid
1615             assert images[imgpath1].cid != images[imgpath3].cid
1616             assert images[imgpath2].cid != images[imgpath3].cid
1617
1618         @pytest.mark.imgproc
1619         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1620             imgpath = "file:///path/to/image.png"
1621             text = f'![inline local image]({imgpath} "{const1}")'
1622             text, html, images, mdwn = markdown_with_inline_image_support(text)
1623             assert images[imgpath].desc == const1
1624
1625         @pytest.mark.imgproc
1626         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1627             imgpath = "file:///path/to/image.png"
1628             text = f"![{const1}]({imgpath})"
1629             text, html, images, mdwn = markdown_with_inline_image_support(text)
1630             assert images[imgpath].desc == const1
1631
1632         @pytest.mark.imgproc
1633         def test_markdown_inline_image_processor_title_over_alt_desc(
1634             self, const1, const2
1635         ):
1636             imgpath = "file:///path/to/image.png"
1637             text = f'![{const1}]({imgpath} "{const2}")'
1638             text, html, images, mdwn = markdown_with_inline_image_support(text)
1639             assert images[imgpath].desc == const2
1640
1641         @pytest.mark.imgproc
1642         def test_markdown_inline_image_not_external(self):
1643             imgpath = "https://path/to/image.png"
1644             text = f"![inline image]({imgpath})"
1645             text, html, images, mdwn = markdown_with_inline_image_support(text)
1646
1647             assert 'src="cid:' not in html
1648             assert "](cid:" not in text
1649             assert len(images) == 0
1650
1651         @pytest.mark.imgproc
1652         def test_markdown_inline_image_local_file(self):
1653             imgpath = "/path/to/image.png"
1654             text = f"![inline image]({imgpath})"
1655             text, html, images, mdwn = markdown_with_inline_image_support(text)
1656
1657             for k, v in images.items():
1658                 assert k == f"file://{imgpath}"
1659                 break
1660
1661         @pytest.mark.imgproc
1662         def test_markdown_inline_image_expanduser(self):
1663             imgpath = pathlib.Path("~/image.png")
1664             text = f"![inline image]({imgpath})"
1665             text, html, images, mdwn = markdown_with_inline_image_support(text)
1666
1667             for k, v in images.items():
1668                 assert k == f"file://{imgpath.expanduser()}"
1669                 break
1670
1671         @pytest.fixture
1672         def test_png(self):
1673             return (
1674                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1675                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1676             )
1677
1678         @pytest.mark.imgproc
1679         def test_markdown_inline_image_processor_base64(self, test_png):
1680             text = f"![1px white inlined]({test_png})"
1681             text, html, images, mdwn = markdown_with_inline_image_support(text)
1682
1683             assert 'src="cid:' in html
1684             assert "](cid:" in text
1685             assert len(images) == 1
1686             assert test_png in images
1687
1688         @pytest.mark.converter
1689         def test_converter_tree_inline_image_base64(
1690             self, test_png, fakefilefactory
1691         ):
1692             text = f"![inline base64 image]({test_png})"
1693             with fakefilefactory(content=text) as draft_f:
1694                 tree = convert_markdown_to_html(
1695                     draft_f,
1696                     filefactory=fakefilefactory,
1697                     related_to_html_only=False,
1698                 )
1699             assert tree.subtype == "relative"
1700             assert tree.children[0].subtype == "alternative"
1701             assert tree.children[1].subtype == "png"
1702             written = fakefilefactory.pop()
1703             assert tree.children[1].path == written[0]
1704             assert b"PNG" in written[1].read()
1705
1706         @pytest.mark.converter
1707         def test_converter_tree_inline_image_base64_related_to_html(
1708             self, test_png, fakefilefactory
1709         ):
1710             text = f"![inline base64 image]({test_png})"
1711             with fakefilefactory(content=text) as draft_f:
1712                 tree = convert_markdown_to_html(
1713                     draft_f,
1714                     filefactory=fakefilefactory,
1715                     related_to_html_only=True,
1716                 )
1717             assert tree.subtype == "alternative"
1718             assert tree.children[1].subtype == "relative"
1719             assert tree.children[1].children[1].subtype == "png"
1720             written = fakefilefactory.pop()
1721             assert tree.children[1].children[1].path == written[0]
1722             assert b"PNG" in written[1].read()
1723
1724         @pytest.mark.converter
1725         def test_converter_tree_inline_image_cid(
1726             self, const1, fakefilefactory
1727         ):
1728             text = f"![inline base64 image](cid:{const1})"
1729             with fakefilefactory(content=text) as draft_f:
1730                 tree = convert_markdown_to_html(
1731                     draft_f,
1732                     filefactory=fakefilefactory,
1733                     related_to_html_only=False,
1734                 )
1735             assert len(tree.children) == 2
1736             assert tree.children[0].cid != const1
1737             assert tree.children[0].type != "image"
1738             assert tree.children[1].cid != const1
1739             assert tree.children[1].type != "image"
1740
1741         @pytest.fixture
1742         def fakefilefactory(self):
1743             return FakeFileFactory()
1744
1745         @pytest.mark.imgcoll
1746         def test_inline_image_collection(
1747             self, test_png, const1, const2, fakefilefactory
1748         ):
1749             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1750             relparts = collect_inline_images(
1751                 test_images, filefactory=fakefilefactory
1752             )
1753
1754             written = fakefilefactory.pop()
1755             assert b"PNG" in written[1].read()
1756
1757             assert relparts[0].subtype == "png"
1758             assert relparts[0].path == written[0]
1759             assert relparts[0].cid == const1
1760             assert const2 in relparts[0].desc
1761
1762         if _PYNLINER:
1763
1764             @pytest.mark.styling
1765             def test_apply_stylesheet(self):
1766                 html = "<p>Hello, world!</p>"
1767                 css = "p { color:red }"
1768                 out = apply_styling(html, css)
1769                 assert 'p style="color' in out
1770
1771             @pytest.mark.styling
1772             def test_apply_no_stylesheet(self, const1):
1773                 out = apply_styling(const1, None)
1774
1775             @pytest.mark.massage
1776             @pytest.mark.styling
1777             def test_massage_styling_to_converter(self):
1778                 css = "p { color:red }"
1779                 css_applied = []
1780
1781                 def converter(draft_f, css_f, **kwargs):
1782                     css = css_f.read()
1783                     css_applied.append(css)
1784                     return Part("text", "plain", draft_f.path, orig=True)
1785
1786                 with (
1787                     File() as draft_f,
1788                     File(mode="w") as cmd_f,
1789                     File(content=css) as css_f,
1790                 ):
1791                     do_massage(
1792                         draft_f=draft_f,
1793                         cmd_f=cmd_f,
1794                         css_f=css_f,
1795                         converter=converter,
1796                     )
1797                 assert css_applied[0] == css
1798
1799             @pytest.mark.converter
1800             @pytest.mark.styling
1801             def test_converter_apply_styles(
1802                 self, const1, monkeypatch, fakepath, fakefilefactory
1803             ):
1804                 css = "p { color:red }"
1805                 with (
1806                     monkeypatch.context() as m,
1807                     fakefilefactory(fakepath, content=const1) as draft_f,
1808                     fakefilefactory(content=css) as css_f,
1809                 ):
1810                     m.setattr(
1811                         markdown.Markdown,
1812                         "convert",
1813                         lambda s, t: f"<p>{t}</p>",
1814                     )
1815                     convert_markdown_to_html(
1816                         draft_f, css_f=css_f, filefactory=fakefilefactory
1817                     )
1818                 assert re.search(
1819                     r"color:.*red",
1820                     fakefilefactory[fakepath.with_suffix(".html")].read(),
1821                 )
1822
1823         if _PYGMENTS_CSS:
1824
1825             @pytest.mark.styling
1826             def test_apply_stylesheet_pygments(self):
1827                 html = (
1828                     f'<div class="{_CODEHILITE_CLASS}">'
1829                     "<pre>def foo():\n    return</pre></div>"
1830                 )
1831                 out = apply_styling(html, _PYGMENTS_CSS)
1832                 assert f'{_CODEHILITE_CLASS}" style="' in out
1833
1834         @pytest.mark.sig
1835         def test_signature_extraction_no_signature(self, const1):
1836             assert (const1, None, None) == extract_signature(const1)
1837
1838         @pytest.mark.sig
1839         def test_signature_extraction_just_text(self, const1, const2):
1840             origtext, textsig, htmlsig = extract_signature(
1841                 f"{const1}{EMAIL_SIG_SEP}{const2}"
1842             )
1843             assert origtext == const1
1844             assert textsig == const2
1845             assert htmlsig is None
1846
1847         @pytest.mark.sig
1848         def test_signature_extraction_html(
1849             self, fakepath, fakefilefactory, const1, const2
1850         ):
1851             sigconst = "HTML signature from {path} but as a string"
1852             sig = f'<div id="signature">{sigconst.format(path=fakepath)}</div>'
1853
1854             sig_f = fakefilefactory(fakepath, content=sig)
1855
1856             origtext, textsig, htmlsig = extract_signature(
1857                 f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {fakepath}\n{const2}",
1858                 filefactory=fakefilefactory,
1859             )
1860             assert origtext == const1
1861             assert textsig == const2
1862             assert htmlsig == sigconst.format(path=fakepath)
1863
1864         @pytest.mark.sig
1865         def test_signature_extraction_file_not_found(self, fakepath, const1):
1866             with pytest.raises(FileNotFoundError):
1867                 origtext, textsig, htmlsig = extract_signature(
1868                     f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{fakepath}\n{const1}"
1869                 )
1870
1871         @pytest.mark.imgproc
1872         def test_image_registry(self, const1):
1873             reg = ImageRegistry()
1874             cid = reg.register(const1)
1875             assert "@" in cid
1876             assert not cid.startswith("<")
1877             assert not cid.endswith(">")
1878             assert const1 in reg
1879
1880         @pytest.mark.imgproc
1881         def test_image_registry_domain(self, const1, const2):
1882             reg = ImageRegistry()
1883             cid = reg.register(const1, domain=const2)
1884             assert f"@{const2}" in cid
1885             assert not cid.startswith("<")
1886             assert not cid.endswith(">")
1887             assert const1 in reg
1888
1889         @pytest.mark.imgproc
1890         def test_image_registry_file_uri(self, const1):
1891             reg = ImageRegistry()
1892             reg.register("/some/path")
1893             for path in reg:
1894                 assert path.startswith("file://")
1895                 break
1896
1897         @pytest.mark.converter
1898         @pytest.mark.sig
1899         def test_converter_signature_handling(
1900             self, fakepath, fakefilefactory, monkeypatch
1901         ):
1902             mailparts = (
1903                 "This is the mail body\n",
1904                 f"{EMAIL_SIG_SEP}",
1905                 "This is a plain-text signature only",
1906             )
1907
1908             with (
1909                 fakefilefactory(
1910                     fakepath, content="".join(mailparts)
1911                 ) as draft_f,
1912                 monkeypatch.context() as m,
1913             ):
1914                 m.setattr(markdown.Markdown, "convert", lambda s, t: t)
1915                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
1916
1917             soup = bs4.BeautifulSoup(
1918                 fakefilefactory[fakepath.with_suffix(".html")].read(),
1919                 "html.parser",
1920             )
1921             body = soup.body.contents
1922
1923             assert mailparts[0] in body.pop(0)
1924
1925             sig = soup.select_one("#signature")
1926             assert sig == body.pop(0)
1927
1928             sep = sig.select_one("span.sig_separator")
1929             assert sep == sig.contents[0]
1930             assert f"\n{sep.text}\n" == EMAIL_SIG_SEP
1931
1932             assert mailparts[2] in sig.contents[1]
1933
1934         @pytest.mark.converter
1935         @pytest.mark.sig
1936         def test_converter_signature_handling_htmlsig(
1937             self, fakepath, fakepath2, fakefilefactory, monkeypatch
1938         ):
1939             mailparts = (
1940                 "This is the mail body",
1941                 f"{EMAIL_SIG_SEP}",
1942                 f"{HTML_SIG_MARKER}{fakepath2}\n",
1943                 "This is the plain-text version",
1944             )
1945             htmlsig = "HTML Signature from {path} but as a string"
1946             html = f'<div id="signature"><p>{htmlsig.format(path=fakepath2)}</p></div>'
1947
1948             sig_f = fakefilefactory(fakepath2, content=html)
1949
1950             def mdwn_fn(t):
1951                 return t.upper()
1952
1953             with (
1954                 fakefilefactory(
1955                     fakepath, content="".join(mailparts)
1956                 ) as draft_f,
1957                 monkeypatch.context() as m,
1958             ):
1959                 m.setattr(
1960                     markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
1961                 )
1962                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
1963
1964             soup = bs4.BeautifulSoup(
1965                 fakefilefactory[fakepath.with_suffix(".html")].read(),
1966                 "html.parser",
1967             )
1968             sig = soup.select_one("#signature")
1969             sig.span.extract()
1970
1971             assert HTML_SIG_MARKER not in sig.text
1972             assert htmlsig.format(path=fakepath2) == sig.text.strip()
1973
1974             plaintext = fakefilefactory[fakepath].read()
1975             assert plaintext.endswith(EMAIL_SIG_SEP + mailparts[-1])
1976
1977         @pytest.mark.converter
1978         @pytest.mark.sig
1979         def test_converter_signature_handling_htmlsig_with_image(
1980             self, fakepath, fakepath2, fakefilefactory, monkeypatch, test_png
1981         ):
1982             mailparts = (
1983                 "This is the mail body",
1984                 f"{EMAIL_SIG_SEP}",
1985                 f"{HTML_SIG_MARKER}{fakepath2}\n",
1986                 "This is the plain-text version",
1987             )
1988             htmlsig = (
1989                 "HTML Signature from {path} with image\n"
1990                 f'<img src="{test_png}">\n'
1991             )
1992             html = (
1993                 f'<div id="signature">{htmlsig.format(path=fakepath2)}</div>'
1994             )
1995
1996             sig_f = fakefilefactory(fakepath2, content=html)
1997
1998             def mdwn_fn(t):
1999                 return t.upper()
2000
2001             with (
2002                 fakefilefactory(
2003                     fakepath, content="".join(mailparts)
2004                 ) as draft_f,
2005                 monkeypatch.context() as m,
2006             ):
2007                 m.setattr(
2008                     markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
2009                 )
2010                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2011
2012             assert fakefilefactory.pop()[0].suffix == ".png"
2013
2014             soup = bs4.BeautifulSoup(
2015                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2016                 "html.parser",
2017             )
2018             assert soup.img.attrs["src"].startswith("cid:")
2019
2020         @pytest.mark.converter
2021         @pytest.mark.sig
2022         def test_converter_signature_handling_textsig_with_image(
2023             self, fakepath, fakefilefactory, test_png
2024         ):
2025             mailparts = (
2026                 "This is the mail body",
2027                 f"{EMAIL_SIG_SEP}",
2028                 "This is the plain-text version with image\n",
2029                 f"![Inline]({test_png})",
2030             )
2031             with (
2032                 fakefilefactory(
2033                     fakepath, content="".join(mailparts)
2034                 ) as draft_f,
2035             ):
2036                 tree = convert_markdown_to_html(
2037                     draft_f, filefactory=fakefilefactory
2038                 )
2039
2040             assert tree.subtype == "relative"
2041             assert tree.children[0].subtype == "alternative"
2042             assert tree.children[1].subtype == "png"
2043             written = fakefilefactory.pop()
2044             assert tree.children[1].path == written[0]
2045             assert written[1].read() == request.urlopen(test_png).read()
2046
2047         @pytest.mark.converter
2048         def test_converter_attribution_to_admonition(
2049             self, fakepath, fakefilefactory
2050         ):
2051             mailparts = (
2052                 "Regarding whatever",
2053                 "> blockquote line1",
2054                 "> blockquote line2",
2055                 "> ",
2056                 "> new para with **bold** text",
2057             )
2058             with fakefilefactory(
2059                 fakepath, content="\n".join(mailparts)
2060             ) as draft_f:
2061                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2062
2063             soup = bs4.BeautifulSoup(
2064                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2065                 "html.parser",
2066             )
2067             quote = soup.select_one("div.admonition.quote")
2068             assert quote
2069             assert (
2070                 soup.select_one("p.admonition-title").extract().text.strip()
2071                 == mailparts[0]
2072             )
2073
2074             p = quote.p.extract()
2075             assert p.text.strip() == "\n".join(p[2:] for p in mailparts[1:3])
2076
2077             p = quote.p.extract()
2078             assert p.contents[1].name == "strong"
2079
2080         @pytest.mark.converter
2081         def test_converter_attribution_to_admonition_with_blockquote(
2082             self, fakepath, fakefilefactory
2083         ):
2084             mailparts = (
2085                 "Regarding whatever",
2086                 "> blockquote line1",
2087                 "> blockquote line2",
2088                 "> ",
2089                 "> new para with **bold** text",
2090             )
2091             with fakefilefactory(
2092                 fakepath, content="\n".join(mailparts)
2093             ) as draft_f:
2094                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2095
2096             soup = bs4.BeautifulSoup(
2097                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2098                 "html.parser",
2099             )
2100             quote = soup.select_one("div.admonition.quote")
2101             assert quote.blockquote
2102
2103         @pytest.mark.converter
2104         def test_converter_attribution_to_admonition_multiple(
2105             self, fakepath, fakefilefactory
2106         ):
2107             mailparts = (
2108                 "Regarding whatever",
2109                 "> blockquote line1",
2110                 "> blockquote line2",
2111                 "",
2112                 "Normal text",
2113                 "",
2114                 "> continued emailquote",
2115                 "",
2116                 "Another email-quote",
2117                 "> something",
2118             )
2119             with fakefilefactory(
2120                 fakepath, content="\n".join(mailparts)
2121             ) as draft_f:
2122                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2123
2124             soup = bs4.BeautifulSoup(
2125                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2126                 "html.parser",
2127             )
2128             quote = soup.select_one("div.admonition.quote.continued").extract()
2129             assert quote
2130             assert (
2131                 quote.select_one("p.admonition-title").extract().text.strip()
2132                 == mailparts[0]
2133             )
2134
2135             p = quote.p.extract()
2136             assert p
2137
2138             quote = soup.select_one("div.admonition.quote.continued").extract()
2139             assert quote
2140             assert (
2141                 quote.select_one("p.admonition-title").extract().text.strip()
2142                 == mailparts[-2]
2143             )
2144
2145         @pytest.mark.converter
2146         def test_converter_format_flowed_with_nl2br(
2147             self, fakepath, fakefilefactory
2148         ):
2149             mailparts = (
2150                 "This is format=flowed text ",
2151                 "with spaces at the end ",
2152                 "and there ought be no newlines.",
2153                 "",
2154                 "[link](https://example.org) ",
2155                 "and text.",
2156                 "",
2157                 "[link text ",
2158                 "broken up](https://example.org).",
2159                 "",
2160                 "This is on a new line with a hard break  ",
2161                 "due to the double space",
2162             )
2163             with fakefilefactory(
2164                 fakepath, content="\n".join(mailparts)
2165             ) as draft_f:
2166                 convert_markdown_to_html(
2167                     draft_f, extensions=["nl2br"], filefactory=fakefilefactory
2168                 )
2169
2170             soup = bs4.BeautifulSoup(
2171                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2172                 "html.parser",
2173             )
2174             import ipdb
2175
2176             p = soup.p.extract().text
2177             assert "".join(mailparts[0:3]) == p
2178             p = ''.join(map(str, soup.p.extract().contents))
2179             assert p == '<a href="https://example.org">link</a> and text.'
2180             p = ''.join(map(str, soup.p.extract().contents))
2181             assert (
2182                 p == '<a href="https://example.org">link text broken up</a>.'
2183             )
2184
2185         @pytest.mark.fileio
2186         def test_file_class_contextmanager(self, const1, monkeypatch):
2187             state = dict(o=False, c=False)
2188
2189             def fn(t):
2190                 state[t] = True
2191
2192             with monkeypatch.context() as m:
2193                 m.setattr(File, "open", lambda s: fn("o"))
2194                 m.setattr(File, "close", lambda s: fn("c"))
2195                 with File() as f:
2196                     assert state["o"]
2197                     assert not state["c"]
2198             assert state["c"]
2199
2200         @pytest.mark.fileio
2201         def test_file_class_no_path(self, const1):
2202             with File(mode="w+") as f:
2203                 f.write(const1, cache=False)
2204                 assert f.read(cache=False) == const1
2205
2206         @pytest.mark.fileio
2207         def test_file_class_path(self, const1, tmp_path):
2208             with File(tmp_path / "file", mode="w+") as f:
2209                 f.write(const1, cache=False)
2210                 assert f.read(cache=False) == const1
2211
2212         @pytest.mark.fileio
2213         def test_file_class_path_no_exists(self, fakepath):
2214             with pytest.raises(FileNotFoundError):
2215                 File(fakepath, mode="r").open()
2216
2217         @pytest.mark.fileio
2218         def test_file_class_cache(self, tmp_path, const1, const2):
2219             path = tmp_path / "file"
2220             file = File(path, mode="w+")
2221             with file as f:
2222                 f.write(const1, cache=True)
2223             with open(path, mode="w") as f:
2224                 f.write(const2)
2225             with file as f:
2226                 assert f.read(cache=True) == const1
2227
2228         @pytest.mark.fileio
2229         def test_file_class_cache_init(self, const1):
2230             file = File(path=None, mode="r", content=const1)
2231             with file as f:
2232                 assert f.read() == const1
2233
2234         @pytest.mark.fileio
2235         def test_file_class_content_or_path(self, fakepath, const1):
2236             with pytest.raises(RuntimeError):
2237                 file = File(path=fakepath, content=const1)
2238
2239         @pytest.mark.fileio
2240         def test_file_class_content_needs_read(self, const1):
2241             with pytest.raises(RuntimeError):
2242                 file = File(mode="w", content=const1)
2243
2244         @pytest.mark.fileio
2245         def test_file_class_write_persists_close(self, const1):
2246             f = File(mode="w+")
2247             with f:
2248                 f.write(const1)
2249             with f:
2250                 assert f.read() == const1
2251
2252         @pytest.mark.fileio
2253         def test_file_class_write_resets_read_cache(self, const1, const2):
2254             with File(mode="w+", content=const1) as f:
2255                 assert f.read() == const1
2256                 f.write(const2)
2257                 assert f.read() == const2
2258
2259         @pytest.mark.fileio
2260         def test_file_factory(self):
2261             fact = FileFactory()
2262             f = fact()
2263             assert isinstance(f, File)
2264             assert len(fact) == 1
2265             assert f in fact
2266             assert f == fact[0]
2267
2268         @pytest.mark.fileio
2269         def test_fake_file_factory(self, fakepath, fakefilefactory):
2270             fact = FakeFileFactory()
2271             f = fakefilefactory(fakepath)
2272             assert f.path == fakepath
2273             assert f == fakefilefactory[fakepath]
2274
2275         @pytest.mark.fileio
2276         def test_fake_file_factory_path_persistence(
2277             self, fakepath, fakefilefactory
2278         ):
2279             f1 = fakefilefactory(fakepath)
2280             assert f1 == fakefilefactory(fakepath)
2281
2282 except ImportError:
2283     pass