.config/neomutt/buildmimetree.py

   1 #!/usr/bin/python3
   2 #
   3 # NeoMutt helper script to create multipart/* emails with Markdown → HTML
   4 # alternative conversion, and handling of inline images, using NeoMutt's
   5 # ability to manually craft MIME trees, but automating this process.
   6 #
   7 # Configuration:
   8 #   neomuttrc (needs to be a single line):
   9 #     set my_mdwn_extensions="extra,admonition,codehilite,sane_lists,smarty"
  10 #     macro compose B "\
  11 #       <enter-command> source '$my_confdir/buildmimetree.py \
  12 #       --tempdir $tempdir --extensions $my_mdwn_extensions \
  13 #       --css-file $my_confdir/htmlmail.css |'<enter>\
  14 #       <enter-command> source \$my_mdwn_postprocess_cmd_file<enter>\
  15 #     " "Convert message into a modern MIME tree with inline images"
  16 #
  17 #     (Yes, we need to call source twice, as mutt only starts to process output
  18 #     from a source command when the command exits, and since we need to react
  19 #     to the output, we need to be invoked again, using a $my_ variable to pass
  20 #     information)
  21 #
  22 # Requirements:
  23 #   - python3
  24 #   - python3-markdown
  25 #   - python3-beautifulsoup4
  26 # Optional:
  27 #   - pytest
  28 #   - Pynliner, provides --css-file and thus inline styling of HTML output
  29 #   - Pygments, then syntax highlighting for fenced code is enabled
  30 #
  31 # Running tests:
  32 #   pytest -x buildmimetree.py
  33 #
  34 # Latest version:
  35 #   https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
  36 #
  37 # Copyright © 2023–24 martin f. krafft <madduck@madduck.net>
  38 # Released under the GPL-2+ licence, just like NeoMutt itself.
  39 #
  40
  41 import sys
  42 import os.path
  43 import pathlib
  44 import markdown
  45 import tempfile
  46 import argparse
  47 import re
  48 import mimetypes
  49 import bs4
  50 import xml.etree.ElementTree as etree
  51 import io
  52 import enum
  53 import warnings
  54 from contextlib import contextmanager
  55 from collections import namedtuple, OrderedDict
  56 from markdown.extensions import Extension
  57 from markdown.blockprocessors import BlockProcessor
  58 from markdown.inlinepatterns import (
  59     SimpleTextInlineProcessor,
  60     ImageInlineProcessor,
  61     IMAGE_LINK_RE,
  62 )
  63 from email.utils import make_msgid
  64 from urllib import request
  65
  66
  67 def parse_cli_args(*args, **kwargs):
  68     parser = argparse.ArgumentParser(
  69         description=(
  70             "NeoMutt helper to turn text/markdown email parts "
  71             "into full-fledged MIME trees"
  72         )
  73     )
  74     parser.epilog = (
  75         "Copyright © 2023-24 martin f. krafft <madduck@madduck.net>.\n"
  76         "Released under the MIT licence"
  77     )
  78
  79     parser.add_argument(
  80         "--extensions",
  81         metavar="EXT[,EXT[,EXT]]",
  82         type=str,
  83         default="",
  84         help="Markdown extension to use (comma-separated list)",
  85     )
  86
  87     if _PYNLINER:
  88         parser.add_argument(
  89             "--css-file",
  90             metavar="FILE",
  91             type=pathlib.Path,
  92             default=os.devnull,
  93             help="CSS file to merge with the final HTML",
  94         )
  95     else:
  96         parser.set_defaults(css_file=None)
  97
  98     parser.add_argument(
  99         "--related-to-html-only",
 100         action="store_true",
 101         help="Make related content be sibling to HTML parts only",
 102     )
 103
 104     def positive_integer(value):
 105         try:
 106             if int(value) > 0:
 107                 return int(value)
 108
 109         except ValueError:
 110             pass
 111
 112         raise ValueError("Must be a positive integer")
 113
 114     parser.add_argument(
 115         "--max-number-other-attachments",
 116         metavar="INTEGER",
 117         type=positive_integer,
 118         default=20,
 119         help="Maximum number of other attachments to expect",
 120     )
 121
 122     parser.add_argument(
 123         "--only-build",
 124         "--just-build",
 125         action="store_true",
 126         help="Only build, don't send the message",
 127     )
 128
 129     parser.add_argument(
 130         "--domain",
 131         help="Domain to use in content IDs",
 132     )
 133
 134     parser.add_argument(
 135         "--tempdir",
 136         metavar="DIR",
 137         type=pathlib.Path,
 138         help="Specify temporary directory to use for attachments",
 139     )
 140
 141     parser.add_argument(
 142         "--debug-commands",
 143         action="store_true",
 144         help="Turn on debug logging of commands generated to stderr",
 145     )
 146
 147     parser.add_argument(
 148         "--debug-walk",
 149         action="store_true",
 150         help="Turn on debugging to stderr of the MIME tree walk",
 151     )
 152
 153     parser.add_argument(
 154         "--dump-html",
 155         metavar="FILE",
 156         type=pathlib.Path,
 157         help="Write the generated HTML to the file",
 158     )
 159
 160     subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
 161     massage_p = subp.add_parser(
 162         "massage", help="Massaging phase (internal use)"
 163     )
 164
 165     massage_p.add_argument(
 166         "--write-commands-to",
 167         "-o",
 168         metavar="FILE",
 169         dest="cmdpath",
 170         type=pathlib.Path,
 171         required=True,
 172         help="Temporary file path to write commands to",
 173     )
 174
 175     massage_p.add_argument(
 176         "MAILDRAFT",
 177         nargs="?",
 178         type=pathlib.Path,
 179         help="If provided, the script is invoked as editor on the mail draft",
 180     )
 181
 182     return parser.parse_args(*args, **kwargs)
 183
 184
 185 # [ FILE I/O HANDLING ] #######################################################
 186
 187
 188 class File:
 189     class Op(enum.Enum):
 190         R = enum.auto()
 191         W = enum.auto()
 192
 193     def __init__(self, path=None, mode="r", content=None, **kwargs):
 194         if path:
 195             if content:
 196                 raise RuntimeError("Cannot specify path and content for File")
 197
 198             self._path = (
 199                 path if isinstance(path, pathlib.Path) else pathlib.Path(path)
 200             )
 201         else:
 202             self._path = None
 203
 204         if content and not re.search(r"[r+]", mode):
 205             raise RuntimeError("Cannot specify content without read mode")
 206
 207         self._cache = {File.Op.R: [content] if content else [], File.Op.W: []}
 208         self._lastop = None
 209         self._mode = mode
 210         self._kwargs = kwargs
 211         self._file = None
 212
 213     def open(self):
 214         if self._path:
 215             self._file = open(self._path, self._mode, **self._kwargs)
 216         elif "b" in self._mode:
 217             self._file = io.BytesIO()
 218         else:
 219             self._file = io.StringIO()
 220
 221     def __enter__(self):
 222         self.open()
 223         return self
 224
 225     def __exit__(self, exc_type, exc_val, exc_tb):
 226         self.close()
 227
 228     def close(self):
 229         self._file.close()
 230         self._file = None
 231         self._cache[File.Op.R] = self._cache[File.Op.W]
 232         self._lastop = None
 233
 234     def _get_cache(self, op):
 235         return (b"" if "b" in self._mode else "").join(self._cache[op])
 236
 237     def _add_to_cache(self, op, s):
 238         self._cache[op].append(s)
 239
 240     def read(self, *, cache=True):
 241         if cache and self._cache[File.Op.R]:
 242             return self._get_cache(File.Op.R)
 243
 244         if self._lastop == File.Op.W:
 245             try:
 246                 self._file.seek(0)
 247             except io.UnsupportedOperation:
 248                 pass
 249
 250         self._lastop = File.Op.R
 251
 252         if cache:
 253             self._add_to_cache(File.Op.R, self._file.read())
 254             return self._get_cache(File.Op.R)
 255         else:
 256             return self._file.read()
 257
 258     def write(self, s, *, cache=True):
 259         if self._lastop == File.Op.R:
 260             try:
 261                 self._file.seek(0)
 262             except io.UnsupportedOperation:
 263                 pass
 264
 265         if cache:
 266             self._add_to_cache(File.Op.W, s)
 267
 268         self._cache[File.Op.R] = self._cache[File.Op.W]
 269
 270         written = self._file.write(s)
 271         self._file.flush()
 272         self._lastop = File.Op.W
 273         return written
 274
 275     path = property(lambda s: s._path)
 276
 277     def __repr__(self):
 278         return (
 279             f'<File path={self._path or "(buffered)"} open={bool(self._file)} '
 280             f"rcache={sum(len(c) for c in self._rcache) if self._rcache is not None else False} "
 281             f"wcache={sum(len(c) for c in self._wcache) if self._wcache is not None else False}>"
 282         )
 283
 284
 285 class FileFactory:
 286     def __init__(self):
 287         self._files = []
 288
 289     def __call__(self, path=None, mode="r", content=None, **kwargs):
 290         f = File(path, mode, content, **kwargs)
 291         self._files.append(f)
 292         return f
 293
 294     def __len__(self):
 295         return self._files.__len__()
 296
 297     def pop(self, idx=-1):
 298         return self._files.pop(idx)
 299
 300     def __getitem__(self, idx):
 301         return self._files.__getitem__(idx)
 302
 303     def __contains__(self, f):
 304         return self._files.__contains__(f)
 305
 306
 307 class FakeFileFactory(FileFactory):
 308     def __init__(self):
 309         super().__init__()
 310         self._paths2files = OrderedDict()
 311
 312     def __call__(self, path=None, mode="r", content=None, **kwargs):
 313         if path in self._paths2files:
 314             return self._paths2files[path]
 315
 316         f = super().__call__(None, mode, content, **kwargs)
 317         self._paths2files[path] = f
 318
 319         mypath = path
 320
 321         class FakeFile(File):
 322             path = mypath
 323
 324         # this is quality Python! We do this so that the fake file, which has
 325         # no path, fake-pretends to have a path for testing purposes.
 326
 327         f.__class__ = FakeFile
 328         return f
 329
 330     def __getitem__(self, path):
 331         return self._paths2files.__getitem__(path)
 332
 333     def get(self, path, default):
 334         return self._paths2files.get(path, default)
 335
 336     def pop(self, last=True):
 337         return self._paths2files.popitem(last)
 338
 339     def __repr__(self):
 340         return (
 341             f"<FakeFileFactory nfiles={len(self._files)} "
 342             f"paths={len(self._paths2files)}>"
 343         )
 344
 345
 346 # [ IMAGE HANDLING ] ##########################################################
 347
 348
 349 InlineImageInfo = namedtuple(
 350     "InlineImageInfo", ["cid", "desc"], defaults=[None]
 351 )
 352
 353
 354 class ImageRegistry:
 355     def __init__(self):
 356         self._images = OrderedDict()
 357
 358     def register(self, path, description=None, *, domain=None):
 359         # path = str(pathlib.Path(path).expanduser())
 360         path = os.path.expanduser(path)
 361         if path.startswith("/"):
 362             path = f"file://{path}"
 363         cid = make_msgid(domain=domain)[1:-1]
 364         self._images[path] = InlineImageInfo(cid, description)
 365         return cid
 366
 367     def __iter__(self):
 368         return self._images.__iter__()
 369
 370     def __getitem__(self, idx):
 371         return self._images.__getitem__(idx)
 372
 373     def __len__(self):
 374         return self._images.__len__()
 375
 376     def items(self):
 377         return self._images.items()
 378
 379     def __repr__(self):
 380         return f"<ImageRegistry(items={len(self._images)})>"
 381
 382     def __str__(self):
 383         return self._images.__str__()
 384
 385
 386 class InlineImageExtension(Extension):
 387     class RelatedImageInlineProcessor(ImageInlineProcessor):
 388         def __init__(self, re, md, registry):
 389             super().__init__(re, md)
 390             self._registry = registry
 391
 392         def handleMatch(self, m, data):
 393             el, start, end = super().handleMatch(m, data)
 394             if "src" in el.attrib:
 395                 src = el.attrib["src"]
 396                 if "://" not in src or src.startswith("file://"):
 397                     # We only inline local content
 398                     cid = self._registry.register(
 399                         el.attrib["src"],
 400                         el.attrib.get("title", el.attrib.get("alt")),
 401                     )
 402                     el.attrib["src"] = f"cid:{cid}"
 403             return el, start, end
 404
 405     def __init__(self, registry):
 406         super().__init__()
 407         self._image_registry = registry
 408
 409     INLINE_PATTERN_NAME = "image_link"
 410
 411     def extendMarkdown(self, md):
 412         md.registerExtension(self)
 413         inline_image_proc = self.RelatedImageInlineProcessor(
 414             IMAGE_LINK_RE, md, self._image_registry
 415         )
 416         md.inlinePatterns.register(
 417             inline_image_proc, InlineImageExtension.INLINE_PATTERN_NAME, 150
 418         )
 419
 420
 421 def markdown_with_inline_image_support(
 422     text,
 423     *,
 424     mdwn=None,
 425     image_registry=None,
 426     extensions=None,
 427     extension_configs=None,
 428 ):
 429     registry = (
 430         image_registry if image_registry is not None else ImageRegistry()
 431     )
 432     inline_image_handler = InlineImageExtension(registry=registry)
 433     extensions = extensions or []
 434     extensions.append(inline_image_handler)
 435     mdwn = markdown.Markdown(
 436         extensions=extensions, extension_configs=extension_configs
 437     )
 438
 439     htmltext = mdwn.convert(text)
 440
 441     def replace_image_with_cid(matchobj):
 442         for m in (matchobj.group(1), f"file://{matchobj.group(1)}"):
 443             if m in registry:
 444                 return f"(cid:{registry[m].cid}"
 445         return matchobj.group(0)
 446
 447     text = re.sub(r"\(([^)\s]+)", replace_image_with_cid, text)
 448     return text, htmltext, registry, mdwn
 449
 450
 451 # [ CSS STYLING ] #############################################################
 452
 453
 454 try:
 455     with warnings.catch_warnings():
 456         # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1081037
 457         warnings.filterwarnings("ignore", category=SyntaxWarning)
 458         import pynliner
 459
 460     _PYNLINER = True
 461
 462 except ImportError:
 463     _PYNLINER = False
 464
 465 try:
 466     from pygments.formatters import get_formatter_by_name
 467
 468     _CODEHILITE_CLASS = "codehilite"
 469
 470     _PYGMENTS_CSS = get_formatter_by_name(
 471         "html", style="default"
 472     ).get_style_defs(f".{_CODEHILITE_CLASS}")
 473
 474 except ImportError:
 475     _PYGMENTS_CSS = None
 476
 477
 478 def apply_styling(html, css):
 479     return (
 480         pynliner.Pynliner()
 481         .from_string(html)
 482         .with_cssString("\n".join(s for s in [_PYGMENTS_CSS, css] if s))
 483         .run()
 484     )
 485
 486
 487 # [ FORMAT=FLOWED HANDLING ] ##################################################
 488
 489
 490 class FormatFlowedNewlineExtension(Extension):
 491     FFNL_RE = r"(?!\S)(\s)\n"
 492
 493     def extendMarkdown(self, md):
 494         ffnl = SimpleTextInlineProcessor(self.FFNL_RE)
 495         md.inlinePatterns.register(ffnl, "ffnl", 125)
 496
 497
 498 # [ QUOTE HANDLING ] ##########################################################
 499
 500
 501 class QuoteToAdmonitionExtension(Extension):
 502     class BlockProcessor(BlockProcessor):
 503         RE = re.compile(r"(?:^|\n)>\s*(.*)")
 504
 505         def __init__(self, parser):
 506             super().__init__(parser)
 507             self._title = None
 508             self._disable = False
 509
 510         def test(self, parent, blocks):
 511             if self._disable:
 512                 return False
 513
 514             if markdown.util.nearing_recursion_limit():
 515                 return False
 516
 517             lines = blocks.splitlines()
 518             if len(lines) < 2:
 519                 if not self._title:
 520                     return False
 521
 522                 elif not self.RE.search(lines[0]):
 523                     return False
 524
 525                 return len(lines) > 0
 526
 527             elif not self.RE.search(lines[0]) and self.RE.search(lines[1]):
 528                 return True
 529
 530             elif self._title and self.RE.search(lines[1]):
 531                 return True
 532
 533             return False
 534
 535         def run(self, parent, blocks):
 536             quotelines = blocks.pop(0).splitlines()
 537
 538             cont = bool(self._title)
 539             if not self.RE.search(quotelines[0]):
 540                 self._title = quotelines.pop(0)
 541
 542             admonition = etree.SubElement(parent, "div")
 543             admonition.set(
 544                 "class", f"admonition quote{' continued' if cont else ''}"
 545             )
 546             self.parser.parseChunk(admonition, self._title)
 547
 548             admonition[0].set("class", "admonition-title")
 549             with self.disable():
 550                 self.parser.parseChunk(admonition, "\n".join(quotelines))
 551
 552         @contextmanager
 553         def disable(self):
 554             self._disable = True
 555             yield True
 556             self._disable = False
 557
 558         @classmethod
 559         def clean(klass, line):
 560             m = klass.RE.match(line)
 561             return m.group(1) if m else line
 562
 563     def extendMarkdown(self, md):
 564         md.registerExtension(self)
 565         email_quote_proc = self.BlockProcessor(md.parser)
 566         md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25)
 567
 568
 569 # [ PARTS GENERATION ] ########################################################
 570
 571
 572 class Part(
 573     namedtuple(
 574         "Part",
 575         ["type", "subtype", "path", "desc", "cid", "orig"],
 576         defaults=[None, None, False],
 577     )
 578 ):
 579     def __str__(self):
 580         ret = f"<{self.type}/{self.subtype}>"
 581         if self.cid:
 582             ret = f"{ret} cid:{self.cid}"
 583         if self.orig:
 584             ret = f"{ret} ORIGINAL"
 585         return ret
 586
 587
 588 class Multipart(
 589     namedtuple("Multipart", ["subtype", "children", "desc"], defaults=[None])
 590 ):
 591     def __str__(self):
 592         return f"<multipart/{self.subtype}> children={len(self.children)}"
 593
 594     def __hash__(self):
 595         return hash(str(self.subtype) + "".join(str(self.children)))
 596
 597
 598 def collect_inline_images(
 599     image_registry, *, tempdir=None, filefactory=FileFactory()
 600 ):
 601     relparts = []
 602     for path, info in image_registry.items():
 603         if path.startswith("cid:"):
 604             continue
 605
 606         data = request.urlopen(path)
 607
 608         mimetype = data.headers["Content-Type"]
 609         ext = mimetypes.guess_extension(mimetype)
 610         tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
 611         path = pathlib.Path(tempfilename[1])
 612
 613         with filefactory(path, "w+b") as out_f:
 614             out_f.write(data.read())
 615
 616         # filewriter_fn(path, data.read(), "w+b")
 617
 618         desc = (
 619             f'Inline image: "{info.desc}"'
 620             if info.desc
 621             else f"Inline image {str(len(relparts)+1)}"
 622         )
 623         relparts.append(
 624             Part(*mimetype.split("/"), path, cid=info.cid, desc=desc)
 625         )
 626
 627     return relparts
 628
 629
 630 EMAIL_SIG_SEP = "\n-- \n"
 631 HTML_SIG_MARKER = "=htmlsig "
 632
 633
 634 def make_html_doc(body, sig=None):
 635     ret = (
 636         "<!DOCTYPE html>\n"
 637         "<html>\n"
 638         "<head>\n"
 639         '<meta http-equiv="content-type" content="text/html; charset=UTF-8">\n'  # noqa: E501
 640         '<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'  # noqa: E501
 641         "</head>\n"
 642         "<body>\n"
 643         f"{body}\n"
 644     )
 645
 646     if sig:
 647         nl = "\n"
 648         ret = (
 649             f'{ret}<div id="signature"><span class="sig_separator">{EMAIL_SIG_SEP.strip(nl)}</span>\n'  # noqa: E501
 650             f"{sig}\n"
 651             "</div>"
 652         )
 653
 654     return f"{ret}\n  </body>\n</html>"
 655
 656
 657 def make_text_mail(text, sig=None):
 658     return EMAIL_SIG_SEP.join((text, sig)) if sig else text
 659
 660
 661 def extract_signature(text, *, filefactory=FileFactory()):
 662     parts = text.split(EMAIL_SIG_SEP, 1)
 663     if len(parts) == 1:
 664         return text, None, None
 665
 666     lines = parts[1].splitlines()
 667     if lines[0].startswith(HTML_SIG_MARKER):
 668         path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1])
 669         textsig = "\n".join(lines)
 670
 671         with filefactory(path.expanduser()) as sig_f:
 672             sig_input = sig_f.read()
 673
 674         soup = bs4.BeautifulSoup(sig_input, "html.parser")
 675
 676         style = str(soup.style.extract()) if soup.style else ""
 677         for sig_selector in (
 678             "#signature",
 679             "#signatur",
 680             "#emailsig",
 681             ".signature",
 682             ".signatur",
 683             ".emailsig",
 684             "body",
 685             "div",
 686         ):
 687             sig = soup.select_one(sig_selector)
 688             if sig:
 689                 break
 690
 691         if not sig:
 692             return parts[0], textsig, style + sig_input
 693
 694         if sig.attrs.get("id") == "signature":
 695             sig = "".join(str(c) for c in sig.children)
 696
 697         return parts[0], textsig, style + str(sig)
 698
 699     return parts[0], parts[1], None
 700
 701
 702 def convert_markdown_to_html(
 703     draft_f,
 704     *,
 705     related_to_html_only=False,
 706     css_f=None,
 707     htmldump_f=None,
 708     filefactory=FileFactory(),
 709     tempdir=None,
 710     extensions=None,
 711     extension_configs=None,
 712     domain=None,
 713 ):
 714     # TODO extension_configs need to be handled differently
 715     extension_configs = extension_configs or {}
 716     extension_configs.setdefault("pymdownx.highlight", {})[
 717         "css_class"
 718     ] = _CODEHILITE_CLASS
 719
 720     extensions = extensions or []
 721     extensions.append(FormatFlowedNewlineExtension())
 722     extensions.append(QuoteToAdmonitionExtension())
 723
 724     draft = draft_f.read()
 725     origtext, textsig, htmlsig = extract_signature(
 726         draft, filefactory=filefactory
 727     )
 728
 729     (
 730         origtext,
 731         htmltext,
 732         image_registry,
 733         mdwn,
 734     ) = markdown_with_inline_image_support(
 735         origtext, extensions=extensions, extension_configs=extension_configs
 736     )
 737
 738     if htmlsig:
 739         if not textsig:
 740             # TODO: decide what to do if there is no plain-text version
 741             raise NotImplementedError("HTML signature but no text alternative")
 742
 743         soup = bs4.BeautifulSoup(htmlsig, "html.parser")
 744         for img in soup.find_all("img"):
 745             uri = img.attrs["src"]
 746             desc = img.attrs.get("title", img.attrs.get("alt"))
 747             cid = image_registry.register(uri, desc, domain=domain)
 748             img.attrs["src"] = f"cid:{cid}"
 749
 750         htmlsig = str(soup)
 751
 752     elif textsig:
 753         (
 754             textsig,
 755             htmlsig,
 756             image_registry,
 757             mdwn,
 758         ) = markdown_with_inline_image_support(
 759             textsig,
 760             extensions=extensions,
 761             extension_configs=extension_configs,
 762             image_registry=image_registry,
 763             mdwn=mdwn,
 764         )
 765
 766     origtext = make_text_mail(origtext, textsig)
 767     draft_f.write(origtext)
 768     textpart = Part(
 769         "text", "plain", draft_f.path, "Plain-text version", orig=True
 770     )
 771
 772     htmltext = make_html_doc(htmltext, htmlsig)
 773     htmltext = apply_styling(htmltext, css_f.read() if css_f else None)
 774
 775     if draft_f.path:
 776         htmlpath = draft_f.path.with_suffix(".html")
 777     else:
 778         htmlpath = pathlib.Path(
 779             tempfile.mkstemp(suffix=".html", dir=tempdir)[1]
 780         )
 781     with filefactory(
 782         htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
 783     ) as out_f:
 784         out_f.write(htmltext)
 785     htmlpart = Part("text", "html", htmlpath, "HTML version")
 786
 787     if htmldump_f:
 788         htmldump_f.write(htmltext)
 789
 790     imgparts = collect_inline_images(
 791         image_registry, tempdir=tempdir, filefactory=filefactory
 792     )
 793
 794     if related_to_html_only:
 795         # If there are inline image part, they will be contained within a
 796         # multipart/related part along with the HTML part only
 797         if imgparts:
 798             # replace htmlpart with a multipart/related container of the HTML
 799             # parts and the images
 800             htmlpart = Multipart(
 801                 "relative", [htmlpart] + imgparts, "Group of related content"
 802             )
 803
 804         return Multipart(
 805             "alternative", [textpart, htmlpart], "Group of alternative content"
 806         )
 807
 808     else:
 809         # If there are inline image part, they will be siblings to the
 810         # multipart/alternative tree within a multipart/related part
 811         altpart = Multipart(
 812             "alternative", [textpart, htmlpart], "Group of alternative content"
 813         )
 814         if imgparts:
 815             return Multipart(
 816                 "relative", [altpart] + imgparts, "Group of related content"
 817             )
 818         else:
 819             return altpart
 820
 821
 822 class MIMETreeDFWalker:
 823     def __init__(self, *, visitor_fn=None, debug=False):
 824         self._visitor_fn = visitor_fn or self._echovisit
 825         self._debug = debug
 826
 827     def _echovisit(self, node, ancestry, debugprint):
 828         debugprint(f"node={node} ancestry={ancestry}")
 829
 830     def walk(self, root, *, visitor_fn=None):
 831         """
 832         Recursive function to implement a depth-dirst walk of the MIME-tree
 833         rooted at `root`.
 834         """
 835         if isinstance(root, list):
 836             if len(root) > 1:
 837                 root = Multipart("mixed", children=root)
 838             else:
 839                 root = root[0]
 840
 841         self._walk(
 842             root,
 843             ancestry=[],
 844             descendents=[],
 845             visitor_fn=visitor_fn or self._visitor_fn,
 846         )
 847
 848     def _walk(self, node, *, ancestry, descendents, visitor_fn):
 849         # Let's start by enumerating the parts at the current level. At the
 850         # root level, ancestry will be the empty list, and we expect a
 851         # multipart/* container at this level. Later, e.g. within a
 852         # mutlipart/alternative container, the subtree will just be the
 853         # alternative parts, while the top of the ancestry will be the
 854         # multipart/alternative container, which we will process after the
 855         # following loop.
 856
 857         lead = f"{'│ '*len(ancestry)}"
 858         if isinstance(node, Multipart):
 859             self.debugprint(
 860                 f"{lead}├{node} ancestry={[s.subtype for s in ancestry]}"
 861             )
 862
 863             # Depth-first, so push the current container onto the ancestry
 864             # stack, then descend …
 865             ancestry.append(node)
 866             self.debugprint(lead + "│ " * 2)
 867             for child in node.children:
 868                 self._walk(
 869                     child,
 870                     ancestry=ancestry,
 871                     descendents=descendents,
 872                     visitor_fn=visitor_fn,
 873                 )
 874             assert ancestry.pop() == node
 875             sibling_descendents = descendents
 876             descendents.extend(node.children)
 877
 878         else:
 879             self.debugprint(f"{lead}├{node}")
 880             sibling_descendents = descendents
 881
 882         if False and ancestry:
 883             self.debugprint(lead[:-1] + " │")
 884
 885         if visitor_fn:
 886             visitor_fn(
 887                 node, ancestry, sibling_descendents, debugprint=self.debugprint
 888             )
 889
 890     def debugprint(self, s, **kwargs):
 891         if self._debug:
 892             print(s, file=sys.stderr, **kwargs)
 893
 894
 895 # [ RUN MODES ] ###############################################################
 896
 897
 898 class MuttCommands:
 899     """
 900     Stupid class to interface writing out Mutt commands. This is quite a hack
 901     to deal with the fact that Mutt runs "push" commands in reverse order, so
 902     all of a sudden, things become very complicated when mixing with "real"
 903     commands.
 904
 905     Hence we keep two sets of commands, and one set of pushes. Commands are
 906     added to the first until a push is added, after which commands are added to
 907     the second set of commands.
 908
 909     On flush(), the first set is printed, followed by the pushes in reverse,
 910     and then the second set is printed. All 3 sets are then cleared.
 911     """
 912
 913     def __init__(self, out_f=sys.stdout, *, debug=False):
 914         self._cmd1, self._push, self._cmd2 = [], [], []
 915         self._out_f = out_f
 916         self._debug = debug
 917
 918     def cmd(self, s):
 919         self.debugprint(s)
 920         if self._push:
 921             self._cmd2.append(s)
 922         else:
 923             self._cmd1.append(s)
 924
 925     def push(self, s):
 926         s = s.replace('"', r"\"")
 927         s = f'push "{s}"'
 928         self.debugprint(s)
 929         self._push.insert(0, s)
 930
 931     def flush(self):
 932         print(
 933             "\n".join(self._cmd1 + self._push + self._cmd2), file=self._out_f
 934         )
 935         self._cmd1, self._push, self._cmd2 = [], [], []
 936
 937     def debugprint(self, s, **kwargs):
 938         if self._debug:
 939             print(s, file=sys.stderr, **kwargs)
 940
 941
 942 def do_setup(
 943     *,
 944     out_f=sys.stdout,
 945     temppath=None,
 946     tempdir=None,
 947     debug_commands=False,
 948 ):
 949     temppath = temppath or pathlib.Path(
 950         tempfile.mkstemp(prefix="muttmdwn-", dir=tempdir)[1]
 951     )
 952     cmds = MuttCommands(out_f, debug=debug_commands)
 953
 954     editor = f"{' '.join(sys.argv)} massage --write-commands-to {temppath}"
 955
 956     cmds.cmd('set my_editor="$editor"')
 957     cmds.cmd('set my_edit_headers="$edit_headers"')
 958     cmds.cmd(f'set editor="{editor}"')
 959     cmds.cmd("unset edit_headers")
 960     cmds.cmd(f"set my_mdwn_postprocess_cmd_file={temppath}")
 961     cmds.push("<first-entry><edit-file>")
 962     cmds.flush()
 963
 964
 965 def do_massage(
 966     draft_f,
 967     cmd_f,
 968     *,
 969     extensions=None,
 970     css_f=None,
 971     htmldump_f=None,
 972     converter=convert_markdown_to_html,
 973     related_to_html_only=True,
 974     only_build=False,
 975     max_other_attachments=20,
 976     tempdir=None,
 977     domain=None,
 978     debug_commands=False,
 979     debug_walk=False,
 980 ):
 981     # Here's the big picture: we're being invoked as the editor on the email
 982     # draft, and whatever commands we write to the file given as cmdpath will
 983     # be run by the second source command in the macro definition.
 984
 985     # Let's start by cleaning up what the setup did (see above), i.e. we
 986     # restore the $editor and $edit_headers variables, and also unset the
 987     # variable used to identify the command file we're currently writing
 988     # to.
 989     cmds = MuttCommands(cmd_f, debug=debug_commands)
 990
 991     extensions = extensions.split(",") if extensions else []
 992     tree = converter(
 993         draft_f,
 994         css_f=css_f,
 995         htmldump_f=htmldump_f,
 996         related_to_html_only=related_to_html_only,
 997         tempdir=tempdir,
 998         extensions=extensions,
 999         domain=domain,
1000     )
1001
1002     mimetree = MIMETreeDFWalker(debug=debug_walk)
1003
1004     state = dict(pos=1, tags={}, parts=1)
1005
1006     def visitor_fn(item, ancestry, descendents, *, debugprint=None):
1007         """
1008         Visitor function called for every node (part) of the MIME tree,
1009         depth-first, and responsible for telling NeoMutt how to assemble
1010         the tree.
1011         """
1012         KILL_LINE = r"\Ca\Ck"
1013
1014         if isinstance(item, Part):
1015             # We've hit a leaf-node, i.e. an alternative or a related part
1016             # with actual content.
1017
1018             # Let's add the part
1019             if item.orig:
1020                 # The original source already exists in the NeoMutt tree, but
1021                 # the underlying file may have been modified, so we need to
1022                 # update the encoding, but that's it:
1023                 cmds.push("<first-entry>")
1024                 cmds.push("<update-encoding>")
1025
1026                 # We really just need to be able to assume that at this point,
1027                 # NeoMutt is at position 1, and that we've processed only this
1028                 # part so far. Nevermind about actual attachments, we can
1029                 # safely ignore those as they stay at the end.
1030                 assert state["pos"] == 1
1031                 assert state["parts"] == 1
1032             else:
1033                 # … whereas all other parts need to be added, and they're all
1034                 # considered to be temporary and inline:
1035                 cmds.push(f"<attach-file>{item.path}<enter>")
1036                 cmds.push("<toggle-unlink><toggle-disposition>")
1037
1038                 # This added a part at the end of the list of parts, and that's
1039                 # just how many parts we've seen so far, so it's position in
1040                 # the NeoMutt compose list is the count of parts
1041                 state["parts"] += 1
1042                 state["pos"] = state["parts"]
1043
1044             # If the item (including the original) comes with additional
1045             # information, then we might just as well update the NeoMutt
1046             # tree now:
1047             if item.cid:
1048                 cmds.push(f"<edit-content-id>{KILL_LINE}{item.cid}<enter>")
1049
1050             # Now for the biggest hack in this script, which is to handle
1051             # attachments, such as PDFs, that aren't related or alternatives.
1052             # The problem is that when we add an inline image, it always gets
1053             # appended to the list, i.e. inserted *after* other attachments.
1054             # Since we don't know the number of attachments, we also cannot
1055             # infer the postition of the new attachment. Therefore, we bubble
1056             # it all the way to the top, only to then move it down again:
1057             if state["pos"] > 1:  # skip for the first part
1058                 for i in range(max_other_attachments):
1059                     # could use any number here, but has to be larger than the
1060                     # number of possible attachments. The performance
1061                     # difference of using a high number is negligible.
1062                     # Bubble up the new part
1063                     cmds.push("<move-up>")
1064
1065                 # As we push the part to the right position in the list (i.e.
1066                 # the last of the subset of attachments this script added), we
1067                 # must handle the situation that subtrees are skipped by
1068                 # NeoMutt. Hence, the actual number of positions to move down
1069                 # is decremented by the number of descendents so far
1070                 # encountered.
1071                 for i in range(1, state["pos"] - len(descendents)):
1072                     cmds.push("<move-down>")
1073
1074         elif isinstance(item, Multipart):
1075             # This node has children, but we already visited them (see
1076             # above). The tags dictionary of State should contain a list of
1077             # their positions in the NeoMutt compose window, so iterate those
1078             # and tag the parts there:
1079             n_tags = len(state["tags"][item])
1080             for tag in state["tags"][item]:
1081                 cmds.push(f"<jump>{tag}<enter><tag-entry>")
1082
1083             if item.subtype == "alternative":
1084                 cmds.push("<group-alternatives>")
1085             elif item.subtype in ("relative", "related"):
1086                 cmds.push("<group-related>")
1087             elif item.subtype == "multilingual":
1088                 cmds.push("<group-multilingual>")
1089             else:
1090                 raise NotImplementedError(
1091                     f"Handling of multipart/{item.subtype} is not implemented"
1092                 )
1093
1094             state["pos"] -= n_tags - 1
1095             state["parts"] += 1
1096
1097         else:
1098             # We should never get here
1099             raise RuntimeError(f"Type {type(item)} is unexpected: {item}")
1100
1101         # If the item has a description, we might just as well add it
1102         if item.desc:
1103             cmds.push(f"<edit-description>{KILL_LINE}{item.desc}<enter>")
1104
1105         if ancestry:
1106             # If there's an ancestry, record the current (assumed) position in
1107             # the NeoMutt compose window as needed-to-tag by our direct parent
1108             # (i.e. the last item of the ancestry)
1109             state["tags"].setdefault(ancestry[-1], []).append(state["pos"])
1110
1111             lead = "│ " * (len(ancestry) + 1) + "* "
1112             debugprint(
1113                 f"{lead}ancestry={[a.subtype for a in ancestry]}\n"
1114                 f"{lead}descendents={[d.subtype for d in descendents]}\n"
1115                 f"{lead}children_positions={state['tags'][ancestry[-1]]}\n"
1116                 f"{lead}pos={state['pos']}, parts={state['parts']}"
1117             )
1118
1119     # -----------------
1120     # End of visitor_fn
1121
1122     # Let's walk the tree and visit every node with our fancy visitor
1123     # function
1124     mimetree.walk(tree, visitor_fn=visitor_fn)
1125
1126     if not only_build:
1127         cmds.push("<send-message>")
1128
1129     # Finally, cleanup. Since we're responsible for removing the temporary
1130     # file, how's this for a little hack?
1131     try:
1132         filename = cmd_f.name
1133     except AttributeError:
1134         filename = "pytest_internal_file"
1135     cmds.cmd(f"source 'rm -f {filename}|'")
1136     cmds.cmd('set editor="$my_editor"')
1137     cmds.cmd('set edit_headers="$my_edit_headers"')
1138     cmds.cmd("unset my_editor")
1139     cmds.cmd("unset my_edit_headers")
1140     cmds.cmd("unset my_mdwn_postprocess_cmd_file")
1141     cmds.flush()
1142
1143
1144 # [ CLI ENTRY ] ###############################################################
1145
1146 if __name__ == "__main__":
1147     args = parse_cli_args()
1148
1149     if args.mode is None:
1150         do_setup(
1151             tempdir=args.tempdir,
1152             debug_commands=args.debug_commands,
1153         )
1154
1155     elif args.mode == "massage":
1156         with (
1157             File(args.MAILDRAFT, "r+") as draft_f,
1158             File(args.cmdpath, "w") as cmd_f,
1159             File(args.css_file, "r") as css_f,
1160             File(args.dump_html, "w") as htmldump_f,
1161         ):
1162             do_massage(
1163                 draft_f,
1164                 cmd_f,
1165                 extensions=args.extensions,
1166                 css_f=css_f,
1167                 htmldump_f=htmldump_f,
1168                 related_to_html_only=args.related_to_html_only,
1169                 max_other_attachments=args.max_number_other_attachments,
1170                 only_build=args.only_build,
1171                 tempdir=args.tempdir,
1172                 domain=args.domain,
1173                 debug_commands=args.debug_commands,
1174                 debug_walk=args.debug_walk,
1175             )
1176
1177
1178 # [ TESTS ] ###################################################################
1179
1180 try:
1181     import pytest
1182
1183     class Tests:
1184         @pytest.fixture
1185         def const1(self):
1186             return "Curvature Vest Usher Dividing+T#iceps Senior"
1187
1188         @pytest.fixture
1189         def const2(self):
1190             return "Habitant Celestial 2litzy Resurf/ce Headpiece Harmonics"
1191
1192         @pytest.fixture
1193         def fakepath(self):
1194             return pathlib.Path("/does/not/exist")
1195
1196         @pytest.fixture
1197         def fakepath2(self):
1198             return pathlib.Path("/does/not/exist/either")
1199
1200         # NOTE: tests using the capsys fixture must specify sys.stdout to the
1201         # functions they call, else old stdout is used and not captured
1202
1203         @pytest.mark.muttctrl
1204         def test_MuttCommands_cmd(self, const1, const2, capsys):
1205             "Assert order of commands"
1206             cmds = MuttCommands(out_f=sys.stdout)
1207             cmds.cmd(const1)
1208             cmds.cmd(const2)
1209             cmds.flush()
1210             captured = capsys.readouterr()
1211             assert captured.out == "\n".join((const1, const2, ""))
1212
1213         @pytest.mark.muttctrl
1214         def test_MuttCommands_push(self, const1, const2, capsys):
1215             "Assert reverse order of pushes"
1216             cmds = MuttCommands(out_f=sys.stdout)
1217             cmds.push(const1)
1218             cmds.push(const2)
1219             cmds.flush()
1220             captured = capsys.readouterr()
1221             assert (
1222                 captured.out
1223                 == ('"\npush "'.join(("", const2, const1, "")))[2:-6]
1224             )
1225
1226         @pytest.mark.muttctrl
1227         def test_MuttCommands_push_escape(self, const1, const2, capsys):
1228             cmds = MuttCommands(out_f=sys.stdout)
1229             cmds.push(f'"{const1}"')
1230             cmds.flush()
1231             captured = capsys.readouterr()
1232             assert f'"\\"{const1}\\""' in captured.out
1233
1234         @pytest.mark.muttctrl
1235         def test_MuttCommands_cmd_push_mixed(self, const1, const2, capsys):
1236             "Assert reverse order of pushes"
1237             cmds = MuttCommands(out_f=sys.stdout)
1238             lines = ["000", "001", "010", "011", "100", "101", "110", "111"]
1239             for i in range(2):
1240                 cmds.cmd(lines[4 * i + 0])
1241                 cmds.cmd(lines[4 * i + 1])
1242                 cmds.push(lines[4 * i + 2])
1243                 cmds.push(lines[4 * i + 3])
1244             cmds.flush()
1245
1246             captured = capsys.readouterr()
1247             lines_out = captured.out.splitlines()
1248             assert lines[0] in lines_out[0]
1249             assert lines[1] in lines_out[1]
1250             assert lines[7] in lines_out[2]
1251             assert lines[6] in lines_out[3]
1252             assert lines[3] in lines_out[4]
1253             assert lines[2] in lines_out[5]
1254             assert lines[4] in lines_out[6]
1255             assert lines[5] in lines_out[7]
1256
1257         @pytest.fixture
1258         def mime_tree_related_to_alternative(self):
1259             return Multipart(
1260                 "relative",
1261                 children=[
1262                     Multipart(
1263                         "alternative",
1264                         children=[
1265                             Part(
1266                                 "text",
1267                                 "plain",
1268                                 "part.txt",
1269                                 desc="Plain",
1270                                 orig=True,
1271                             ),
1272                             Part("text", "html", "part.html", desc="HTML"),
1273                         ],
1274                         desc="Alternative",
1275                     ),
1276                     Part(
1277                         "text", "png", "logo.png", cid="logo.png", desc="Logo"
1278                     ),
1279                 ],
1280                 desc="Related",
1281             )
1282
1283         @pytest.fixture
1284         def mime_tree_related_to_html(self):
1285             return Multipart(
1286                 "alternative",
1287                 children=[
1288                     Part(
1289                         "text",
1290                         "plain",
1291                         "part.txt",
1292                         desc="Plain",
1293                         orig=True,
1294                     ),
1295                     Multipart(
1296                         "relative",
1297                         children=[
1298                             Part("text", "html", "part.html", desc="HTML"),
1299                             Part(
1300                                 "text",
1301                                 "png",
1302                                 "logo.png",
1303                                 cid="logo.png",
1304                                 desc="Logo",
1305                             ),
1306                         ],
1307                         desc="Related",
1308                     ),
1309                 ],
1310                 desc="Alternative",
1311             )
1312
1313         @pytest.fixture
1314         def mime_tree_nested(self):
1315             return Multipart(
1316                 "relative",
1317                 children=[
1318                     Multipart(
1319                         "alternative",
1320                         children=[
1321                             Part(
1322                                 "text",
1323                                 "plain",
1324                                 "part.txt",
1325                                 desc="Plain",
1326                                 orig=True,
1327                             ),
1328                             Multipart(
1329                                 "alternative",
1330                                 children=[
1331                                     Part(
1332                                         "text",
1333                                         "plain",
1334                                         "part.txt",
1335                                         desc="Nested plain",
1336                                     ),
1337                                     Part(
1338                                         "text",
1339                                         "html",
1340                                         "part.html",
1341                                         desc="Nested HTML",
1342                                     ),
1343                                 ],
1344                                 desc="Nested alternative",
1345                             ),
1346                         ],
1347                         desc="Alternative",
1348                     ),
1349                     Part(
1350                         "text",
1351                         "png",
1352                         "logo.png",
1353                         cid="logo.png",
1354                         desc="Logo",
1355                     ),
1356                 ],
1357                 desc="Related",
1358             )
1359
1360         @pytest.mark.treewalk
1361         def test_MIMETreeDFWalker_depth_first_walk(
1362             self, mime_tree_related_to_alternative
1363         ):
1364             mimetree = MIMETreeDFWalker()
1365
1366             items = []
1367
1368             def visitor_fn(item, ancestry, descendents, debugprint):
1369                 items.append((item, len(ancestry), len(descendents)))
1370
1371             mimetree.walk(
1372                 mime_tree_related_to_alternative, visitor_fn=visitor_fn
1373             )
1374             assert len(items) == 5
1375             assert items[0][0].subtype == "plain"
1376             assert items[0][1] == 2
1377             assert items[0][2] == 0
1378             assert items[1][0].subtype == "html"
1379             assert items[1][1] == 2
1380             assert items[1][2] == 0
1381             assert items[2][0].subtype == "alternative"
1382             assert items[2][1] == 1
1383             assert items[2][2] == 2
1384             assert items[3][0].subtype == "png"
1385             assert items[3][1] == 1
1386             assert items[3][2] == 2
1387             assert items[4][0].subtype == "relative"
1388             assert items[4][1] == 0
1389             assert items[4][2] == 4
1390
1391         @pytest.mark.treewalk
1392         def test_MIMETreeDFWalker_list_to_mixed(self, const1):
1393             mimetree = MIMETreeDFWalker()
1394             items = []
1395
1396             def visitor_fn(item, ancestry, descendents, debugprint):
1397                 items.append(item)
1398
1399             p = Part("text", "plain", const1)
1400             mimetree.walk([p], visitor_fn=visitor_fn)
1401             assert items[-1].subtype == "plain"
1402             mimetree.walk([p, p], visitor_fn=visitor_fn)
1403             assert items[-1].subtype == "mixed"
1404
1405         @pytest.mark.treewalk
1406         def test_MIMETreeDFWalker_visitor_in_constructor(
1407             self, mime_tree_related_to_alternative
1408         ):
1409             items = []
1410
1411             def visitor_fn(item, ancestry, descendents, debugprint):
1412                 items.append(item)
1413
1414             mimetree = MIMETreeDFWalker(visitor_fn=visitor_fn)
1415             mimetree.walk(mime_tree_related_to_alternative)
1416             assert len(items) == 5
1417
1418         @pytest.fixture
1419         def string_io(self, const1, text=None):
1420             return StringIO(text or const1)
1421
1422         @pytest.mark.massage
1423         def test_do_massage_basic(self):
1424             def converter(draft_f, **kwargs):
1425                 return Part("text", "plain", draft_f.path, orig=True)
1426
1427             with File() as draft_f, File() as cmd_f:
1428                 do_massage(
1429                     draft_f=draft_f,
1430                     cmd_f=cmd_f,
1431                     converter=converter,
1432                 )
1433                 lines = cmd_f.read().splitlines()
1434
1435             assert "send-message" in lines.pop(0)
1436             assert "update-encoding" in lines.pop(0)
1437             assert "first-entry" in lines.pop(0)
1438             assert "source 'rm -f " in lines.pop(0)
1439             assert '="$my_editor"' in lines.pop(0)
1440             assert '="$my_edit_headers"' in lines.pop(0)
1441             assert "unset my_editor" == lines.pop(0)
1442             assert "unset my_edit_headers" == lines.pop(0)
1443             assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
1444
1445         @pytest.mark.massage
1446         def test_do_massage_fulltree(self, mime_tree_related_to_alternative):
1447             def converter(draft_f, **kwargs):
1448                 return mime_tree_related_to_alternative
1449
1450             max_attachments = 5
1451
1452             with File() as draft_f, File() as cmd_f:
1453                 do_massage(
1454                     draft_f=draft_f,
1455                     cmd_f=cmd_f,
1456                     max_other_attachments=max_attachments,
1457                     converter=converter,
1458                 )
1459                 lines = cmd_f.read().splitlines()[:-6]
1460
1461             assert "first-entry" in lines.pop()
1462             assert "update-encoding" in lines.pop()
1463             assert "Plain" in lines.pop()
1464             assert "part.html" in lines.pop()
1465             assert "toggle-unlink" in lines.pop()
1466             for i in range(max_attachments):
1467                 assert "move-up" in lines.pop()
1468             assert "move-down" in lines.pop()
1469             assert "HTML" in lines.pop()
1470             assert "jump>1" in lines.pop()
1471             assert "jump>2" in lines.pop()
1472             assert "group-alternatives" in lines.pop()
1473             assert "Alternative" in lines.pop()
1474             assert "logo.png" in lines.pop()
1475             assert "toggle-unlink" in lines.pop()
1476             assert "content-id" in lines.pop()
1477             for i in range(max_attachments):
1478                 assert "move-up" in lines.pop()
1479             assert "move-down" in lines.pop()
1480             assert "Logo" in lines.pop()
1481             assert "jump>1" in lines.pop()
1482             assert "jump>4" in lines.pop()
1483             assert "group-related" in lines.pop()
1484             assert "Related" in lines.pop()
1485             assert "send-message" in lines.pop()
1486             assert len(lines) == 0
1487
1488         @pytest.mark.massage
1489         def test_mime_tree_relative_within_alternative(
1490             self, mime_tree_related_to_html
1491         ):
1492             def converter(draft_f, **kwargs):
1493                 return mime_tree_related_to_html
1494
1495             with File() as draft_f, File() as cmd_f:
1496                 do_massage(
1497                     draft_f=draft_f,
1498                     cmd_f=cmd_f,
1499                     converter=converter,
1500                 )
1501                 lines = cmd_f.read().splitlines()[:-6]
1502
1503             assert "first-entry" in lines.pop()
1504             assert "update-encoding" in lines.pop()
1505             assert "Plain" in lines.pop()
1506             assert "part.html" in lines.pop()
1507             assert "toggle-unlink" in lines.pop()
1508             assert "move-up" in lines.pop()
1509             while True:
1510                 top = lines.pop()
1511                 if "move-up" not in top:
1512                     break
1513             assert "move-down" in top
1514             assert "HTML" in lines.pop()
1515             assert "logo.png" in lines.pop()
1516             assert "toggle-unlink" in lines.pop()
1517             assert "content-id" in lines.pop()
1518             assert "move-up" in lines.pop()
1519             while True:
1520                 top = lines.pop()
1521                 if "move-up" not in top:
1522                     break
1523             assert "move-down" in top
1524             assert "move-down" in lines.pop()
1525             assert "Logo" in lines.pop()
1526             assert "jump>2" in lines.pop()
1527             assert "jump>3" in lines.pop()
1528             assert "group-related" in lines.pop()
1529             assert "Related" in lines.pop()
1530             assert "jump>1" in lines.pop()
1531             assert "jump>2" in lines.pop()
1532             assert "group-alternative" in lines.pop()
1533             assert "Alternative" in lines.pop()
1534             assert "send-message" in lines.pop()
1535             assert len(lines) == 0
1536
1537         @pytest.mark.massage
1538         def test_mime_tree_nested_trees_does_not_break_positioning(
1539             self, mime_tree_nested
1540         ):
1541             def converter(draft_f, **kwargs):
1542                 return mime_tree_nested
1543
1544             with File() as draft_f, File() as cmd_f:
1545                 do_massage(
1546                     draft_f=draft_f,
1547                     cmd_f=cmd_f,
1548                     converter=converter,
1549                 )
1550                 lines = cmd_f.read().splitlines()
1551
1552             while "logo.png" not in lines.pop():
1553                 pass
1554             lines.pop()
1555             assert "content-id" in lines.pop()
1556             assert "move-up" in lines.pop()
1557             while True:
1558                 top = lines.pop()
1559                 if "move-up" not in top:
1560                     break
1561             assert "move-down" in top
1562             # Due to the nested trees, the number of descendents of the sibling
1563             # actually needs to be considered, not just the nieces. So to move
1564             # from position 1 to position 6, it only needs one <move-down>
1565             # because that jumps over the entire sibling tree. Thus what
1566             # follows next must not be another <move-down>
1567             assert "Logo" in lines.pop()
1568
1569         @pytest.mark.converter
1570         def test_converter_tree_basic(self, fakepath, const1, fakefilefactory):
1571             with fakefilefactory(fakepath, content=const1) as draft_f:
1572                 tree = convert_markdown_to_html(
1573                     draft_f, filefactory=fakefilefactory
1574                 )
1575
1576             assert tree.subtype == "alternative"
1577             assert len(tree.children) == 2
1578             assert tree.children[0].subtype == "plain"
1579             assert tree.children[0].path == draft_f.path
1580             assert tree.children[0].orig
1581             assert tree.children[1].subtype == "html"
1582             assert tree.children[1].path == fakepath.with_suffix(".html")
1583
1584         @pytest.mark.converter
1585         def test_converter_writes(
1586             self, fakepath, fakefilefactory, const1, monkeypatch
1587         ):
1588             with fakefilefactory(fakepath, content=const1) as draft_f:
1589                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
1590
1591             html = fakefilefactory.pop()
1592             assert fakepath.with_suffix(".html") == html[0]
1593             assert const1 in html[1].read()
1594             text = fakefilefactory.pop()
1595             assert fakepath == text[0]
1596             assert const1 == text[1].read()
1597
1598         @pytest.mark.imgproc
1599         def test_markdown_inline_image_processor(self):
1600             imgpath1 = "file:/path/to/image.png"
1601             imgpath2 = "file:///path/to/image.png?url=params"
1602             imgpath3 = "/path/to/image.png"
1603             text = f"""![inline local image]({imgpath1})
1604                        ![image inlined
1605                          with newline]({imgpath2})
1606                        ![image local path]({imgpath3})"""
1607             text, html, images, mdwn = markdown_with_inline_image_support(text)
1608
1609             # local paths have been normalised to URLs:
1610             imgpath3 = f"file://{imgpath3}"
1611
1612             assert 'src="cid:' in html
1613             assert "](cid:" in text
1614             assert len(images) == 3
1615             assert imgpath1 in images
1616             assert imgpath2 in images
1617             assert imgpath3 in images
1618             assert images[imgpath1].cid != images[imgpath2].cid
1619             assert images[imgpath1].cid != images[imgpath3].cid
1620             assert images[imgpath2].cid != images[imgpath3].cid
1621
1622         @pytest.mark.imgproc
1623         def test_markdown_inline_image_processor_title_to_desc(self, const1):
1624             imgpath = "file:///path/to/image.png"
1625             text = f'![inline local image]({imgpath} "{const1}")'
1626             text, html, images, mdwn = markdown_with_inline_image_support(text)
1627             assert images[imgpath].desc == const1
1628
1629         @pytest.mark.imgproc
1630         def test_markdown_inline_image_processor_alt_to_desc(self, const1):
1631             imgpath = "file:///path/to/image.png"
1632             text = f"![{const1}]({imgpath})"
1633             text, html, images, mdwn = markdown_with_inline_image_support(text)
1634             assert images[imgpath].desc == const1
1635
1636         @pytest.mark.imgproc
1637         def test_markdown_inline_image_processor_title_over_alt_desc(
1638             self, const1, const2
1639         ):
1640             imgpath = "file:///path/to/image.png"
1641             text = f'![{const1}]({imgpath} "{const2}")'
1642             text, html, images, mdwn = markdown_with_inline_image_support(text)
1643             assert images[imgpath].desc == const2
1644
1645         @pytest.mark.imgproc
1646         def test_markdown_inline_image_not_external(self):
1647             imgpath = "https://path/to/image.png"
1648             text = f"![inline image]({imgpath})"
1649             text, html, images, mdwn = markdown_with_inline_image_support(text)
1650
1651             assert 'src="cid:' not in html
1652             assert "](cid:" not in text
1653             assert len(images) == 0
1654
1655         @pytest.mark.imgproc
1656         def test_markdown_inline_image_local_file(self):
1657             imgpath = "/path/to/image.png"
1658             text = f"![inline image]({imgpath})"
1659             text, html, images, mdwn = markdown_with_inline_image_support(text)
1660
1661             for k, v in images.items():
1662                 assert k == f"file://{imgpath}"
1663                 break
1664
1665         @pytest.mark.imgproc
1666         def test_markdown_inline_image_expanduser(self):
1667             imgpath = pathlib.Path("~/image.png")
1668             text = f"![inline image]({imgpath})"
1669             text, html, images, mdwn = markdown_with_inline_image_support(text)
1670
1671             for k, v in images.items():
1672                 assert k == f"file://{imgpath.expanduser()}"
1673                 break
1674
1675         @pytest.fixture
1676         def test_png(self):
1677             return (
1678                 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE"
1679                 "AAAABCAAAAAA6fptVAAAACklEQVQI12P4DwABAQEAG7buVgAA"
1680             )
1681
1682         @pytest.mark.imgproc
1683         def test_markdown_inline_image_processor_base64(self, test_png):
1684             text = f"![1px white inlined]({test_png})"
1685             text, html, images, mdwn = markdown_with_inline_image_support(text)
1686
1687             assert 'src="cid:' in html
1688             assert "](cid:" in text
1689             assert len(images) == 1
1690             assert test_png in images
1691
1692         @pytest.mark.converter
1693         def test_converter_tree_inline_image_base64(
1694             self, test_png, fakefilefactory
1695         ):
1696             text = f"![inline base64 image]({test_png})"
1697             with fakefilefactory(content=text) as draft_f:
1698                 tree = convert_markdown_to_html(
1699                     draft_f,
1700                     filefactory=fakefilefactory,
1701                     related_to_html_only=False,
1702                 )
1703             assert tree.subtype == "relative"
1704             assert tree.children[0].subtype == "alternative"
1705             assert tree.children[1].subtype == "png"
1706             written = fakefilefactory.pop()
1707             assert tree.children[1].path == written[0]
1708             assert b"PNG" in written[1].read()
1709
1710         @pytest.mark.converter
1711         def test_converter_tree_inline_image_base64_related_to_html(
1712             self, test_png, fakefilefactory
1713         ):
1714             text = f"![inline base64 image]({test_png})"
1715             with fakefilefactory(content=text) as draft_f:
1716                 tree = convert_markdown_to_html(
1717                     draft_f,
1718                     filefactory=fakefilefactory,
1719                     related_to_html_only=True,
1720                 )
1721             assert tree.subtype == "alternative"
1722             assert tree.children[1].subtype == "relative"
1723             assert tree.children[1].children[1].subtype == "png"
1724             written = fakefilefactory.pop()
1725             assert tree.children[1].children[1].path == written[0]
1726             assert b"PNG" in written[1].read()
1727
1728         @pytest.mark.converter
1729         def test_converter_tree_inline_image_cid(
1730             self, const1, fakefilefactory
1731         ):
1732             text = f"![inline base64 image](cid:{const1})"
1733             with fakefilefactory(content=text) as draft_f:
1734                 tree = convert_markdown_to_html(
1735                     draft_f,
1736                     filefactory=fakefilefactory,
1737                     related_to_html_only=False,
1738                 )
1739             assert len(tree.children) == 2
1740             assert tree.children[0].cid != const1
1741             assert tree.children[0].type != "image"
1742             assert tree.children[1].cid != const1
1743             assert tree.children[1].type != "image"
1744
1745         @pytest.fixture
1746         def fakefilefactory(self):
1747             return FakeFileFactory()
1748
1749         @pytest.mark.imgcoll
1750         def test_inline_image_collection(
1751             self, test_png, const1, const2, fakefilefactory
1752         ):
1753             test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
1754             relparts = collect_inline_images(
1755                 test_images, filefactory=fakefilefactory
1756             )
1757
1758             written = fakefilefactory.pop()
1759             assert b"PNG" in written[1].read()
1760
1761             assert relparts[0].subtype == "png"
1762             assert relparts[0].path == written[0]
1763             assert relparts[0].cid == const1
1764             assert const2 in relparts[0].desc
1765
1766         if _PYNLINER:
1767
1768             @pytest.mark.styling
1769             def test_apply_stylesheet(self):
1770                 html = "<p>Hello, world!</p>"
1771                 css = "p { color:red }"
1772                 out = apply_styling(html, css)
1773                 assert 'p style="color' in out
1774
1775             @pytest.mark.styling
1776             def test_apply_no_stylesheet(self, const1):
1777                 out = apply_styling(const1, None)
1778
1779             @pytest.mark.massage
1780             @pytest.mark.styling
1781             def test_massage_styling_to_converter(self):
1782                 css = "p { color:red }"
1783                 css_applied = []
1784
1785                 def converter(draft_f, css_f, **kwargs):
1786                     css = css_f.read()
1787                     css_applied.append(css)
1788                     return Part("text", "plain", draft_f.path, orig=True)
1789
1790                 with (
1791                     File() as draft_f,
1792                     File(mode="w") as cmd_f,
1793                     File(content=css) as css_f,
1794                 ):
1795                     do_massage(
1796                         draft_f=draft_f,
1797                         cmd_f=cmd_f,
1798                         css_f=css_f,
1799                         converter=converter,
1800                     )
1801                 assert css_applied[0] == css
1802
1803             @pytest.mark.converter
1804             @pytest.mark.styling
1805             def test_converter_apply_styles(
1806                 self, const1, monkeypatch, fakepath, fakefilefactory
1807             ):
1808                 css = "p { color:red }"
1809                 with (
1810                     monkeypatch.context() as m,
1811                     fakefilefactory(fakepath, content=const1) as draft_f,
1812                     fakefilefactory(content=css) as css_f,
1813                 ):
1814                     m.setattr(
1815                         markdown.Markdown,
1816                         "convert",
1817                         lambda s, t: f"<p>{t}</p>",
1818                     )
1819                     convert_markdown_to_html(
1820                         draft_f, css_f=css_f, filefactory=fakefilefactory
1821                     )
1822                 assert re.search(
1823                     r"color:.*red",
1824                     fakefilefactory[fakepath.with_suffix(".html")].read(),
1825                 )
1826
1827         if _PYGMENTS_CSS:
1828
1829             @pytest.mark.styling
1830             def test_apply_stylesheet_pygments(self):
1831                 html = (
1832                     f'<div class="{_CODEHILITE_CLASS}">'
1833                     "<pre>def foo():\n    return</pre></div>"
1834                 )
1835                 out = apply_styling(html, _PYGMENTS_CSS)
1836                 assert f'{_CODEHILITE_CLASS}" style="' in out
1837
1838         @pytest.mark.sig
1839         def test_signature_extraction_no_signature(self, const1):
1840             assert (const1, None, None) == extract_signature(const1)
1841
1842         @pytest.mark.sig
1843         def test_signature_extraction_just_text(self, const1, const2):
1844             origtext, textsig, htmlsig = extract_signature(
1845                 f"{const1}{EMAIL_SIG_SEP}{const2}"
1846             )
1847             assert origtext == const1
1848             assert textsig == const2
1849             assert htmlsig is None
1850
1851         @pytest.mark.sig
1852         def test_signature_extraction_html(
1853             self, fakepath, fakefilefactory, const1, const2
1854         ):
1855             sigconst = "HTML signature from {path} but as a string"
1856             sig = f'<div id="signature">{sigconst.format(path=fakepath)}</div>'
1857
1858             sig_f = fakefilefactory(fakepath, content=sig)
1859
1860             origtext, textsig, htmlsig = extract_signature(
1861                 f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {fakepath}\n{const2}",
1862                 filefactory=fakefilefactory,
1863             )
1864             assert origtext == const1
1865             assert textsig == const2
1866             assert htmlsig == sigconst.format(path=fakepath)
1867
1868         @pytest.mark.sig
1869         def test_signature_extraction_file_not_found(self, fakepath, const1):
1870             with pytest.raises(FileNotFoundError):
1871                 origtext, textsig, htmlsig = extract_signature(
1872                     f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{fakepath}\n{const1}"
1873                 )
1874
1875         @pytest.mark.imgproc
1876         def test_image_registry(self, const1):
1877             reg = ImageRegistry()
1878             cid = reg.register(const1)
1879             assert "@" in cid
1880             assert not cid.startswith("<")
1881             assert not cid.endswith(">")
1882             assert const1 in reg
1883
1884         @pytest.mark.imgproc
1885         def test_image_registry_domain(self, const1, const2):
1886             reg = ImageRegistry()
1887             cid = reg.register(const1, domain=const2)
1888             assert f"@{const2}" in cid
1889             assert not cid.startswith("<")
1890             assert not cid.endswith(">")
1891             assert const1 in reg
1892
1893         @pytest.mark.imgproc
1894         def test_image_registry_file_uri(self, const1):
1895             reg = ImageRegistry()
1896             reg.register("/some/path")
1897             for path in reg:
1898                 assert path.startswith("file://")
1899                 break
1900
1901         @pytest.mark.converter
1902         @pytest.mark.sig
1903         def test_converter_signature_handling(
1904             self, fakepath, fakefilefactory, monkeypatch
1905         ):
1906             mailparts = (
1907                 "This is the mail body\n",
1908                 f"{EMAIL_SIG_SEP}",
1909                 "This is a plain-text signature only",
1910             )
1911
1912             with (
1913                 fakefilefactory(
1914                     fakepath, content="".join(mailparts)
1915                 ) as draft_f,
1916                 monkeypatch.context() as m,
1917             ):
1918                 m.setattr(markdown.Markdown, "convert", lambda s, t: t)
1919                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
1920
1921             soup = bs4.BeautifulSoup(
1922                 fakefilefactory[fakepath.with_suffix(".html")].read(),
1923                 "html.parser",
1924             )
1925             body = soup.body.contents
1926
1927             assert mailparts[0] in body.pop(0)
1928
1929             sig = soup.select_one("#signature")
1930             assert sig == body.pop(0)
1931
1932             sep = sig.select_one("span.sig_separator")
1933             assert sep == sig.contents[0]
1934             assert f"\n{sep.text}\n" == EMAIL_SIG_SEP
1935
1936             assert mailparts[2] in sig.contents[1]
1937
1938         @pytest.mark.converter
1939         @pytest.mark.sig
1940         def test_converter_signature_handling_htmlsig(
1941             self, fakepath, fakepath2, fakefilefactory, monkeypatch
1942         ):
1943             mailparts = (
1944                 "This is the mail body",
1945                 f"{EMAIL_SIG_SEP}",
1946                 f"{HTML_SIG_MARKER}{fakepath2}\n",
1947                 "This is the plain-text version",
1948             )
1949             htmlsig = "HTML Signature from {path} but as a string"
1950             html = f'<div id="signature"><p>{htmlsig.format(path=fakepath2)}</p></div>'
1951
1952             sig_f = fakefilefactory(fakepath2, content=html)
1953
1954             def mdwn_fn(t):
1955                 return t.upper()
1956
1957             with (
1958                 fakefilefactory(
1959                     fakepath, content="".join(mailparts)
1960                 ) as draft_f,
1961                 monkeypatch.context() as m,
1962             ):
1963                 m.setattr(
1964                     markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
1965                 )
1966                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
1967
1968             soup = bs4.BeautifulSoup(
1969                 fakefilefactory[fakepath.with_suffix(".html")].read(),
1970                 "html.parser",
1971             )
1972             sig = soup.select_one("#signature")
1973             sig.span.extract()
1974
1975             assert HTML_SIG_MARKER not in sig.text
1976             assert htmlsig.format(path=fakepath2) == sig.text.strip()
1977
1978             plaintext = fakefilefactory[fakepath].read()
1979             assert plaintext.endswith(EMAIL_SIG_SEP + mailparts[-1])
1980
1981         @pytest.mark.converter
1982         @pytest.mark.sig
1983         def test_converter_signature_handling_htmlsig_with_image(
1984             self, fakepath, fakepath2, fakefilefactory, monkeypatch, test_png
1985         ):
1986             mailparts = (
1987                 "This is the mail body",
1988                 f"{EMAIL_SIG_SEP}",
1989                 f"{HTML_SIG_MARKER}{fakepath2}\n",
1990                 "This is the plain-text version",
1991             )
1992             htmlsig = (
1993                 "HTML Signature from {path} with image\n"
1994                 f'<img src="{test_png}">\n'
1995             )
1996             html = (
1997                 f'<div id="signature">{htmlsig.format(path=fakepath2)}</div>'
1998             )
1999
2000             sig_f = fakefilefactory(fakepath2, content=html)
2001
2002             def mdwn_fn(t):
2003                 return t.upper()
2004
2005             with (
2006                 fakefilefactory(
2007                     fakepath, content="".join(mailparts)
2008                 ) as draft_f,
2009                 monkeypatch.context() as m,
2010             ):
2011                 m.setattr(
2012                     markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
2013                 )
2014                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2015
2016             assert fakefilefactory.pop()[0].suffix == ".png"
2017
2018             soup = bs4.BeautifulSoup(
2019                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2020                 "html.parser",
2021             )
2022             assert soup.img.attrs["src"].startswith("cid:")
2023
2024         @pytest.mark.converter
2025         @pytest.mark.sig
2026         def test_converter_signature_handling_textsig_with_image(
2027             self, fakepath, fakefilefactory, test_png
2028         ):
2029             mailparts = (
2030                 "This is the mail body",
2031                 f"{EMAIL_SIG_SEP}",
2032                 "This is the plain-text version with image\n",
2033                 f"![Inline]({test_png})",
2034             )
2035             with (
2036                 fakefilefactory(
2037                     fakepath, content="".join(mailparts)
2038                 ) as draft_f,
2039             ):
2040                 tree = convert_markdown_to_html(
2041                     draft_f, filefactory=fakefilefactory
2042                 )
2043
2044             assert tree.subtype == "relative"
2045             assert tree.children[0].subtype == "alternative"
2046             assert tree.children[1].subtype == "png"
2047             written = fakefilefactory.pop()
2048             assert tree.children[1].path == written[0]
2049             assert written[1].read() == request.urlopen(test_png).read()
2050
2051         @pytest.mark.converter
2052         def test_converter_attribution_to_admonition(
2053             self, fakepath, fakefilefactory
2054         ):
2055             mailparts = (
2056                 "Regarding whatever",
2057                 "> blockquote line1",
2058                 "> blockquote line2",
2059                 "> ",
2060                 "> new para with **bold** text",
2061             )
2062             with fakefilefactory(
2063                 fakepath, content="\n".join(mailparts)
2064             ) as draft_f:
2065                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2066
2067             soup = bs4.BeautifulSoup(
2068                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2069                 "html.parser",
2070             )
2071             quote = soup.select_one("div.admonition.quote")
2072             assert quote
2073             assert (
2074                 soup.select_one("p.admonition-title").extract().text.strip()
2075                 == mailparts[0]
2076             )
2077
2078             p = quote.p.extract()
2079             assert p.text.strip() == "\n".join(p[2:] for p in mailparts[1:3])
2080
2081             p = quote.p.extract()
2082             assert p.contents[1].name == "strong"
2083
2084         @pytest.mark.converter
2085         def test_converter_attribution_to_admonition_with_blockquote(
2086             self, fakepath, fakefilefactory
2087         ):
2088             mailparts = (
2089                 "Regarding whatever",
2090                 "> blockquote line1",
2091                 "> blockquote line2",
2092                 "> ",
2093                 "> new para with **bold** text",
2094             )
2095             with fakefilefactory(
2096                 fakepath, content="\n".join(mailparts)
2097             ) as draft_f:
2098                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2099
2100             soup = bs4.BeautifulSoup(
2101                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2102                 "html.parser",
2103             )
2104             quote = soup.select_one("div.admonition.quote")
2105             assert quote.blockquote
2106
2107         @pytest.mark.converter
2108         def test_converter_attribution_to_admonition_multiple(
2109             self, fakepath, fakefilefactory
2110         ):
2111             mailparts = (
2112                 "Regarding whatever",
2113                 "> blockquote line1",
2114                 "> blockquote line2",
2115                 "",
2116                 "Normal text",
2117                 "",
2118                 "> continued emailquote",
2119                 "",
2120                 "Another email-quote",
2121                 "> something",
2122             )
2123             with fakefilefactory(
2124                 fakepath, content="\n".join(mailparts)
2125             ) as draft_f:
2126                 convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
2127
2128             soup = bs4.BeautifulSoup(
2129                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2130                 "html.parser",
2131             )
2132             quote = soup.select_one("div.admonition.quote.continued").extract()
2133             assert quote
2134             assert (
2135                 quote.select_one("p.admonition-title").extract().text.strip()
2136                 == mailparts[0]
2137             )
2138
2139             p = quote.p.extract()
2140             assert p
2141
2142             quote = soup.select_one("div.admonition.quote.continued").extract()
2143             assert quote
2144             assert (
2145                 quote.select_one("p.admonition-title").extract().text.strip()
2146                 == mailparts[-2]
2147             )
2148
2149         @pytest.mark.converter
2150         def test_converter_format_flowed_with_nl2br(
2151             self, fakepath, fakefilefactory
2152         ):
2153             mailparts = (
2154                 "This is format=flowed text ",
2155                 "with spaces at the end ",
2156                 "and there ought be no newlines.",
2157                 "",
2158                 "[link](https://example.org) ",
2159                 "and text.",
2160                 "",
2161                 "[link text ",
2162                 "broken up](https://example.org).",
2163                 "",
2164                 "This is on a new line with a hard break  ",
2165                 "due to the double space",
2166             )
2167             with fakefilefactory(
2168                 fakepath, content="\n".join(mailparts)
2169             ) as draft_f:
2170                 convert_markdown_to_html(
2171                     draft_f, extensions=["nl2br"], filefactory=fakefilefactory
2172                 )
2173
2174             soup = bs4.BeautifulSoup(
2175                 fakefilefactory[fakepath.with_suffix(".html")].read(),
2176                 "html.parser",
2177             )
2178             import ipdb
2179
2180             p = soup.p.extract().text
2181             assert "".join(mailparts[0:3]) == p
2182             p = ''.join(map(str, soup.p.extract().contents))
2183             assert p == '<a href="https://example.org">link</a> and text.'
2184             p = ''.join(map(str, soup.p.extract().contents))
2185             assert (
2186                 p == '<a href="https://example.org">link text broken up</a>.'
2187             )
2188
2189         @pytest.mark.fileio
2190         def test_file_class_contextmanager(self, const1, monkeypatch):
2191             state = dict(o=False, c=False)
2192
2193             def fn(t):
2194                 state[t] = True
2195
2196             with monkeypatch.context() as m:
2197                 m.setattr(File, "open", lambda s: fn("o"))
2198                 m.setattr(File, "close", lambda s: fn("c"))
2199                 with File() as f:
2200                     assert state["o"]
2201                     assert not state["c"]
2202             assert state["c"]
2203
2204         @pytest.mark.fileio
2205         def test_file_class_no_path(self, const1):
2206             with File(mode="w+") as f:
2207                 f.write(const1, cache=False)
2208                 assert f.read(cache=False) == const1
2209
2210         @pytest.mark.fileio
2211         def test_file_class_path(self, const1, tmp_path):
2212             with File(tmp_path / "file", mode="w+") as f:
2213                 f.write(const1, cache=False)
2214                 assert f.read(cache=False) == const1
2215
2216         @pytest.mark.fileio
2217         def test_file_class_path_no_exists(self, fakepath):
2218             with pytest.raises(FileNotFoundError):
2219                 File(fakepath, mode="r").open()
2220
2221         @pytest.mark.fileio
2222         def test_file_class_cache(self, tmp_path, const1, const2):
2223             path = tmp_path / "file"
2224             file = File(path, mode="w+")
2225             with file as f:
2226                 f.write(const1, cache=True)
2227             with open(path, mode="w") as f:
2228                 f.write(const2)
2229             with file as f:
2230                 assert f.read(cache=True) == const1
2231
2232         @pytest.mark.fileio
2233         def test_file_class_cache_init(self, const1):
2234             file = File(path=None, mode="r", content=const1)
2235             with file as f:
2236                 assert f.read() == const1
2237
2238         @pytest.mark.fileio
2239         def test_file_class_content_or_path(self, fakepath, const1):
2240             with pytest.raises(RuntimeError):
2241                 file = File(path=fakepath, content=const1)
2242
2243         @pytest.mark.fileio
2244         def test_file_class_content_needs_read(self, const1):
2245             with pytest.raises(RuntimeError):
2246                 file = File(mode="w", content=const1)
2247
2248         @pytest.mark.fileio
2249         def test_file_class_write_persists_close(self, const1):
2250             f = File(mode="w+")
2251             with f:
2252                 f.write(const1)
2253             with f:
2254                 assert f.read() == const1
2255
2256         @pytest.mark.fileio
2257         def test_file_class_write_resets_read_cache(self, const1, const2):
2258             with File(mode="w+", content=const1) as f:
2259                 assert f.read() == const1
2260                 f.write(const2)
2261                 assert f.read() == const2
2262
2263         @pytest.mark.fileio
2264         def test_file_factory(self):
2265             fact = FileFactory()
2266             f = fact()
2267             assert isinstance(f, File)
2268             assert len(fact) == 1
2269             assert f in fact
2270             assert f == fact[0]
2271
2272         @pytest.mark.fileio
2273         def test_fake_file_factory(self, fakepath, fakefilefactory):
2274             fact = FakeFileFactory()
2275             f = fakefilefactory(fakepath)
2276             assert f.path == fakepath
2277             assert f == fakefilefactory[fakepath]
2278
2279         @pytest.mark.fileio
2280         def test_fake_file_factory_path_persistence(
2281             self, fakepath, fakefilefactory
2282         ):
2283             f1 = fakefilefactory(fakepath)
2284             assert f1 == fakefilefactory(fakepath)
2285
2286 except ImportError:
2287     pass