# - Pynliner, provides --css-file and thus inline styling of HTML output
# - Pygments, then syntax highlighting for fenced code is enabled
#
+# Running tests:
+# pytest -x buildmimetree.py
+#
# Latest version:
# https://git.madduck.net/etc/neomutt.git/blob_plain/HEAD:/.config/neomutt/buildmimetree.py
#
#
import sys
-import os
import os.path
import pathlib
import markdown
import re
import mimetypes
import bs4
+import xml.etree.ElementTree as etree
+import io
+import enum
+from contextlib import contextmanager
from collections import namedtuple, OrderedDict
from markdown.extensions import Extension
-from markdown.inlinepatterns import ImageInlineProcessor, IMAGE_LINK_RE
+from markdown.blockprocessors import BlockProcessor
+from markdown.inlinepatterns import (
+ SimpleTextInlineProcessor,
+ ImageInlineProcessor,
+ IMAGE_LINK_RE,
+)
from email.utils import make_msgid
from urllib import request
help="Turn on debugging to stderr of the MIME tree walk",
)
+ parser.add_argument(
+ "--dump-html",
+ metavar="FILE",
+ type=pathlib.Path,
+ help="Write the generated HTML to the file",
+ )
+
subp = parser.add_subparsers(help="Sub-command parsers", dest="mode")
massage_p = subp.add_parser(
"massage", help="Massaging phase (internal use)"
return parser.parse_args(*args, **kwargs)
+# [ FILE I/O HANDLING ] #######################################################
+
+
+class File:
+ class Op(enum.Enum):
+ R = enum.auto()
+ W = enum.auto()
+
+ def __init__(self, path=None, mode="r", content=None, **kwargs):
+ if path:
+ if content:
+ raise RuntimeError("Cannot specify path and content for File")
+
+ self._path = (
+ path if isinstance(path, pathlib.Path) else pathlib.Path(path)
+ )
+ else:
+ self._path = None
+
+ if content and not re.search(r"[r+]", mode):
+ raise RuntimeError("Cannot specify content without read mode")
+
+ self._cache = {File.Op.R: [content] if content else [], File.Op.W: []}
+ self._lastop = None
+ self._mode = mode
+ self._kwargs = kwargs
+ self._file = None
+
+ def open(self):
+ if self._path:
+ self._file = open(self._path, self._mode, **self._kwargs)
+ elif "b" in self._mode:
+ self._file = io.BytesIO()
+ else:
+ self._file = io.StringIO()
+
+ def __enter__(self):
+ self.open()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ def close(self):
+ self._file.close()
+ self._file = None
+ self._cache[File.Op.R] = self._cache[File.Op.W]
+ self._lastop = None
+
+ def _get_cache(self, op):
+ return (b"" if "b" in self._mode else "").join(self._cache[op])
+
+ def _add_to_cache(self, op, s):
+ self._cache[op].append(s)
+
+ def read(self, *, cache=True):
+ if cache and self._cache[File.Op.R]:
+ return self._get_cache(File.Op.R)
+
+ if self._lastop == File.Op.W:
+ try:
+ self._file.seek(0)
+ except io.UnsupportedOperation:
+ pass
+
+ self._lastop = File.Op.R
+
+ if cache:
+ self._add_to_cache(File.Op.R, self._file.read())
+ return self._get_cache(File.Op.R)
+ else:
+ return self._file.read()
+
+ def write(self, s, *, cache=True):
+ if self._lastop == File.Op.R:
+ try:
+ self._file.seek(0)
+ except io.UnsupportedOperation:
+ pass
+
+ if cache:
+ self._add_to_cache(File.Op.W, s)
+
+ self._cache[File.Op.R] = self._cache[File.Op.W]
+
+ written = self._file.write(s)
+ self._file.flush()
+ self._lastop = File.Op.W
+ return written
+
+ path = property(lambda s: s._path)
+
+ def __repr__(self):
+ return (
+ f'<File path={self._path or "(buffered)"} open={bool(self._file)} '
+ f"rcache={sum(len(c) for c in self._rcache) if self._rcache is not None else False} "
+ f"wcache={sum(len(c) for c in self._wcache) if self._wcache is not None else False}>"
+ )
+
+
+class FileFactory:
+ def __init__(self):
+ self._files = []
+
+ def __call__(self, path=None, mode="r", content=None, **kwargs):
+ f = File(path, mode, content, **kwargs)
+ self._files.append(f)
+ return f
+
+ def __len__(self):
+ return self._files.__len__()
+
+ def pop(self, idx=-1):
+ return self._files.pop(idx)
+
+ def __getitem__(self, idx):
+ return self._files.__getitem__(idx)
+
+ def __contains__(self, f):
+ return self._files.__contains__(f)
+
+
+class FakeFileFactory(FileFactory):
+ def __init__(self):
+ super().__init__()
+ self._paths2files = OrderedDict()
+
+ def __call__(self, path=None, mode="r", content=None, **kwargs):
+ if path in self._paths2files:
+ return self._paths2files[path]
+
+ f = super().__call__(None, mode, content, **kwargs)
+ self._paths2files[path] = f
+
+ mypath = path
+
+ class FakeFile(File):
+ path = mypath
+
+ # this is quality Python! We do this so that the fake file, which has
+ # no path, fake-pretends to have a path for testing purposes.
+
+ f.__class__ = FakeFile
+ return f
+
+ def __getitem__(self, path):
+ return self._paths2files.__getitem__(path)
+
+ def get(self, path, default):
+ return self._paths2files.get(path, default)
+
+ def pop(self, last=True):
+ return self._paths2files.popitem(last)
+
+ def __repr__(self):
+ return (
+ f"<FakeFileFactory nfiles={len(self._files)} "
+ f"paths={len(self._paths2files)}>"
+ )
+
+
# [ IMAGE HANDLING ] ##########################################################
self._images = OrderedDict()
def register(self, path, description=None):
+ # path = str(pathlib.Path(path).expanduser())
path = os.path.expanduser(path)
if path.startswith("/"):
path = f"file://{path}"
)
+# [ FORMAT=FLOWED HANDLING ] ##################################################
+
+
+class FormatFlowedNewlineExtension(Extension):
+ FFNL_RE = r"(?!\S)(\s)\n"
+
+ def extendMarkdown(self, md):
+ ffnl = SimpleTextInlineProcessor(self.FFNL_RE)
+ md.inlinePatterns.register(ffnl, "ffnl", 125)
+
+
+# [ QUOTE HANDLING ] ##########################################################
+
+
+class QuoteToAdmonitionExtension(Extension):
+ class BlockProcessor(BlockProcessor):
+ RE = re.compile(r"(?:^|\n)>\s*(.*)")
+
+ def __init__(self, parser):
+ super().__init__(parser)
+ self._title = None
+ self._disable = False
+
+ def test(self, parent, blocks):
+ if self._disable:
+ return False
+
+ if markdown.util.nearing_recursion_limit():
+ return False
+
+ lines = blocks.splitlines()
+ if len(lines) < 2:
+ if not self._title:
+ return False
+
+ elif not self.RE.search(lines[0]):
+ return False
+
+ return len(lines) > 0
+
+ elif not self.RE.search(lines[0]) and self.RE.search(lines[1]):
+ return True
+
+ elif self._title and self.RE.search(lines[1]):
+ return True
+
+ return False
+
+ def run(self, parent, blocks):
+ quotelines = blocks.pop(0).splitlines()
+
+ cont = bool(self._title)
+ if not self.RE.search(quotelines[0]):
+ self._title = quotelines.pop(0)
+
+ admonition = etree.SubElement(parent, "div")
+ admonition.set(
+ "class", f"admonition quote{' continued' if cont else ''}"
+ )
+ self.parser.parseChunk(admonition, self._title)
+
+ admonition[0].set("class", "admonition-title")
+ with self.disable():
+ self.parser.parseChunk(admonition, "\n".join(quotelines))
+
+ @contextmanager
+ def disable(self):
+ self._disable = True
+ yield True
+ self._disable = False
+
+ @classmethod
+ def clean(klass, line):
+ m = klass.RE.match(line)
+ return m.group(1) if m else line
+
+ def extendMarkdown(self, md):
+ md.registerExtension(self)
+ email_quote_proc = self.BlockProcessor(md.parser)
+ md.parser.blockprocessors.register(email_quote_proc, "emailquote", 25)
+
+
# [ PARTS GENERATION ] ########################################################
return hash(str(self.subtype) + "".join(str(self.children)))
-def filereader_fn(path, mode="r", **kwargs):
- with open(path, mode, **kwargs) as in_f:
- return in_f.read()
-
-
-def filewriter_fn(path, content, mode="w", **kwargs):
- with open(path, mode, **kwargs) as out_f:
- out_f.write(content)
-
-
def collect_inline_images(
- image_registry, *, tempdir=None, filewriter_fn=filewriter_fn
+ image_registry, *, tempdir=None, filefactory=FileFactory()
):
relparts = []
for path, info in image_registry.items():
tempfilename = tempfile.mkstemp(prefix="img", suffix=ext, dir=tempdir)
path = pathlib.Path(tempfilename[1])
- filewriter_fn(path, data.read(), "w+b")
+ with filefactory(path, "w+b") as out_f:
+ out_f.write(data.read())
+
+ # filewriter_fn(path, data.read(), "w+b")
desc = (
f'Inline image: "{info.desc}"'
return EMAIL_SIG_SEP.join((text, sig)) if sig else text
-def extract_signature(text, *, filereader_fn=filereader_fn):
+def extract_signature(text, *, filefactory=FileFactory()):
parts = text.split(EMAIL_SIG_SEP, 1)
if len(parts) == 1:
return text, None, None
path = pathlib.Path(re.split(r" +", lines.pop(0), maxsplit=1)[1])
textsig = "\n".join(lines)
- sig_input = filereader_fn(path.expanduser())
+ with filefactory(path.expanduser()) as sig_f:
+ sig_input = sig_f.read()
+
soup = bs4.BeautifulSoup(sig_input, "html.parser")
style = str(soup.style.extract()) if soup.style else ""
def convert_markdown_to_html(
- origtext,
- draftpath,
+ draft_f,
*,
related_to_html_only=False,
- css=None,
- filewriter_fn=filewriter_fn,
- filereader_fn=filereader_fn,
+ css_f=None,
+ htmldump_f=None,
+ filefactory=FileFactory(),
tempdir=None,
extensions=None,
extension_configs=None,
):
# TODO extension_configs need to be handled differently
extension_configs = extension_configs or {}
- extension_configs.setdefault("pymdownx.highlight", {})
- extension_configs["pymdownx.highlight"]["css_class"] = _CODEHILITE_CLASS
+ extension_configs.setdefault("pymdownx.highlight", {})[
+ "css_class"
+ ] = _CODEHILITE_CLASS
+
+ extensions = extensions or []
+ extensions.append(FormatFlowedNewlineExtension())
+ extensions.append(QuoteToAdmonitionExtension())
+ draft = draft_f.read()
origtext, textsig, htmlsig = extract_signature(
- origtext, filereader_fn=filereader_fn
+ draft, filefactory=filefactory
)
(
)
origtext = make_text_mail(origtext, textsig)
-
- filewriter_fn(draftpath, origtext, encoding="utf-8")
+ draft_f.write(origtext)
textpart = Part(
- "text", "plain", draftpath, "Plain-text version", orig=True
+ "text", "plain", draft_f.path, "Plain-text version", orig=True
)
htmltext = make_html_doc(htmltext, htmlsig)
- htmltext = apply_styling(htmltext, css)
+ htmltext = apply_styling(htmltext, css_f.read() if css_f else None)
- htmlpath = draftpath.with_suffix(".html")
- filewriter_fn(
- htmlpath, htmltext, encoding="utf-8", errors="xmlcharrefreplace"
- )
+ if draft_f.path:
+ htmlpath = draft_f.path.with_suffix(".html")
+ else:
+ htmlpath = pathlib.Path(
+ tempfile.mkstemp(suffix=".html", dir=tempdir)[1]
+ )
+ with filefactory(
+ htmlpath, "w", encoding="utf-8", errors="xmlcharrefreplace"
+ ) as out_f:
+ out_f.write(htmltext)
htmlpart = Part("text", "html", htmlpath, "HTML version")
+ if htmldump_f:
+ htmldump_f.write(htmltext)
+
imgparts = collect_inline_images(
- image_registry, tempdir=tempdir, filewriter_fn=filewriter_fn
+ image_registry, tempdir=tempdir, filefactory=filefactory
)
if related_to_html_only:
def do_massage(
draft_f,
- draftpath,
cmd_f,
*,
extensions=None,
css_f=None,
+ htmldump_f=None,
converter=convert_markdown_to_html,
related_to_html_only=True,
only_build=False,
# variable used to identify the command file we're currently writing
# to.
cmds = MuttCommands(cmd_f, debug=debug_commands)
- cmds.cmd('set editor="$my_editor"')
- cmds.cmd('set edit_headers="$my_edit_headers"')
- cmds.cmd("unset my_editor")
- cmds.cmd("unset my_edit_headers")
-
- # let's flush those commands, as there'll be a lot of pushes from now
- # on, which need to be run in reverse order
- cmds.flush()
extensions = extensions.split(",") if extensions else []
tree = converter(
- draft_f.read(),
- draftpath,
- css=css_f.read() if css_f else None,
+ draft_f,
+ css_f=css_f,
+ htmldump_f=htmldump_f,
related_to_html_only=related_to_html_only,
tempdir=tempdir,
extensions=extensions,
except AttributeError:
filename = "pytest_internal_file"
cmds.cmd(f"source 'rm -f {filename}|'")
+ cmds.cmd('set editor="$my_editor"')
+ cmds.cmd('set edit_headers="$my_edit_headers"')
+ cmds.cmd("unset my_editor")
+ cmds.cmd("unset my_edit_headers")
cmds.cmd("unset my_mdwn_postprocess_cmd_file")
cmds.flush()
)
elif args.mode == "massage":
- with open(args.MAILDRAFT, "r") as draft_f, open(
- args.cmdpath, "w"
- ) as cmd_f, open(args.css_file, "r") as css_f:
+ with (
+ File(args.MAILDRAFT, "r+") as draft_f,
+ File(args.cmdpath, "w") as cmd_f,
+ File(args.css_file, "r") as css_f,
+ File(args.dump_html, "w") as htmldump_f,
+ ):
do_massage(
draft_f,
- args.MAILDRAFT,
cmd_f,
extensions=args.extensions,
css_f=css_f,
+ htmldump_f=htmldump_f,
related_to_html_only=args.related_to_html_only,
max_other_attachments=args.max_number_other_attachments,
only_build=args.only_build,
try:
import pytest
- from io import StringIO
class Tests:
@pytest.fixture
def const1(self):
- return "CONSTANT STRING 1"
+ return "Curvature Vest Usher Dividing+T#iceps Senior"
@pytest.fixture
def const2(self):
- return "CONSTANT STRING 2"
+ return "Habitant Celestial 2litzy Resurf/ce Headpiece Harmonics"
+
+ @pytest.fixture
+ def fakepath(self):
+ return pathlib.Path("/does/not/exist")
+
+ @pytest.fixture
+ def fakepath2(self):
+ return pathlib.Path("/does/not/exist/either")
# NOTE: tests using the capsys fixture must specify sys.stdout to the
# functions they call, else old stdout is used and not captured
desc="Alternative",
)
+ @pytest.fixture
+ def mime_tree_nested(self):
+ return Multipart(
+ "relative",
+ children=[
+ Multipart(
+ "alternative",
+ children=[
+ Part(
+ "text",
+ "plain",
+ "part.txt",
+ desc="Plain",
+ orig=True,
+ ),
+ Multipart(
+ "alternative",
+ children=[
+ Part(
+ "text",
+ "plain",
+ "part.txt",
+ desc="Nested plain",
+ ),
+ Part(
+ "text",
+ "html",
+ "part.html",
+ desc="Nested HTML",
+ ),
+ ],
+ desc="Nested alternative",
+ ),
+ ],
+ desc="Alternative",
+ ),
+ Part(
+ "text",
+ "png",
+ "logo.png",
+ cid="logo.png",
+ desc="Logo",
+ ),
+ ],
+ desc="Related",
+ )
+
@pytest.mark.treewalk
def test_MIMETreeDFWalker_depth_first_walk(
self, mime_tree_related_to_alternative
return StringIO(text or const1)
@pytest.mark.massage
- def test_do_massage_basic(self, const1, string_io, capsys):
- def converter(
- drafttext,
- draftpath,
- css,
- related_to_html_only,
- extensions,
- tempdir,
- ):
- return Part("text", "plain", draftpath, orig=True)
+ def test_do_massage_basic(self):
+ def converter(draft_f, **kwargs):
+ return Part("text", "plain", draft_f.path, orig=True)
- do_massage(
- draft_f=string_io,
- draftpath=const1,
- cmd_f=sys.stdout,
- converter=converter,
- )
+ with File() as draft_f, File() as cmd_f:
+ do_massage(
+ draft_f=draft_f,
+ cmd_f=cmd_f,
+ converter=converter,
+ )
+ lines = cmd_f.read().splitlines()
- captured = capsys.readouterr()
- lines = captured.out.splitlines()
- assert '="$my_editor"' in lines.pop(0)
- assert '="$my_edit_headers"' in lines.pop(0)
- assert "unset my_editor" == lines.pop(0)
- assert "unset my_edit_headers" == lines.pop(0)
assert "send-message" in lines.pop(0)
assert "update-encoding" in lines.pop(0)
assert "first-entry" in lines.pop(0)
assert "source 'rm -f " in lines.pop(0)
+ assert '="$my_editor"' in lines.pop(0)
+ assert '="$my_edit_headers"' in lines.pop(0)
+ assert "unset my_editor" == lines.pop(0)
+ assert "unset my_edit_headers" == lines.pop(0)
assert "unset my_mdwn_postprocess_cmd_file" == lines.pop(0)
@pytest.mark.massage
- def test_do_massage_fulltree(
- self, string_io, const1, mime_tree_related_to_alternative, capsys
- ):
- def converter(
- drafttext,
- draftpath,
- css,
- related_to_html_only,
- extensions,
- tempdir,
- ):
+ def test_do_massage_fulltree(self, mime_tree_related_to_alternative):
+ def converter(draft_f, **kwargs):
return mime_tree_related_to_alternative
max_attachments = 5
- do_massage(
- draft_f=string_io,
- draftpath=const1,
- cmd_f=sys.stdout,
- max_other_attachments=max_attachments,
- converter=converter,
- )
- captured = capsys.readouterr()
- lines = captured.out.splitlines()[4:-2]
+ with File() as draft_f, File() as cmd_f:
+ do_massage(
+ draft_f=draft_f,
+ cmd_f=cmd_f,
+ max_other_attachments=max_attachments,
+ converter=converter,
+ )
+ lines = cmd_f.read().splitlines()[:-6]
+
assert "first-entry" in lines.pop()
assert "update-encoding" in lines.pop()
assert "Plain" in lines.pop()
assert "send-message" in lines.pop()
assert len(lines) == 0
- @pytest.fixture
- def fake_filewriter(self):
- class FileWriter:
- def __init__(self):
- self._writes = []
+ @pytest.mark.massage
+ def test_mime_tree_relative_within_alternative(
+ self, mime_tree_related_to_html
+ ):
+ def converter(draft_f, **kwargs):
+ return mime_tree_related_to_html
+
+ with File() as draft_f, File() as cmd_f:
+ do_massage(
+ draft_f=draft_f,
+ cmd_f=cmd_f,
+ converter=converter,
+ )
+ lines = cmd_f.read().splitlines()[:-6]
- def __call__(self, path, content, mode="w", **kwargs):
- self._writes.append((path, content))
+ assert "first-entry" in lines.pop()
+ assert "update-encoding" in lines.pop()
+ assert "Plain" in lines.pop()
+ assert "part.html" in lines.pop()
+ assert "toggle-unlink" in lines.pop()
+ assert "move-up" in lines.pop()
+ while True:
+ top = lines.pop()
+ if "move-up" not in top:
+ break
+ assert "move-down" in top
+ assert "HTML" in lines.pop()
+ assert "logo.png" in lines.pop()
+ assert "toggle-unlink" in lines.pop()
+ assert "content-id" in lines.pop()
+ assert "move-up" in lines.pop()
+ while True:
+ top = lines.pop()
+ if "move-up" not in top:
+ break
+ assert "move-down" in top
+ assert "move-down" in lines.pop()
+ assert "Logo" in lines.pop()
+ assert "jump>2" in lines.pop()
+ assert "jump>3" in lines.pop()
+ assert "group-related" in lines.pop()
+ assert "Related" in lines.pop()
+ assert "jump>1" in lines.pop()
+ assert "jump>2" in lines.pop()
+ assert "group-alternative" in lines.pop()
+ assert "Alternative" in lines.pop()
+ assert "send-message" in lines.pop()
+ assert len(lines) == 0
- def pop(self, index=-1):
- return self._writes.pop(index)
+ @pytest.mark.massage
+ def test_mime_tree_nested_trees_does_not_break_positioning(
+ self, mime_tree_nested
+ ):
+ def converter(draft_f, **kwargs):
+ return mime_tree_nested
- return FileWriter()
+ with File() as draft_f, File() as cmd_f:
+ do_massage(
+ draft_f=draft_f,
+ cmd_f=cmd_f,
+ converter=converter,
+ )
+ lines = cmd_f.read().splitlines()
- @pytest.fixture
- def markdown_non_converter(self, const1, const2):
- return lambda s, text: f"{const1}{text}{const2}"
+ while "logo.png" not in lines.pop():
+ pass
+ lines.pop()
+ assert "content-id" in lines.pop()
+ assert "move-up" in lines.pop()
+ while True:
+ top = lines.pop()
+ if "move-up" not in top:
+ break
+ assert "move-down" in top
+ # Due to the nested trees, the number of descendents of the sibling
+ # actually needs to be considered, not just the nieces. So to move
+ # from position 1 to position 6, it only needs one <move-down>
+ # because that jumps over the entire sibling tree. Thus what
+ # follows next must not be another <move-down>
+ assert "Logo" in lines.pop()
@pytest.mark.converter
- def test_converter_tree_basic(self, const1, const2, fake_filewriter):
- path = pathlib.Path(const2)
- tree = convert_markdown_to_html(
- const1, path, filewriter_fn=fake_filewriter
- )
+ def test_converter_tree_basic(self, fakepath, const1, fakefilefactory):
+ with fakefilefactory(fakepath, content=const1) as draft_f:
+ tree = convert_markdown_to_html(
+ draft_f, filefactory=fakefilefactory
+ )
assert tree.subtype == "alternative"
assert len(tree.children) == 2
assert tree.children[0].subtype == "plain"
- assert tree.children[0].path == path
+ assert tree.children[0].path == draft_f.path
assert tree.children[0].orig
assert tree.children[1].subtype == "html"
- assert tree.children[1].path == path.with_suffix(".html")
+ assert tree.children[1].path == fakepath.with_suffix(".html")
+ @pytest.mark.converter
def test_converter_writes(
- self,
- const1,
- const2,
- fake_filewriter,
- monkeypatch,
- markdown_non_converter,
+ self, fakepath, fakefilefactory, const1, monkeypatch
):
- path = pathlib.Path(const2)
+ with fakefilefactory(fakepath, content=const1) as draft_f:
+ convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
- with monkeypatch.context() as m:
- m.setattr(markdown.Markdown, "convert", markdown_non_converter)
- convert_markdown_to_html(
- const1, path, filewriter_fn=fake_filewriter
- )
-
- assert (path, const1) == fake_filewriter.pop(0)
- written = fake_filewriter.pop(0)
- assert path.with_suffix(".html") == written[0]
- assert const1 in written[1]
+ html = fakefilefactory.pop()
+ assert fakepath.with_suffix(".html") == html[0]
+ assert const1 in html[1].read()
+ text = fakefilefactory.pop()
+ assert fakepath == text[0]
+ assert const1 == text[1].read()
@pytest.mark.imgproc
def test_markdown_inline_image_processor(self):
@pytest.mark.converter
def test_converter_tree_inline_image_base64(
- self, test_png, const1, fake_filewriter
+ self, test_png, fakefilefactory
):
text = f""
- path = pathlib.Path(const1)
- tree = convert_markdown_to_html(
- text,
- path,
- filewriter_fn=fake_filewriter,
- related_to_html_only=False,
- )
+ with fakefilefactory(content=text) as draft_f:
+ tree = convert_markdown_to_html(
+ draft_f,
+ filefactory=fakefilefactory,
+ related_to_html_only=False,
+ )
assert tree.subtype == "relative"
assert tree.children[0].subtype == "alternative"
assert tree.children[1].subtype == "png"
- written = fake_filewriter.pop()
+ written = fakefilefactory.pop()
assert tree.children[1].path == written[0]
- assert written[1] == request.urlopen(test_png).read()
+ assert b"PNG" in written[1].read()
@pytest.mark.converter
def test_converter_tree_inline_image_base64_related_to_html(
- self, test_png, const1, fake_filewriter
+ self, test_png, fakefilefactory
):
text = f""
- path = pathlib.Path(const1)
- tree = convert_markdown_to_html(
- text,
- path,
- filewriter_fn=fake_filewriter,
- related_to_html_only=True,
- )
+ with fakefilefactory(content=text) as draft_f:
+ tree = convert_markdown_to_html(
+ draft_f,
+ filefactory=fakefilefactory,
+ related_to_html_only=True,
+ )
assert tree.subtype == "alternative"
assert tree.children[1].subtype == "relative"
assert tree.children[1].children[1].subtype == "png"
- written = fake_filewriter.pop()
+ written = fakefilefactory.pop()
assert tree.children[1].children[1].path == written[0]
- assert written[1] == request.urlopen(test_png).read()
+ assert b"PNG" in written[1].read()
@pytest.mark.converter
def test_converter_tree_inline_image_cid(
- self, const1, fake_filewriter
+ self, const1, fakefilefactory
):
text = f""
- path = pathlib.Path(const1)
- tree = convert_markdown_to_html(
- text,
- path,
- filewriter_fn=fake_filewriter,
- related_to_html_only=False,
- )
+ with fakefilefactory(content=text) as draft_f:
+ tree = convert_markdown_to_html(
+ draft_f,
+ filefactory=fakefilefactory,
+ related_to_html_only=False,
+ )
assert len(tree.children) == 2
assert tree.children[0].cid != const1
assert tree.children[0].type != "image"
assert tree.children[1].cid != const1
assert tree.children[1].type != "image"
+ @pytest.fixture
+ def fakefilefactory(self):
+ return FakeFileFactory()
+
@pytest.mark.imgcoll
def test_inline_image_collection(
- self, test_png, const1, const2, fake_filewriter
+ self, test_png, const1, const2, fakefilefactory
):
test_images = {test_png: InlineImageInfo(cid=const1, desc=const2)}
relparts = collect_inline_images(
- test_images, filewriter_fn=fake_filewriter
+ test_images, filefactory=fakefilefactory
)
- written = fake_filewriter.pop()
- assert b"PNG" in written[1]
+ written = fakefilefactory.pop()
+ assert b"PNG" in written[1].read()
assert relparts[0].subtype == "png"
assert relparts[0].path == written[0]
assert relparts[0].cid == const1
- assert relparts[0].desc.endswith(const2)
+ assert const2 in relparts[0].desc
if _PYNLINER:
assert 'p style="color' in out
@pytest.mark.styling
- def test_massage_styling_to_converter(self, string_io, const1):
+ def test_apply_no_stylesheet(self, const1):
+ out = apply_styling(const1, None)
+
+ @pytest.mark.massage
+ @pytest.mark.styling
+ def test_massage_styling_to_converter(self):
css = "p { color:red }"
- css_f = StringIO(css)
- out_f = StringIO()
css_applied = []
- def converter(
- drafttext,
- draftpath,
- css,
- related_to_html_only,
- extensions,
- tempdir,
- ):
+ def converter(draft_f, css_f, **kwargs):
+ css = css_f.read()
css_applied.append(css)
- return Part("text", "plain", draftpath, orig=True)
+ return Part("text", "plain", draft_f.path, orig=True)
- do_massage(
- draft_f=string_io,
- draftpath=const1,
- cmd_f=out_f,
- css_f=css_f,
- converter=converter,
- )
+ with (
+ File() as draft_f,
+ File(mode="w") as cmd_f,
+ File(content=css) as css_f,
+ ):
+ do_massage(
+ draft_f=draft_f,
+ cmd_f=cmd_f,
+ css_f=css_f,
+ converter=converter,
+ )
assert css_applied[0] == css
@pytest.mark.converter
+ @pytest.mark.styling
def test_converter_apply_styles(
- self, const1, fake_filewriter, monkeypatch
+ self, const1, monkeypatch, fakepath, fakefilefactory
):
- path = pathlib.Path(const1)
- text = "Hello, world!"
css = "p { color:red }"
- with monkeypatch.context() as m:
+ with (
+ monkeypatch.context() as m,
+ fakefilefactory(fakepath, content=const1) as draft_f,
+ fakefilefactory(content=css) as css_f,
+ ):
m.setattr(
markdown.Markdown,
"convert",
lambda s, t: f"<p>{t}</p>",
)
convert_markdown_to_html(
- text, path, css=css, filewriter_fn=fake_filewriter
+ draft_f, css_f=css_f, filefactory=fakefilefactory
)
- assert "color: red" in fake_filewriter.pop()[1]
+ assert re.search(
+ r"color:.*red",
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ )
if _PYGMENTS_CSS:
out = apply_styling(html, _PYGMENTS_CSS)
assert f'{_CODEHILITE_CLASS}" style="' in out
- @pytest.mark.massage
- def test_mime_tree_relative_within_alternative(
- self, string_io, const1, capsys, mime_tree_related_to_html
- ):
- def converter(
- drafttext,
- draftpath,
- css,
- related_to_html_only,
- extensions,
- tempdir,
- ):
- return mime_tree_related_to_html
-
- do_massage(
- draft_f=string_io,
- draftpath=const1,
- cmd_f=sys.stdout,
- converter=converter,
- )
-
- captured = capsys.readouterr()
- lines = captured.out.splitlines()[4:-2]
- assert "first-entry" in lines.pop()
- assert "update-encoding" in lines.pop()
- assert "Plain" in lines.pop()
- assert "part.html" in lines.pop()
- assert "toggle-unlink" in lines.pop()
- assert "move-up" in lines.pop()
- while True:
- top = lines.pop()
- if "move-up" not in top:
- break
- assert "move-down" in top
- assert "HTML" in lines.pop()
- assert "logo.png" in lines.pop()
- assert "toggle-unlink" in lines.pop()
- assert "content-id" in lines.pop()
- assert "move-up" in lines.pop()
- while True:
- top = lines.pop()
- if "move-up" not in top:
- break
- assert "move-down" in top
- assert "move-down" in lines.pop()
- assert "Logo" in lines.pop()
- assert "jump>2" in lines.pop()
- assert "jump>3" in lines.pop()
- assert "group-related" in lines.pop()
- assert "Related" in lines.pop()
- assert "jump>1" in lines.pop()
- assert "jump>2" in lines.pop()
- assert "group-alternative" in lines.pop()
- assert "Alternative" in lines.pop()
- assert "send-message" in lines.pop()
- assert len(lines) == 0
-
- @pytest.mark.massage
- def test_mime_tree_nested_trees_does_not_break_positioning(
- self, string_io, const1, capsys
- ):
- def converter(
- drafttext,
- draftpath,
- css,
- related_to_html_only,
- extensions,
- tempdir,
- ):
- return Multipart(
- "relative",
- children=[
- Multipart(
- "alternative",
- children=[
- Part(
- "text",
- "plain",
- "part.txt",
- desc="Plain",
- orig=True,
- ),
- Multipart(
- "alternative",
- children=[
- Part(
- "text",
- "plain",
- "part.txt",
- desc="Nested plain",
- ),
- Part(
- "text",
- "html",
- "part.html",
- desc="Nested HTML",
- ),
- ],
- desc="Nested alternative",
- ),
- ],
- desc="Alternative",
- ),
- Part(
- "text",
- "png",
- "logo.png",
- cid="logo.png",
- desc="Logo",
- ),
- ],
- desc="Related",
- )
-
- do_massage(
- draft_f=string_io,
- draftpath=const1,
- cmd_f=sys.stdout,
- converter=converter,
- )
-
- captured = capsys.readouterr()
- lines = captured.out.splitlines()
- while "logo.png" not in lines.pop():
- pass
- lines.pop()
- assert "content-id" in lines.pop()
- assert "move-up" in lines.pop()
- while True:
- top = lines.pop()
- if "move-up" not in top:
- break
- assert "move-down" in top
- # Due to the nested trees, the number of descendents of the sibling
- # actually needs to be considered, not just the nieces. So to move
- # from position 1 to position 6, it only needs one <move-down>
- # because that jumps over the entire sibling tree. Thus what
- # follows next must not be another <move-down>
- assert "Logo" in lines.pop()
-
@pytest.mark.sig
def test_signature_extraction_no_signature(self, const1):
assert (const1, None, None) == extract_signature(const1)
assert htmlsig is None
@pytest.mark.sig
- def test_signature_extraction_html(self, const1, const2):
- path = pathlib.Path("somepath")
+ def test_signature_extraction_html(
+ self, fakepath, fakefilefactory, const1, const2
+ ):
sigconst = "HTML signature from {path} but as a string"
+ sig = f'<div id="signature">{sigconst.format(path=fakepath)}</div>'
- def filereader_fn(path):
- return (
- f'<div id="signature">{sigconst.format(path=path)}</div>'
- )
+ sig_f = fakefilefactory(fakepath, content=sig)
origtext, textsig, htmlsig = extract_signature(
- f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {path}\n{const2}",
- filereader_fn=filereader_fn,
+ f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER} {fakepath}\n{const2}",
+ filefactory=fakefilefactory,
)
assert origtext == const1
assert textsig == const2
- assert htmlsig == sigconst.format(path=path)
+ assert htmlsig == sigconst.format(path=fakepath)
@pytest.mark.sig
- def test_signature_extraction_file_not_found(self, const1):
- path = pathlib.Path("/does/not/exist")
+ def test_signature_extraction_file_not_found(self, fakepath, const1):
with pytest.raises(FileNotFoundError):
origtext, textsig, htmlsig = extract_signature(
- f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{path}\n{const1}"
+ f"{const1}{EMAIL_SIG_SEP}{HTML_SIG_MARKER}{fakepath}\n{const1}"
)
@pytest.mark.imgproc
@pytest.mark.converter
@pytest.mark.sig
def test_converter_signature_handling(
- self, const1, fake_filewriter, monkeypatch
+ self, fakepath, fakefilefactory, monkeypatch
):
- path = pathlib.Path(const1)
-
mailparts = (
"This is the mail body\n",
f"{EMAIL_SIG_SEP}",
"This is a plain-text signature only",
)
- def filereader_fn(path):
- return ""
-
- with monkeypatch.context() as m:
+ with (
+ fakefilefactory(
+ fakepath, content="".join(mailparts)
+ ) as draft_f,
+ monkeypatch.context() as m,
+ ):
m.setattr(markdown.Markdown, "convert", lambda s, t: t)
- convert_markdown_to_html(
- "".join(mailparts),
- path,
- filewriter_fn=fake_filewriter,
- filereader_fn=filereader_fn,
- )
+ convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
- soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+ soup = bs4.BeautifulSoup(
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ "html.parser",
+ )
body = soup.body.contents
assert mailparts[0] in body.pop(0)
@pytest.mark.converter
@pytest.mark.sig
def test_converter_signature_handling_htmlsig(
- self, const1, fake_filewriter, monkeypatch
+ self, fakepath, fakepath2, fakefilefactory, monkeypatch
):
- path = pathlib.Path(const1)
-
mailparts = (
"This is the mail body",
f"{EMAIL_SIG_SEP}",
- f"{HTML_SIG_MARKER}{path}\n",
+ f"{HTML_SIG_MARKER}{fakepath2}\n",
"This is the plain-text version",
)
+ htmlsig = "HTML Signature from {path} but as a string"
+ html = f'<div id="signature"><p>{htmlsig.format(path=fakepath2)}</p></div>'
- htmlsig = "HTML Signature from {path}"
-
- def filereader_fn(path):
- return f'<div id="signature">{htmlsig.format(path=path)}</div>'
+ sig_f = fakefilefactory(fakepath2, content=html)
def mdwn_fn(t):
return t.upper()
- with monkeypatch.context() as m:
+ with (
+ fakefilefactory(
+ fakepath, content="".join(mailparts)
+ ) as draft_f,
+ monkeypatch.context() as m,
+ ):
m.setattr(
markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
)
- convert_markdown_to_html(
- "".join(mailparts),
- path,
- filewriter_fn=fake_filewriter,
- filereader_fn=filereader_fn,
- )
+ convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
- soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+ soup = bs4.BeautifulSoup(
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ "html.parser",
+ )
sig = soup.select_one("#signature")
sig.span.extract()
assert HTML_SIG_MARKER not in sig.text
- assert htmlsig.format(path=path) == sig.text.strip()
+ assert htmlsig.format(path=fakepath2) == sig.text.strip()
- plaintext = fake_filewriter.pop()[1]
+ plaintext = fakefilefactory[fakepath].read()
assert plaintext.endswith(EMAIL_SIG_SEP + mailparts[-1])
@pytest.mark.converter
@pytest.mark.sig
def test_converter_signature_handling_htmlsig_with_image(
- self, const1, fake_filewriter, monkeypatch, test_png
+ self, fakepath, fakepath2, fakefilefactory, monkeypatch, test_png
):
- path = pathlib.Path(const1)
-
mailparts = (
"This is the mail body",
f"{EMAIL_SIG_SEP}",
- f"{HTML_SIG_MARKER}{path}\n",
+ f"{HTML_SIG_MARKER}{fakepath2}\n",
"This is the plain-text version",
)
-
htmlsig = (
"HTML Signature from {path} with image\n"
f'<img src="{test_png}">\n'
)
+ html = (
+ f'<div id="signature">{htmlsig.format(path=fakepath2)}</div>'
+ )
- def filereader_fn(path):
- return f'<div id="signature">{htmlsig.format(path=path)}</div>'
+ sig_f = fakefilefactory(fakepath2, content=html)
def mdwn_fn(t):
return t.upper()
- with monkeypatch.context() as m:
+ with (
+ fakefilefactory(
+ fakepath, content="".join(mailparts)
+ ) as draft_f,
+ monkeypatch.context() as m,
+ ):
m.setattr(
markdown.Markdown, "convert", lambda s, t: mdwn_fn(t)
)
- convert_markdown_to_html(
- "".join(mailparts),
- path,
- filewriter_fn=fake_filewriter,
- filereader_fn=filereader_fn,
- )
+ convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
- assert fake_filewriter.pop()[0].suffix == ".png"
+ assert fakefilefactory.pop()[0].suffix == ".png"
- soup = bs4.BeautifulSoup(fake_filewriter.pop()[1], "html.parser")
+ soup = bs4.BeautifulSoup(
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ "html.parser",
+ )
assert soup.img.attrs["src"].startswith("cid:")
@pytest.mark.converter
@pytest.mark.sig
def test_converter_signature_handling_textsig_with_image(
- self, const1, fake_filewriter, test_png
+ self, fakepath, fakefilefactory, test_png
):
mailparts = (
"This is the mail body",
f"{EMAIL_SIG_SEP}",
"This is the plain-text version with image\n",
f"",
-
- )
- tree = convert_markdown_to_html
- "".join(mailparts),
- pathlib.Path(const1),
- filewriter_fn=fake_filewriter,
)
+ with (
+ fakefilefactory(
+ fakepath, content="".join(mailparts)
+ ) as draft_f,
+ ):
+ tree = convert_markdown_to_html(
+ draft_f, filefactory=fakefilefactory
+ )
assert tree.subtype == "relative"
assert tree.children[0].subtype == "alternative"
assert tree.children[1].subtype == "png"
- written = fake_filewriter.pop()
+ written = fakefilefactory.pop()
assert tree.children[1].path == written[0]
- assert written[1] == request.urlopen(test_png).read()
+ assert written[1].read() == request.urlopen(test_png).read()
+
+ @pytest.mark.converter
+ def test_converter_attribution_to_admonition(
+ self, fakepath, fakefilefactory
+ ):
+ mailparts = (
+ "Regarding whatever",
+ "> blockquote line1",
+ "> blockquote line2",
+ "> ",
+ "> new para with **bold** text",
+ )
+ with fakefilefactory(
+ fakepath, content="\n".join(mailparts)
+ ) as draft_f:
+ convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
+
+ soup = bs4.BeautifulSoup(
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ "html.parser",
+ )
+ quote = soup.select_one("div.admonition.quote")
+ assert quote
+ assert (
+ soup.select_one("p.admonition-title").extract().text.strip()
+ == mailparts[0]
+ )
+
+ p = quote.p.extract()
+ assert p.text.strip() == "\n".join(p[2:] for p in mailparts[1:3])
+
+ p = quote.p.extract()
+ assert p.contents[1].name == "strong"
+
+ @pytest.mark.converter
+ def test_converter_attribution_to_admonition_with_blockquote(
+ self, fakepath, fakefilefactory
+ ):
+ mailparts = (
+ "Regarding whatever",
+ "> blockquote line1",
+ "> blockquote line2",
+ "> ",
+ "> new para with **bold** text",
+ )
+ with fakefilefactory(
+ fakepath, content="\n".join(mailparts)
+ ) as draft_f:
+ convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
+
+ soup = bs4.BeautifulSoup(
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ "html.parser",
+ )
+ quote = soup.select_one("div.admonition.quote")
+ assert quote.blockquote
+
+ @pytest.mark.converter
+ def test_converter_attribution_to_admonition_multiple(
+ self, fakepath, fakefilefactory
+ ):
+ mailparts = (
+ "Regarding whatever",
+ "> blockquote line1",
+ "> blockquote line2",
+ "",
+ "Normal text",
+ "",
+ "> continued emailquote",
+ "",
+ "Another email-quote",
+ "> something",
+ )
+ with fakefilefactory(
+ fakepath, content="\n".join(mailparts)
+ ) as draft_f:
+ convert_markdown_to_html(draft_f, filefactory=fakefilefactory)
+
+ soup = bs4.BeautifulSoup(
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ "html.parser",
+ )
+ quote = soup.select_one("div.admonition.quote.continued").extract()
+ assert quote
+ assert (
+ quote.select_one("p.admonition-title").extract().text.strip()
+ == mailparts[0]
+ )
- def test_converter_attribution_to_admonition(self, fake_filewriter):
+ p = quote.p.extract()
+ assert p
+ quote = soup.select_one("div.admonition.quote.continued").extract()
+ assert quote
+ assert (
+ quote.select_one("p.admonition-title").extract().text.strip()
+ == mailparts[-2]
+ )
+
+ @pytest.mark.converter
+ def test_converter_format_flowed_with_nl2br(
+ self, fakepath, fakefilefactory
+ ):
+ mailparts = (
+ "This is format=flowed text ",
+ "with spaces at the end ",
+ "and there ought be no newlines.",
+ "",
+ "[link](https://example.org) ",
+ "and text.",
+ "",
+ "[link text ",
+ "broken up](https://example.org).",
+ "",
+ "This is on a new line with a hard break ",
+ "due to the double space",
+ )
+ with fakefilefactory(
+ fakepath, content="\n".join(mailparts)
+ ) as draft_f:
+ convert_markdown_to_html(
+ draft_f, extensions=["nl2br"], filefactory=fakefilefactory
+ )
+
+ soup = bs4.BeautifulSoup(
+ fakefilefactory[fakepath.with_suffix(".html")].read(),
+ "html.parser",
+ )
+ import ipdb
+
+ p = soup.p.extract().text
+ assert "".join(mailparts[0:3]) == p
+ p = ''.join(map(str, soup.p.extract().contents))
+ assert p == '<a href="https://example.org">link</a> and text.'
+ p = ''.join(map(str, soup.p.extract().contents))
+ assert (
+ p == '<a href="https://example.org">link text broken up</a>.'
+ )
+
+ @pytest.mark.fileio
+ def test_file_class_contextmanager(self, const1, monkeypatch):
+ state = dict(o=False, c=False)
+
+ def fn(t):
+ state[t] = True
+
+ with monkeypatch.context() as m:
+ m.setattr(File, "open", lambda s: fn("o"))
+ m.setattr(File, "close", lambda s: fn("c"))
+ with File() as f:
+ assert state["o"]
+ assert not state["c"]
+ assert state["c"]
+
+ @pytest.mark.fileio
+ def test_file_class_no_path(self, const1):
+ with File(mode="w+") as f:
+ f.write(const1, cache=False)
+ assert f.read(cache=False) == const1
+
+ @pytest.mark.fileio
+ def test_file_class_path(self, const1, tmp_path):
+ with File(tmp_path / "file", mode="w+") as f:
+ f.write(const1, cache=False)
+ assert f.read(cache=False) == const1
+
+ @pytest.mark.fileio
+ def test_file_class_path_no_exists(self, fakepath):
+ with pytest.raises(FileNotFoundError):
+ File(fakepath, mode="r").open()
+
+ @pytest.mark.fileio
+ def test_file_class_cache(self, tmp_path, const1, const2):
+ path = tmp_path / "file"
+ file = File(path, mode="w+")
+ with file as f:
+ f.write(const1, cache=True)
+ with open(path, mode="w") as f:
+ f.write(const2)
+ with file as f:
+ assert f.read(cache=True) == const1
+
+ @pytest.mark.fileio
+ def test_file_class_cache_init(self, const1):
+ file = File(path=None, mode="r", content=const1)
+ with file as f:
+ assert f.read() == const1
+
+ @pytest.mark.fileio
+ def test_file_class_content_or_path(self, fakepath, const1):
+ with pytest.raises(RuntimeError):
+ file = File(path=fakepath, content=const1)
+
+ @pytest.mark.fileio
+ def test_file_class_content_needs_read(self, const1):
+ with pytest.raises(RuntimeError):
+ file = File(mode="w", content=const1)
+
+ @pytest.mark.fileio
+ def test_file_class_write_persists_close(self, const1):
+ f = File(mode="w+")
+ with f:
+ f.write(const1)
+ with f:
+ assert f.read() == const1
+
+ @pytest.mark.fileio
+ def test_file_class_write_resets_read_cache(self, const1, const2):
+ with File(mode="w+", content=const1) as f:
+ assert f.read() == const1
+ f.write(const2)
+ assert f.read() == const2
+
+ @pytest.mark.fileio
+ def test_file_factory(self):
+ fact = FileFactory()
+ f = fact()
+ assert isinstance(f, File)
+ assert len(fact) == 1
+ assert f in fact
+ assert f == fact[0]
+
+ @pytest.mark.fileio
+ def test_fake_file_factory(self, fakepath, fakefilefactory):
+ fact = FakeFileFactory()
+ f = fakefilefactory(fakepath)
+ assert f.path == fakepath
+ assert f == fakefilefactory[fakepath]
+
+ @pytest.mark.fileio
+ def test_fake_file_factory_path_persistence(
+ self, fakepath, fakefilefactory
+ ):
+ f1 = fakefilefactory(fakepath)
+ assert f1 == fakefilefactory(fakepath)
except ImportError:
pass
projects / etc / neomutt.git / blobdiff