]> git.madduck.net Git - etc/vim.git/blob - src/black/strings.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Upgrade to Furo 2023.9.10 to fix docs build (#3873)
[etc/vim.git] / src / black / strings.py
1 """
2 Simple formatting on strings. Further string formatting code is in trans.py.
3 """
4
5 import re
6 import sys
7 from functools import lru_cache
8 from typing import Final, List, Match, Pattern
9
10 from black._width_table import WIDTH_TABLE
11 from blib2to3.pytree import Leaf
12
13 STRING_PREFIX_CHARS: Final = "furbFURB"  # All possible string prefix characters.
14 STRING_PREFIX_RE: Final = re.compile(
15     r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL
16 )
17 FIRST_NON_WHITESPACE_RE: Final = re.compile(r"\s*\t+\s*(\S)")
18 UNICODE_ESCAPE_RE: Final = re.compile(
19     r"(?P<backslashes>\\+)(?P<body>"
20     r"(u(?P<u>[a-fA-F0-9]{4}))"  # Character with 16-bit hex value xxxx
21     r"|(U(?P<U>[a-fA-F0-9]{8}))"  # Character with 32-bit hex value xxxxxxxx
22     r"|(x(?P<x>[a-fA-F0-9]{2}))"  # Character with hex value hh
23     r"|(N\{(?P<N>[a-zA-Z0-9 \-]{2,})\})"  # Character named name in the Unicode database
24     r")",
25     re.VERBOSE,
26 )
27
28
29 def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
30     """Replace `regex` with `replacement` twice on `original`.
31
32     This is used by string normalization to perform replaces on
33     overlapping matches.
34     """
35     return regex.sub(replacement, regex.sub(replacement, original))
36
37
38 def has_triple_quotes(string: str) -> bool:
39     """
40     Returns:
41         True iff @string starts with three quotation characters.
42     """
43     raw_string = string.lstrip(STRING_PREFIX_CHARS)
44     return raw_string[:3] in {'"""', "'''"}
45
46
47 def lines_with_leading_tabs_expanded(s: str) -> List[str]:
48     """
49     Splits string into lines and expands only leading tabs (following the normal
50     Python rules)
51     """
52     lines = []
53     for line in s.splitlines():
54         # Find the index of the first non-whitespace character after a string of
55         # whitespace that includes at least one tab
56         match = FIRST_NON_WHITESPACE_RE.match(line)
57         if match:
58             first_non_whitespace_idx = match.start(1)
59
60             lines.append(
61                 line[:first_non_whitespace_idx].expandtabs()
62                 + line[first_non_whitespace_idx:]
63             )
64         else:
65             lines.append(line)
66     return lines
67
68
69 def fix_docstring(docstring: str, prefix: str) -> str:
70     # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation
71     if not docstring:
72         return ""
73     lines = lines_with_leading_tabs_expanded(docstring)
74     # Determine minimum indentation (first line doesn't count):
75     indent = sys.maxsize
76     for line in lines[1:]:
77         stripped = line.lstrip()
78         if stripped:
79             indent = min(indent, len(line) - len(stripped))
80     # Remove indentation (first line is special):
81     trimmed = [lines[0].strip()]
82     if indent < sys.maxsize:
83         last_line_idx = len(lines) - 2
84         for i, line in enumerate(lines[1:]):
85             stripped_line = line[indent:].rstrip()
86             if stripped_line or i == last_line_idx:
87                 trimmed.append(prefix + stripped_line)
88             else:
89                 trimmed.append("")
90     return "\n".join(trimmed)
91
92
93 def get_string_prefix(string: str) -> str:
94     """
95     Pre-conditions:
96         * assert_is_leaf_string(@string)
97
98     Returns:
99         @string's prefix (e.g. '', 'r', 'f', or 'rf').
100     """
101     assert_is_leaf_string(string)
102
103     prefix = ""
104     prefix_idx = 0
105     while string[prefix_idx] in STRING_PREFIX_CHARS:
106         prefix += string[prefix_idx]
107         prefix_idx += 1
108
109     return prefix
110
111
112 def assert_is_leaf_string(string: str) -> None:
113     """
114     Checks the pre-condition that @string has the format that you would expect
115     of `leaf.value` where `leaf` is some Leaf such that `leaf.type ==
116     token.STRING`. A more precise description of the pre-conditions that are
117     checked are listed below.
118
119     Pre-conditions:
120         * @string starts with either ', ", <prefix>', or <prefix>" where
121         `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`.
122         * @string ends with a quote character (' or ").
123
124     Raises:
125         AssertionError(...) if the pre-conditions listed above are not
126         satisfied.
127     """
128     dquote_idx = string.find('"')
129     squote_idx = string.find("'")
130     if -1 in [dquote_idx, squote_idx]:
131         quote_idx = max(dquote_idx, squote_idx)
132     else:
133         quote_idx = min(squote_idx, dquote_idx)
134
135     assert (
136         0 <= quote_idx < len(string) - 1
137     ), f"{string!r} is missing a starting quote character (' or \")."
138     assert string[-1] in (
139         "'",
140         '"',
141     ), f"{string!r} is missing an ending quote character (' or \")."
142     assert set(string[:quote_idx]).issubset(
143         set(STRING_PREFIX_CHARS)
144     ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}."
145
146
147 def normalize_string_prefix(s: str) -> str:
148     """Make all string prefixes lowercase."""
149     match = STRING_PREFIX_RE.match(s)
150     assert match is not None, f"failed to match string {s!r}"
151     orig_prefix = match.group(1)
152     new_prefix = (
153         orig_prefix.replace("F", "f")
154         .replace("B", "b")
155         .replace("U", "")
156         .replace("u", "")
157     )
158
159     # Python syntax guarantees max 2 prefixes and that one of them is "r"
160     if len(new_prefix) == 2 and "r" != new_prefix[0].lower():
161         new_prefix = new_prefix[::-1]
162     return f"{new_prefix}{match.group(2)}"
163
164
165 # Re(gex) does actually cache patterns internally but this still improves
166 # performance on a long list literal of strings by 5-9% since lru_cache's
167 # caching overhead is much lower.
168 @lru_cache(maxsize=64)
169 def _cached_compile(pattern: str) -> Pattern[str]:
170     return re.compile(pattern)
171
172
173 def normalize_string_quotes(s: str) -> str:
174     """Prefer double quotes but only if it doesn't cause more escaping.
175
176     Adds or removes backslashes as appropriate. Doesn't parse and fix
177     strings nested in f-strings.
178     """
179     value = s.lstrip(STRING_PREFIX_CHARS)
180     if value[:3] == '"""':
181         return s
182
183     elif value[:3] == "'''":
184         orig_quote = "'''"
185         new_quote = '"""'
186     elif value[0] == '"':
187         orig_quote = '"'
188         new_quote = "'"
189     else:
190         orig_quote = "'"
191         new_quote = '"'
192     first_quote_pos = s.find(orig_quote)
193     if first_quote_pos == -1:
194         return s  # There's an internal error
195
196     prefix = s[:first_quote_pos]
197     unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
198     escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
199     escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
200     body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)]
201     if "r" in prefix.casefold():
202         if unescaped_new_quote.search(body):
203             # There's at least one unescaped new_quote in this raw string
204             # so converting is impossible
205             return s
206
207         # Do not introduce or remove backslashes in raw strings
208         new_body = body
209     else:
210         # remove unnecessary escapes
211         new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
212         if body != new_body:
213             # Consider the string without unnecessary escapes as the original
214             body = new_body
215             s = f"{prefix}{orig_quote}{body}{orig_quote}"
216         new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
217         new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
218     if "f" in prefix.casefold():
219         matches = re.findall(
220             r"""
221             (?:(?<!\{)|^)\{  # start of the string or a non-{ followed by a single {
222                 ([^{].*?)  # contents of the brackets except if begins with {{
223             \}(?:(?!\})|$)  # A } followed by end of the string or a non-}
224             """,
225             new_body,
226             re.VERBOSE,
227         )
228         for m in matches:
229             if "\\" in str(m):
230                 # Do not introduce backslashes in interpolated expressions
231                 return s
232
233     if new_quote == '"""' and new_body[-1:] == '"':
234         # edge case:
235         new_body = new_body[:-1] + '\\"'
236     orig_escape_count = body.count("\\")
237     new_escape_count = new_body.count("\\")
238     if new_escape_count > orig_escape_count:
239         return s  # Do not introduce more escaping
240
241     if new_escape_count == orig_escape_count and orig_quote == '"':
242         return s  # Prefer double quotes
243
244     return f"{prefix}{new_quote}{new_body}{new_quote}"
245
246
247 def normalize_unicode_escape_sequences(leaf: Leaf) -> None:
248     """Replace hex codes in Unicode escape sequences with lowercase representation."""
249     text = leaf.value
250     prefix = get_string_prefix(text)
251     if "r" in prefix.lower():
252         return
253
254     def replace(m: Match[str]) -> str:
255         groups = m.groupdict()
256         back_slashes = groups["backslashes"]
257
258         if len(back_slashes) % 2 == 0:
259             return back_slashes + groups["body"]
260
261         if groups["u"]:
262             # \u
263             return back_slashes + "u" + groups["u"].lower()
264         elif groups["U"]:
265             # \U
266             return back_slashes + "U" + groups["U"].lower()
267         elif groups["x"]:
268             # \x
269             return back_slashes + "x" + groups["x"].lower()
270         else:
271             assert groups["N"], f"Unexpected match: {m}"
272             # \N{}
273             return back_slashes + "N{" + groups["N"].upper() + "}"
274
275     leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text)
276
277
278 @lru_cache(maxsize=4096)
279 def char_width(char: str) -> int:
280     """Return the width of a single character as it would be displayed in a
281     terminal or editor (which respects Unicode East Asian Width).
282
283     Full width characters are counted as 2, while half width characters are
284     counted as 1.  Also control characters are counted as 0.
285     """
286     table = WIDTH_TABLE
287     codepoint = ord(char)
288     highest = len(table) - 1
289     lowest = 0
290     idx = highest // 2
291     while True:
292         start_codepoint, end_codepoint, width = table[idx]
293         if codepoint < start_codepoint:
294             highest = idx - 1
295         elif codepoint > end_codepoint:
296             lowest = idx + 1
297         else:
298             return 0 if width < 0 else width
299         if highest < lowest:
300             break
301         idx = (highest + lowest) // 2
302     return 1
303
304
305 def str_width(line_str: str) -> int:
306     """Return the width of `line_str` as it would be displayed in a terminal
307     or editor (which respects Unicode East Asian Width).
308
309     You could utilize this function to determine, for example, if a string
310     is too wide to display in a terminal or editor.
311     """
312     if line_str.isascii():
313         # Fast path for a line consisting of only ASCII characters
314         return len(line_str)
315     return sum(map(char_width, line_str))
316
317
318 def count_chars_in_width(line_str: str, max_width: int) -> int:
319     """Count the number of characters in `line_str` that would fit in a
320     terminal or editor of `max_width` (which respects Unicode East Asian
321     Width).
322     """
323     total_width = 0
324     for i, char in enumerate(line_str):
325         width = char_width(char)
326         if width + total_width > max_width:
327             return i
328         total_width += width
329     return len(line_str)