]> git.madduck.net Git - etc/vim.git/commitdiff

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

normalize numeric literals (#454)
authorJelle Zijlstra <jelle.zijlstra@gmail.com>
Fri, 17 Aug 2018 14:03:58 +0000 (07:03 -0700)
committerŁukasz Langa <lukasz@langa.pl>
Fri, 17 Aug 2018 14:03:58 +0000 (15:03 +0100)
Fixes #452

I ended up making a couple of other normalizations to numeric literals
too (lowercase everything, don't allow leading or trailing . in floats,
remove redundant + sign in exponent). I don't care too much about those,
so I'm happy to change the behavior there.

For reference, here is Python's grammar for numeric literals:
https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals

black.py
tests/data/numeric_literals.py [new file with mode: 0644]
tests/data/numeric_literals_py2.py [new file with mode: 0644]
tests/test_black.py

index c9b8be975d239b30cc2ecf4bc7723c60a34ce13e..7edf2aea059bf2b9c2a84f056e160e7c14b96a82 100644 (file)
--- a/black.py
+++ b/black.py
@@ -605,6 +605,7 @@ def format_str(
         remove_u_prefix=py36 or "unicode_literals" in future_imports,
         is_pyi=is_pyi,
         normalize_strings=normalize_strings,
         remove_u_prefix=py36 or "unicode_literals" in future_imports,
         is_pyi=is_pyi,
         normalize_strings=normalize_strings,
+        allow_underscores=py36,
     )
     elt = EmptyLineTracker(is_pyi=is_pyi)
     empty_line = Line()
     )
     elt = EmptyLineTracker(is_pyi=is_pyi)
     empty_line = Line()
@@ -1391,6 +1392,7 @@ class LineGenerator(Visitor[Line]):
     normalize_strings: bool = True
     current_line: Line = Factory(Line)
     remove_u_prefix: bool = False
     normalize_strings: bool = True
     current_line: Line = Factory(Line)
     remove_u_prefix: bool = False
+    allow_underscores: bool = False
 
     def line(self, indent: int = 0) -> Iterator[Line]:
         """Generate a line.
 
     def line(self, indent: int = 0) -> Iterator[Line]:
         """Generate a line.
@@ -1432,6 +1434,8 @@ class LineGenerator(Visitor[Line]):
             if self.normalize_strings and node.type == token.STRING:
                 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
                 normalize_string_quotes(node)
             if self.normalize_strings and node.type == token.STRING:
                 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
                 normalize_string_quotes(node)
+            if node.type == token.NUMBER:
+                normalize_numeric_literal(node, self.allow_underscores)
             if node.type not in WHITESPACE:
                 self.current_line.append(node)
         yield from super().visit_default(node)
             if node.type not in WHITESPACE:
                 self.current_line.append(node)
         yield from super().visit_default(node)
@@ -2493,6 +2497,61 @@ def normalize_string_quotes(leaf: Leaf) -> None:
     leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
 
 
     leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
 
 
+def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
+    """Normalizes numeric (float, int, and complex) literals."""
+    # We want all letters (e in exponents, j in complex literals, a-f
+    # in hex literals) to be lowercase.
+    text = leaf.value.lower()
+    if text.startswith(("0o", "0x", "0b")):
+        # Leave octal, hex, and binary literals alone for now.
+        pass
+    elif "e" in text:
+        before, after = text.split("e")
+        if after.startswith("-"):
+            after = after[1:]
+            sign = "-"
+        elif after.startswith("+"):
+            after = after[1:]
+            sign = ""
+        else:
+            sign = ""
+        before = format_float_or_int_string(before, allow_underscores)
+        after = format_int_string(after, allow_underscores)
+        text = f"{before}e{sign}{after}"
+    # Complex numbers and Python 2 longs
+    elif "j" in text or "l" in text:
+        number = text[:-1]
+        suffix = text[-1]
+        text = f"{format_float_or_int_string(number, allow_underscores)}{suffix}"
+    else:
+        text = format_float_or_int_string(text, allow_underscores)
+    leaf.value = text
+
+
+def format_float_or_int_string(text: str, allow_underscores: bool) -> str:
+    """Formats a float string like "1.0"."""
+    if "." not in text:
+        return format_int_string(text, allow_underscores)
+    before, after = text.split(".")
+    before = format_int_string(before, allow_underscores) if before else "0"
+    after = format_int_string(after, allow_underscores) if after else "0"
+    return f"{before}.{after}"
+
+
+def format_int_string(text: str, allow_underscores: bool) -> str:
+    """Normalizes underscores in a string to e.g. 1_000_000.
+
+    Input must be a string consisting only of digits and underscores.
+    """
+    if not allow_underscores:
+        return text
+    text = text.replace("_", "")
+    if len(text) <= 6:
+        # No underscores for numbers <= 6 digits long.
+        return text
+    return format(int(text), "3_")
+
+
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 
diff --git a/tests/data/numeric_literals.py b/tests/data/numeric_literals.py
new file mode 100644 (file)
index 0000000..8999b9d
--- /dev/null
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3.6
+
+x = 123456789
+x = 123456
+x = .1
+x = 1.
+x = 1E+1
+x = 1E-1
+x = 123456789.123456789
+x = 123456789.123456789E123456789
+x = 123456789E123456789
+x = 123456789J
+x = 123456789.123456789J
+x = 0XB1ACC
+x = 0B1011
+x = 0O777
+
+# output
+
+
+#!/usr/bin/env python3.6
+
+x = 123_456_789
+x = 123456
+x = 0.1
+x = 1.0
+x = 1e1
+x = 1e-1
+x = 123_456_789.123_456_789
+x = 123_456_789.123_456_789e123_456_789
+x = 123_456_789e123_456_789
+x = 123_456_789j
+x = 123_456_789.123_456_789j
+x = 0xb1acc
+x = 0b1011
+x = 0o777
diff --git a/tests/data/numeric_literals_py2.py b/tests/data/numeric_literals_py2.py
new file mode 100644 (file)
index 0000000..107c39b
--- /dev/null
@@ -0,0 +1,12 @@
+#!/usr/bin/env python2.7
+
+x = 123456789L
+x = 123456789
+
+# output
+
+
+#!/usr/bin/env python2.7
+
+x = 123456789l
+x = 123456789
index 02bad20d66a9c79391b9b980823c7b62d623f657..884d0463a3fa294fcec898aaff2d51ea454adc02 100644 (file)
@@ -373,6 +373,21 @@ class BlackTestCase(unittest.TestCase):
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, line_length=ll)
 
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, line_length=ll)
 
+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_numeric_literals(self) -> None:
+        source, expected = read_data("numeric_literals")
+        actual = fs(source, mode=black.FileMode.PYTHON36)
+        self.assertFormatEqual(expected, actual)
+        black.assert_equivalent(source, actual)
+        black.assert_stable(source, actual, line_length=ll)
+
+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_numeric_literals_py2(self) -> None:
+        source, expected = read_data("numeric_literals_py2")
+        actual = fs(source)
+        self.assertFormatEqual(expected, actual)
+        black.assert_stable(source, actual, line_length=ll)
+
     @patch("black.dump_to_file", dump_to_stderr)
     def test_python2(self) -> None:
         source, expected = read_data("python2")
     @patch("black.dump_to_file", dump_to_stderr)
     def test_python2(self) -> None:
         source, expected = read_data("python2")