Remove numeric underscore normalization (#696)

diff --git a/README.md b/README.md
index bd6ebcd01a4ef2e9c1a56ded3a06eb22590b231c..69468e297a2caa18ea56fdb5d2c3e8051c98d8ce 100644 (file)
--- a/README.md
+++ b/README.md
@@ -87,9 +87,6 @@ Options:
                                   piping source on standard input).
   -S, --skip-string-normalization
                                   Don't normalize string quotes or prefixes.
                                   piping source on standard input).
   -S, --skip-string-normalization
                                   Don't normalize string quotes or prefixes.

-  -N, --skip-numeric-underscore-normalization
-                                  Don't normalize underscores in numeric
-                                  literals.

   --check                         Don't write the files back, just return the
                                   status.  Return code 0 means nothing would
                                   change.  Return code 1 means some files
   --check                         Don't write the files back, just return the
                                   status.  Return code 0 means nothing would
                                   change.  Return code 1 means some files


@@ -395,14 +392,8 @@ an adoption helper, avoid using this for new projects.
 *Black* standardizes most numeric literals to use lowercase letters for the
 syntactic parts and uppercase letters for the digits themselves: `0xAB`
 instead of `0XAB` and `1e10` instead of `1E10`. Python 2 long literals are
 *Black* standardizes most numeric literals to use lowercase letters for the
 syntactic parts and uppercase letters for the digits themselves: `0xAB`
 instead of `0XAB` and `1e10` instead of `1E10`. Python 2 long literals are

-styled as `2L` instead of `2l` to avoid confusion between `l` and `1`. In
-Python 3.6+, *Black* adds underscores to long numeric literals to aid
-readability: `100000000` becomes `100_000_000`.
+styled as `2L` instead of `2l` to avoid confusion between `l` and `1`.

 
 

-For regions where numerals are grouped differently (like [India](https://en.wikipedia.org/wiki/Indian_numbering_system)
-and [China](https://en.wikipedia.org/wiki/Chinese_numerals#Whole_numbers)),
-the `-N` or `--skip-numeric-underscore-normalization` command line option
-makes *Black* preserve underscores in numeric literals.

 
 ### Line breaks & binary operators
 
 
 ### Line breaks & binary operators
 


@@ -823,8 +814,6 @@ The headers controlling how code is formatted are:
  - `X-Skip-String-Normalization`: corresponds to the `--skip-string-normalization`
     command line flag. If present and its value is not the empty string, no string
     normalization will be performed.
  - `X-Skip-String-Normalization`: corresponds to the `--skip-string-normalization`
     command line flag. If present and its value is not the empty string, no string
     normalization will be performed.

- - `X-Skip-Numeric-Underscore-Normalization`: corresponds to the
-    `--skip-numeric-underscore-normalization` command line flag.

  - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as *Black* does when
     passed the `--fast` command line flag.
  - `X-Python-Variant`: if set to `pyi`, `blackd` will act as *Black* does when
  - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as *Black* does when
     passed the `--fast` command line flag.
  - `X-Python-Variant`: if set to `pyi`, `blackd` will act as *Black* does when


@@ -950,7 +939,9 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Change Log
 
 
 ## Change Log
 

-### 18.11b0
+### 19.2b0
+
+* *Black* no longer normalizes numeric literals to include `_` separators.

 
 * new option `--target-version` to control which Python versions
   *Black*-formatted code should target
 
 * new option `--target-version` to control which Python versions
   *Black*-formatted code should target

diff --git a/black.py b/black.py
index b165851be91ab79847de49e2902122a3e80a132e..34bd59eacdddb8998ca7231d0c959f49f19b611e 100644 (file)
--- a/black.py
+++ b/black.py
@@ -168,7 +168,6 @@ VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
 class FileMode:
     target_versions: Set[TargetVersion] = Factory(set)
     line_length: int = DEFAULT_LINE_LENGTH
 class FileMode:
     target_versions: Set[TargetVersion] = Factory(set)
     line_length: int = DEFAULT_LINE_LENGTH

-    numeric_underscore_normalization: bool = True

     string_normalization: bool = True
     is_pyi: bool = False
 
     string_normalization: bool = True
     is_pyi: bool = False
 


@@ -183,7 +182,6 @@ class FileMode:
         parts = [
             version_str,
             str(self.line_length),
         parts = [
             version_str,
             str(self.line_length),

-            str(int(self.numeric_underscore_normalization)),

             str(int(self.string_normalization)),
             str(int(self.is_pyi)),
         ]
             str(int(self.string_normalization)),
             str(int(self.is_pyi)),
         ]


@@ -273,12 +271,6 @@ def read_pyproject_toml(
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )

-@click.option(
-    "-N",
-    "--skip-numeric-underscore-normalization",
-    is_flag=True,
-    help="Don't normalize underscores in numeric literals.",
-)

 @click.option(
     "--check",
     is_flag=True,
 @click.option(
     "--check",
     is_flag=True,


@@ -370,7 +362,6 @@ def main(
     pyi: bool,
     py36: bool,
     skip_string_normalization: bool,
     pyi: bool,
     py36: bool,
     skip_string_normalization: bool,

-    skip_numeric_underscore_normalization: bool,

     quiet: bool,
     verbose: bool,
     include: str,
     quiet: bool,
     verbose: bool,
     include: str,


@@ -396,7 +387,6 @@ def main(
         line_length=line_length,
         is_pyi=pyi,
         string_normalization=not skip_string_normalization,
         line_length=line_length,
         is_pyi=pyi,
         string_normalization=not skip_string_normalization,

-        numeric_underscore_normalization=not skip_numeric_underscore_normalization,

     )
     if config and verbose:
         out(f"Using configuration from {config}.", bold=False, fg="blue")
     )
     if config and verbose:
         out(f"Using configuration from {config}.", bold=False, fg="blue")


@@ -686,8 +676,6 @@ def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
         or supports_feature(versions, Feature.UNICODE_LITERALS),
         is_pyi=mode.is_pyi,
         normalize_strings=mode.string_normalization,
         or supports_feature(versions, Feature.UNICODE_LITERALS),
         is_pyi=mode.is_pyi,
         normalize_strings=mode.string_normalization,

-        allow_underscores=mode.numeric_underscore_normalization
-        and supports_feature(versions, Feature.NUMERIC_UNDERSCORES),

     )
     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
     empty_line = Line()
     )
     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
     empty_line = Line()


@@ -1492,7 +1480,6 @@ class LineGenerator(Visitor[Line]):
     normalize_strings: bool = True
     current_line: Line = Factory(Line)
     remove_u_prefix: bool = False
     normalize_strings: bool = True
     current_line: Line = Factory(Line)
     remove_u_prefix: bool = False

-    allow_underscores: bool = False

 
     def line(self, indent: int = 0) -> Iterator[Line]:
         """Generate a line.
 
     def line(self, indent: int = 0) -> Iterator[Line]:
         """Generate a line.


@@ -1535,7 +1522,7 @@ class LineGenerator(Visitor[Line]):
                 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
                 normalize_string_quotes(node)
             if node.type == token.NUMBER:
                 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
                 normalize_string_quotes(node)
             if node.type == token.NUMBER:

-                normalize_numeric_literal(node, self.allow_underscores)
+                normalize_numeric_literal(node)

             if node.type not in WHITESPACE:
                 self.current_line.append(node)
         yield from super().visit_default(node)
             if node.type not in WHITESPACE:
                 self.current_line.append(node)
         yield from super().visit_default(node)


@@ -2674,11 +2661,11 @@ def normalize_string_quotes(leaf: Leaf) -> None:
     leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
 
 
     leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
 
 

-def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
+def normalize_numeric_literal(leaf: Leaf) -> None:

     """Normalizes numeric (float, int, and complex) literals.
 
     All letters used in the representation are normalized to lowercase (except
     """Normalizes numeric (float, int, and complex) literals.
 
     All letters used in the representation are normalized to lowercase (except

-    in Python 2 long literals), and long number literals are split using underscores.
+    in Python 2 long literals).

     """
     text = leaf.value.lower()
     if text.startswith(("0o", "0b")):
     """
     text = leaf.value.lower()
     if text.startswith(("0o", "0b")):


@@ -2696,8 +2683,7 @@ def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
             sign = "-"
         elif after.startswith("+"):
             after = after[1:]
             sign = "-"
         elif after.startswith("+"):
             after = after[1:]

-        before = format_float_or_int_string(before, allow_underscores)
-        after = format_int_string(after, allow_underscores)
+        before = format_float_or_int_string(before)

         text = f"{before}e{sign}{after}"
     elif text.endswith(("j", "l")):
         number = text[:-1]
         text = f"{before}e{sign}{after}"
     elif text.endswith(("j", "l")):
         number = text[:-1]


@@ -2705,50 +2691,19 @@ def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
         # Capitalize in "2L" because "l" looks too similar to "1".
         if suffix == "l":
             suffix = "L"
         # Capitalize in "2L" because "l" looks too similar to "1".
         if suffix == "l":
             suffix = "L"

-        text = f"{format_float_or_int_string(number, allow_underscores)}{suffix}"
+        text = f"{format_float_or_int_string(number)}{suffix}"

     else:
     else:

-        text = format_float_or_int_string(text, allow_underscores)
+        text = format_float_or_int_string(text)

     leaf.value = text
 
 
     leaf.value = text
 
 

-def format_float_or_int_string(text: str, allow_underscores: bool) -> str:
+def format_float_or_int_string(text: str) -> str:

     """Formats a float string like "1.0"."""
     if "." not in text:
     """Formats a float string like "1.0"."""
     if "." not in text:

-        return format_int_string(text, allow_underscores)
-
-    before, after = text.split(".")
-    before = format_int_string(before, allow_underscores) if before else "0"
-    if after:
-        after = format_int_string(after, allow_underscores, count_from_end=False)
-    else:
-        after = "0"
-    return f"{before}.{after}"
-
-
-def format_int_string(
-    text: str, allow_underscores: bool, count_from_end: bool = True
-) -> str:
-    """Normalizes underscores in a string to e.g. 1_000_000.
-
-    Input must be a string of digits and optional underscores.
-    If count_from_end is False, we add underscores after groups of three digits
-    counting from the beginning instead of the end of the strings. This is used
-    for the fractional part of float literals.
-    """
-    if not allow_underscores:

         return text
 
         return text
 

-    text = text.replace("_", "")
-    if len(text) <= 5:
-        # No underscores for numbers <= 5 digits long.
-        return text
-
-    if count_from_end:
-        # Avoid removing leading zeros, which are important if we're formatting
-        # part of a number like "0.001".
-        return format(int("1" + text), "3_")[1:].lstrip("_")
-    else:
-        return "_".join(text[i : i + 3] for i in range(0, len(text), 3))
+    before, after = text.split(".")
+    return f"{before or 0}.{after or 0}"

 
 
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
 
 
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:

diff --git a/blackd.py b/blackd.py
index 2e3ce60be2e9c14668bc0bdf8402f06e7f2bbd21..bfc410d69bfaea382d41ca5d1545b3dc05fe5743 100644 (file)
--- a/blackd.py
+++ b/blackd.py
@@ -17,7 +17,6 @@ VERSION_HEADER = "X-Protocol-Version"
 LINE_LENGTH_HEADER = "X-Line-Length"
 PYTHON_VARIANT_HEADER = "X-Python-Variant"
 SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization"
 LINE_LENGTH_HEADER = "X-Line-Length"
 PYTHON_VARIANT_HEADER = "X-Python-Variant"
 SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization"

-SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER = "X-Skip-Numeric-Underscore-Normalization"

 FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe"
 
 BLACK_HEADERS = [
 FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe"
 
 BLACK_HEADERS = [


@@ -25,7 +24,6 @@ BLACK_HEADERS = [
     LINE_LENGTH_HEADER,
     PYTHON_VARIANT_HEADER,
     SKIP_STRING_NORMALIZATION_HEADER,
     LINE_LENGTH_HEADER,
     PYTHON_VARIANT_HEADER,
     SKIP_STRING_NORMALIZATION_HEADER,

-    SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER,

     FAST_OR_SAFE_HEADER,
 ]
 
     FAST_OR_SAFE_HEADER,
 ]
 


@@ -95,9 +93,6 @@ async def handle(request: web.Request, executor: Executor) -> web.Response:
         skip_string_normalization = bool(
             request.headers.get(SKIP_STRING_NORMALIZATION_HEADER, False)
         )
         skip_string_normalization = bool(
             request.headers.get(SKIP_STRING_NORMALIZATION_HEADER, False)
         )

-        skip_numeric_underscore_normalization = bool(
-            request.headers.get(SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER, False)
-        )

         fast = False
         if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast":
             fast = True
         fast = False
         if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast":
             fast = True


@@ -106,7 +101,6 @@ async def handle(request: web.Request, executor: Executor) -> web.Response:
             is_pyi=pyi,
             line_length=line_length,
             string_normalization=not skip_string_normalization,
             is_pyi=pyi,
             line_length=line_length,
             string_normalization=not skip_string_normalization,

-            numeric_underscore_normalization=not skip_numeric_underscore_normalization,

         )
         req_bytes = await request.content.read()
         charset = request.charset if request.charset is not None else "utf8"
         )
         req_bytes = await request.content.read()
         charset = request.charset if request.charset is not None else "utf8"

diff --git a/tests/data/function.py b/tests/data/function.py
index 9e2e72f3f8b62a0a9d1052bd049525d12d4189ce..4754588e38d602b4af5afe7470ac62fd1b2879cd 100644 (file)
--- a/tests/data/function.py
+++ b/tests/data/function.py
@@ -144,7 +144,7 @@ def function_signature_stress_test(
 
 
 def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r""):
 
 
 def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r""):

-    offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200_000)))
+    offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200000)))

     assert task._cancel_stack[: len(old_stack)] == old_stack
 
 
     assert task._cancel_stack[: len(old_stack)] == old_stack
 
 

diff --git a/tests/data/numeric_literals.py b/tests/data/numeric_literals.py
index 0d8e3459050eb1e31c7ba004520c26a3deea9bb0..254da68d3308bf76cdf464dbf9aa5a5a74252828 100644 (file)
--- a/tests/data/numeric_literals.py
+++ b/tests/data/numeric_literals.py
@@ -6,7 +6,7 @@ x = .1
 x = 1.
 x = 1E+1
 x = 1E-1
 x = 1.
 x = 1E+1
 x = 1E-1

-x = 1.00000001
+x = 1.000_000_01

 x = 123456789.123456789
 x = 123456789.123456789E123456789
 x = 123456789E123456789
 x = 123456789.123456789
 x = 123456789.123456789E123456789
 x = 123456789E123456789


@@ -24,21 +24,21 @@ x = 133333
 
 #!/usr/bin/env python3.6
 
 
 #!/usr/bin/env python3.6
 

-x = 123_456_789
-x = 123_456
+x = 123456789
+x = 123456

 x = 0.1
 x = 1.0
 x = 1e1
 x = 1e-1
 x = 1.000_000_01
 x = 0.1
 x = 1.0
 x = 1e1
 x = 1e-1
 x = 1.000_000_01

-x = 123_456_789.123_456_789
-x = 123_456_789.123_456_789e123_456_789
-x = 123_456_789e123_456_789
-x = 123_456_789j
-x = 123_456_789.123_456_789j
+x = 123456789.123456789
+x = 123456789.123456789e123456789
+x = 123456789e123456789
+x = 123456789j
+x = 123456789.123456789j

 x = 0xB1ACC
 x = 0b1011
 x = 0o777
 x = 0xB1ACC
 x = 0b1011
 x = 0o777

-x = 0.000_000_006
+x = 0.000000006

 x = 10000
 x = 10000

-x = 133_333
\ No newline at end of file
+x = 133333

diff --git a/tests/test_black.py b/tests/test_black.py
index 5532fc563d328d18cf012770e2d38f7567bf3b45..3404e058e1b069abc46830e41953bb99feeee169 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -437,9 +437,7 @@ class BlackTestCase(unittest.TestCase):
     @patch("black.dump_to_file", dump_to_stderr)
     def test_numeric_literals_ignoring_underscores(self) -> None:
         source, expected = read_data("numeric_literals_skip_underscores")
     @patch("black.dump_to_file", dump_to_stderr)
     def test_numeric_literals_ignoring_underscores(self) -> None:
         source, expected = read_data("numeric_literals_skip_underscores")

-        mode = black.FileMode(
-            numeric_underscore_normalization=False, target_versions=black.PY36_VERSIONS
-        )
+        mode = black.FileMode(target_versions=black.PY36_VERSIONS)

         actual = fs(source, mode=mode)
         self.assertFormatEqual(expected, actual)
         black.assert_equivalent(source, actual)
         actual = fs(source, mode=mode)
         self.assertFormatEqual(expected, actual)
         black.assert_equivalent(source, actual)


@@ -828,8 +826,7 @@ class BlackTestCase(unittest.TestCase):
         )
         node = black.lib2to3_parse(expected)
         self.assertEqual(
         )
         node = black.lib2to3_parse(expected)
         self.assertEqual(

-            black.get_features_used(node),
-            {Feature.TRAILING_COMMA, Feature.F_STRINGS, Feature.NUMERIC_UNDERSCORES},
+            black.get_features_used(node), {Feature.TRAILING_COMMA, Feature.F_STRINGS}

         )
         source, expected = read_data("expression")
         node = black.lib2to3_parse(source)
         )
         source, expected = read_data("expression")
         node = black.lib2to3_parse(source)