Improve Python 2 only syntax detection (GH-2592)

* Improve Python 2 only syntax detection

First of all this fixes a mistake I made in Python 2 deprecation PR
using token.* to check for print/exec statements. Turns out that
for nodes with a type value higher than 256 its numeric type isn't
guaranteed to be constant. Using syms.* instead fixes this.

Also add support for the following cases:

    print "hello, world!"

    exec "print('hello, world!')"

    def set_position((x, y), value):
        pass

    try:
        pass
    except Exception, err:
        pass

    raise RuntimeError, "I feel like crashing today :p"

    `wow_these_really_did_exist`

    10L

* Add octal support, more test cases, and fixup long ints

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

diff --git a/CHANGES.md b/CHANGES.md
index b454a73edab3c33c548be2d5c2fb86dde4493653..215680ff2ee0550af60979ea409f150f82fc3bd1 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,12 @@
 # Change Log
 
 # Change Log
 

+## _Unreleased_
+
+### _Black_
+
+- Warn about Python 2 deprecation in more cases by improving Python 2 only syntax
+  detection (#2592)
+

 ## 21.10b0
 
 ### _Black_
 ## 21.10b0
 
 ### _Black_

diff --git a/src/black/__init__.py b/src/black/__init__.py
index ba4d3dea70ea862b0f19390ed40e0fd9ed94800a..ad4ee1a0d1a6dff08825d6580f9d9cf94793c0ff 100644 (file)
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -1132,8 +1132,17 @@ def get_features_used(node: Node) -> Set[Feature]:  # noqa: C901
                 features.add(Feature.F_STRINGS)
 
         elif n.type == token.NUMBER:
                 features.add(Feature.F_STRINGS)
 
         elif n.type == token.NUMBER:

-            if "_" in n.value:  # type: ignore
+            assert isinstance(n, Leaf)
+            if "_" in n.value:

                 features.add(Feature.NUMERIC_UNDERSCORES)
                 features.add(Feature.NUMERIC_UNDERSCORES)

+            elif n.value.endswith(("L", "l")):
+                # Python 2: 10L
+                features.add(Feature.LONG_INT_LITERAL)
+            elif len(n.value) >= 2 and n.value[0] == "0" and n.value[1].isdigit():
+                # Python 2: 0123; 00123; ...
+                if not all(char == "0" for char in n.value):
+                    # although we don't want to match 0000 or similar
+                    features.add(Feature.OCTAL_INT_LITERAL)

 
         elif n.type == token.SLASH:
             if n.parent and n.parent.type in {
 
         elif n.type == token.SLASH:
             if n.parent and n.parent.type in {


@@ -1171,10 +1180,31 @@ def get_features_used(node: Node) -> Set[Feature]:  # noqa: C901
                         if argch.type in STARS:
                             features.add(feature)
 
                         if argch.type in STARS:
                             features.add(feature)
 

-        elif n.type == token.PRINT_STMT:
+        # Python 2 only features (for its deprecation) except for integers, see above
+        elif n.type == syms.print_stmt:

             features.add(Feature.PRINT_STMT)
             features.add(Feature.PRINT_STMT)

-        elif n.type == token.EXEC_STMT:
+        elif n.type == syms.exec_stmt:

             features.add(Feature.EXEC_STMT)
             features.add(Feature.EXEC_STMT)

+        elif n.type == syms.tfpdef:
+            # def set_position((x, y), value):
+            #     ...
+            features.add(Feature.AUTOMATIC_PARAMETER_UNPACKING)
+        elif n.type == syms.except_clause:
+            # try:
+            #     ...
+            # except Exception, err:
+            #     ...
+            if len(n.children) >= 4:
+                if n.children[-2].type == token.COMMA:
+                    features.add(Feature.COMMA_STYLE_EXCEPT)
+        elif n.type == syms.raise_stmt:
+            # raise Exception, "msg"
+            if len(n.children) >= 4:
+                if n.children[-2].type == token.COMMA:
+                    features.add(Feature.COMMA_STYLE_RAISE)
+        elif n.type == token.BACKQUOTE:
+            # `i'm surprised this ever existed`
+            features.add(Feature.BACKQUOTE_REPR)

 
     return features
 
 
     return features
 

diff --git a/src/black/mode.py b/src/black/mode.py
index 374c47a42eb0a661e28eb097dccb17e102582a3a..01ee336366c8e113d3e5cb0d2eb397c6db0c2760 100644 (file)
--- a/src/black/mode.py
+++ b/src/black/mode.py
@@ -44,6 +44,12 @@ class Feature(Enum):
     # temporary for Python 2 deprecation
     PRINT_STMT = 200
     EXEC_STMT = 201
     # temporary for Python 2 deprecation
     PRINT_STMT = 200
     EXEC_STMT = 201

+    AUTOMATIC_PARAMETER_UNPACKING = 202
+    COMMA_STYLE_EXCEPT = 203
+    COMMA_STYLE_RAISE = 204
+    LONG_INT_LITERAL = 205
+    OCTAL_INT_LITERAL = 206
+    BACKQUOTE_REPR = 207

 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {


@@ -51,6 +57,12 @@ VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
         Feature.ASYNC_IDENTIFIERS,
         Feature.PRINT_STMT,
         Feature.EXEC_STMT,
         Feature.ASYNC_IDENTIFIERS,
         Feature.PRINT_STMT,
         Feature.EXEC_STMT,

+        Feature.AUTOMATIC_PARAMETER_UNPACKING,
+        Feature.COMMA_STYLE_EXCEPT,
+        Feature.COMMA_STYLE_RAISE,
+        Feature.LONG_INT_LITERAL,
+        Feature.OCTAL_INT_LITERAL,
+        Feature.BACKQUOTE_REPR,

     },
     TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
     TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
     },
     TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
     TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},

diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py
index 349ba8023a27110bba09f145f763157909995351..1e0dec9c714d9c5cf518e863e1087761f77a5253 100644 (file)
--- a/src/blib2to3/pgen2/token.py
+++ b/src/blib2to3/pgen2/token.py
@@ -74,9 +74,6 @@ ERRORTOKEN: Final = 58
 COLONEQUAL: Final = 59
 N_TOKENS: Final = 60
 NT_OFFSET: Final = 256
 COLONEQUAL: Final = 59
 N_TOKENS: Final = 60
 NT_OFFSET: Final = 256

-# temporary for Python 2 deprecation
-PRINT_STMT: Final = 316
-EXEC_STMT: Final = 288

 # --end constants--
 
 tok_name: Final[Dict[int, str]] = {}
 # --end constants--
 
 tok_name: Final[Dict[int, str]] = {}

diff --git a/tests/data/python2_detection.py b/tests/data/python2_detection.py
new file mode 100644 (file)
index 0000000..8de2bb5
--- /dev/null
+++ b/tests/data/python2_detection.py
@@ -0,0 +1,90 @@
+# This uses a similar construction to the decorators.py test data file FYI.
+
+print "hello, world!"
+
+###
+
+exec "print('hello, world!')"
+
+###
+
+def set_position((x, y), value):
+    pass
+
+###
+
+try:
+    pass
+except Exception, err:
+    pass
+
+###
+
+raise RuntimeError, "I feel like crashing today :p"
+
+###
+
+`wow_these_really_did_exist`
+
+###
+
+10L
+
+###
+
+10l
+
+###
+
+0123
+
+# output
+
+print("hello python three!")
+
+###
+
+exec("I'm not sure if you can use exec like this but that's not important here!")
+
+###
+
+try:
+    pass
+except make_exception(1, 2):
+    pass
+
+###
+
+try:
+    pass
+except Exception as err:
+    pass
+
+###
+
+raise RuntimeError(make_msg(1, 2))
+
+###
+
+raise RuntimeError("boom!",)
+
+###
+
+def set_position(x, y, value):
+    pass
+
+###
+
+10
+
+###
+
+0
+
+###
+
+000
+
+###
+
+0o12
\ No newline at end of file

diff --git a/tests/test_black.py b/tests/test_black.py
index b96a54385576ba9e458629df3a5ee7b6cfc89200..7dbc3809d26c0bd53523202224ee6b7037df63a5 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -2017,6 +2017,7 @@ class TestFileCollection:
         )
 
 
         )
 
 

+@pytest.mark.python2

 @pytest.mark.parametrize("explicit", [True, False], ids=["explicit", "autodetection"])
 def test_python_2_deprecation_with_target_version(explicit: bool) -> None:
     args = [
 @pytest.mark.parametrize("explicit", [True, False], ids=["explicit", "autodetection"])
 def test_python_2_deprecation_with_target_version(explicit: bool) -> None:
     args = [


@@ -2032,6 +2033,20 @@ def test_python_2_deprecation_with_target_version(explicit: bool) -> None:
     assert "DEPRECATION: Python 2 support will be removed" in result.stderr
 
 
     assert "DEPRECATION: Python 2 support will be removed" in result.stderr
 
 

+@pytest.mark.python2
+def test_python_2_deprecation_autodetection_extended() -> None:
+    # this test has a similar construction to test_get_features_used_decorator
+    python2, non_python2 = read_data("python2_detection")
+    for python2_case in python2.split("###"):
+        node = black.lib2to3_parse(python2_case)
+        assert black.detect_target_versions(node) == {TargetVersion.PY27}, python2_case
+    for non_python2_case in non_python2.split("###"):
+        node = black.lib2to3_parse(non_python2_case)
+        assert black.detect_target_versions(node) != {
+            TargetVersion.PY27
+        }, non_python2_case
+
+

 with open(black.__file__, "r", encoding="utf-8") as _bf:
     black_source_lines = _bf.readlines()
 
 with open(black.__file__, "r", encoding="utf-8") as _bf:
     black_source_lines = _bf.readlines()