From 0753d99519b0c90f0f9f280b73783b537900dc16 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Thu, 11 Nov 2021 20:28:48 -0500 Subject: [PATCH] Improve Python 2 only syntax detection (GH-2592) * Improve Python 2 only syntax detection First of all this fixes a mistake I made in Python 2 deprecation PR using token.* to check for print/exec statements. Turns out that for nodes with a type value higher than 256 its numeric type isn't guaranteed to be constant. Using syms.* instead fixes this. Also add support for the following cases: print "hello, world!" exec "print('hello, world!')" def set_position((x, y), value): pass try: pass except Exception, err: pass raise RuntimeError, "I feel like crashing today :p" `wow_these_really_did_exist` 10L * Add octal support, more test cases, and fixup long ints Co-authored-by: Jelle Zijlstra Co-authored-by: Jelle Zijlstra --- CHANGES.md | 7 +++ src/black/__init__.py | 36 +++++++++++-- src/black/mode.py | 12 +++++ src/blib2to3/pgen2/token.py | 3 -- tests/data/python2_detection.py | 90 +++++++++++++++++++++++++++++++++ tests/test_black.py | 15 ++++++ 6 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 tests/data/python2_detection.py diff --git a/CHANGES.md b/CHANGES.md index b454a73..215680f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ # Change Log +## _Unreleased_ + +### _Black_ + +- Warn about Python 2 deprecation in more cases by improving Python 2 only syntax + detection (#2592) + ## 21.10b0 ### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index ba4d3de..ad4ee1a 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1132,8 +1132,17 @@ def get_features_used(node: Node) -> Set[Feature]: # noqa: C901 features.add(Feature.F_STRINGS) elif n.type == token.NUMBER: - if "_" in n.value: # type: ignore + assert isinstance(n, Leaf) + if "_" in n.value: features.add(Feature.NUMERIC_UNDERSCORES) + elif n.value.endswith(("L", "l")): + # Python 2: 10L + features.add(Feature.LONG_INT_LITERAL) + elif len(n.value) >= 2 and n.value[0] == "0" and n.value[1].isdigit(): + # Python 2: 0123; 00123; ... + if not all(char == "0" for char in n.value): + # although we don't want to match 0000 or similar + features.add(Feature.OCTAL_INT_LITERAL) elif n.type == token.SLASH: if n.parent and n.parent.type in { @@ -1171,10 +1180,31 @@ def get_features_used(node: Node) -> Set[Feature]: # noqa: C901 if argch.type in STARS: features.add(feature) - elif n.type == token.PRINT_STMT: + # Python 2 only features (for its deprecation) except for integers, see above + elif n.type == syms.print_stmt: features.add(Feature.PRINT_STMT) - elif n.type == token.EXEC_STMT: + elif n.type == syms.exec_stmt: features.add(Feature.EXEC_STMT) + elif n.type == syms.tfpdef: + # def set_position((x, y), value): + # ... + features.add(Feature.AUTOMATIC_PARAMETER_UNPACKING) + elif n.type == syms.except_clause: + # try: + # ... + # except Exception, err: + # ... + if len(n.children) >= 4: + if n.children[-2].type == token.COMMA: + features.add(Feature.COMMA_STYLE_EXCEPT) + elif n.type == syms.raise_stmt: + # raise Exception, "msg" + if len(n.children) >= 4: + if n.children[-2].type == token.COMMA: + features.add(Feature.COMMA_STYLE_RAISE) + elif n.type == token.BACKQUOTE: + # `i'm surprised this ever existed` + features.add(Feature.BACKQUOTE_REPR) return features diff --git a/src/black/mode.py b/src/black/mode.py index 374c47a..01ee336 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -44,6 +44,12 @@ class Feature(Enum): # temporary for Python 2 deprecation PRINT_STMT = 200 EXEC_STMT = 201 + AUTOMATIC_PARAMETER_UNPACKING = 202 + COMMA_STYLE_EXCEPT = 203 + COMMA_STYLE_RAISE = 204 + LONG_INT_LITERAL = 205 + OCTAL_INT_LITERAL = 206 + BACKQUOTE_REPR = 207 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = { @@ -51,6 +57,12 @@ VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = { Feature.ASYNC_IDENTIFIERS, Feature.PRINT_STMT, Feature.EXEC_STMT, + Feature.AUTOMATIC_PARAMETER_UNPACKING, + Feature.COMMA_STYLE_EXCEPT, + Feature.COMMA_STYLE_RAISE, + Feature.LONG_INT_LITERAL, + Feature.OCTAL_INT_LITERAL, + Feature.BACKQUOTE_REPR, }, TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py index 349ba80..1e0dec9 100644 --- a/src/blib2to3/pgen2/token.py +++ b/src/blib2to3/pgen2/token.py @@ -74,9 +74,6 @@ ERRORTOKEN: Final = 58 COLONEQUAL: Final = 59 N_TOKENS: Final = 60 NT_OFFSET: Final = 256 -# temporary for Python 2 deprecation -PRINT_STMT: Final = 316 -EXEC_STMT: Final = 288 # --end constants-- tok_name: Final[Dict[int, str]] = {} diff --git a/tests/data/python2_detection.py b/tests/data/python2_detection.py new file mode 100644 index 0000000..8de2bb5 --- /dev/null +++ b/tests/data/python2_detection.py @@ -0,0 +1,90 @@ +# This uses a similar construction to the decorators.py test data file FYI. + +print "hello, world!" + +### + +exec "print('hello, world!')" + +### + +def set_position((x, y), value): + pass + +### + +try: + pass +except Exception, err: + pass + +### + +raise RuntimeError, "I feel like crashing today :p" + +### + +`wow_these_really_did_exist` + +### + +10L + +### + +10l + +### + +0123 + +# output + +print("hello python three!") + +### + +exec("I'm not sure if you can use exec like this but that's not important here!") + +### + +try: + pass +except make_exception(1, 2): + pass + +### + +try: + pass +except Exception as err: + pass + +### + +raise RuntimeError(make_msg(1, 2)) + +### + +raise RuntimeError("boom!",) + +### + +def set_position(x, y, value): + pass + +### + +10 + +### + +0 + +### + +000 + +### + +0o12 \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index b96a543..7dbc380 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2017,6 +2017,7 @@ class TestFileCollection: ) +@pytest.mark.python2 @pytest.mark.parametrize("explicit", [True, False], ids=["explicit", "autodetection"]) def test_python_2_deprecation_with_target_version(explicit: bool) -> None: args = [ @@ -2032,6 +2033,20 @@ def test_python_2_deprecation_with_target_version(explicit: bool) -> None: assert "DEPRECATION: Python 2 support will be removed" in result.stderr +@pytest.mark.python2 +def test_python_2_deprecation_autodetection_extended() -> None: + # this test has a similar construction to test_get_features_used_decorator + python2, non_python2 = read_data("python2_detection") + for python2_case in python2.split("###"): + node = black.lib2to3_parse(python2_case) + assert black.detect_target_versions(node) == {TargetVersion.PY27}, python2_case + for non_python2_case in non_python2.split("###"): + node = black.lib2to3_parse(non_python2_case) + assert black.detect_target_versions(node) != { + TargetVersion.PY27 + }, non_python2_case + + with open(black.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() -- 2.39.5