Automatic detection of deprecated Python 2 forms of print and exec

Note: if those are handled, you can't use --safe because this check is using
Python 3.6+ builtin AST.

Fixes #49

diff --git a/README.md b/README.md
index 951948fe89c397100e208ad398c34a42c4955a0e..35582545e2c6fe53cc4e2ca5f95cafd3ee1fac91 100644 (file)
--- a/README.md
+++ b/README.md
@@ -275,8 +275,7 @@ python setup.py test
 
 But you can reformat Python 2 code with it, too.  *Black* is able to parse
 all of the new syntax supported on Python 3.6 but also *effectively all*
 
 But you can reformat Python 2 code with it, too.  *Black* is able to parse
 all of the new syntax supported on Python 3.6 but also *effectively all*

-the Python 2 syntax at the same time, as long as you're not using print
-statements.
+the Python 2 syntax at the same time.

 
 By making the code exclusively Python 3.6+, I'm able to focus on the
 quality of the formatting and re-use all the nice features of the new
 
 By making the code exclusively Python 3.6+, I'm able to focus on the
 quality of the formatting and re-use all the nice features of the new


@@ -309,6 +308,9 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ### 18.3a4 (unreleased)
 
 
 ### 18.3a4 (unreleased)
 

+* automatic detection of deprecated Python 2 forms of print statements
+  and exec statements in the formatted file (#49)
+

 * only return exit code 1 when --check is used (#50)
 
 * don't remove single trailing commas from square bracket indexing
 * only return exit code 1 when --check is used (#50)
 
 * don't remove single trailing commas from square bracket indexing

diff --git a/black.py b/black.py
index bb8ec2e57a074938bff931076db5f0b80dd49304..7935cdcebb8f721a4fc024106430710962b9e400 100644 (file)
--- a/black.py
+++ b/black.py
@@ -235,23 +235,36 @@ def format_str(src_contents: str, line_length: int) -> FileContent:
     return dst_contents
 
 
     return dst_contents
 
 

+GRAMMARS = [
+    pygram.python_grammar_no_print_statement_no_exec_statement,
+    pygram.python_grammar_no_print_statement,
+    pygram.python_grammar_no_exec_statement,
+    pygram.python_grammar,
+]
+
+

 def lib2to3_parse(src_txt: str) -> Node:
     """Given a string with source, return the lib2to3 Node."""
     grammar = pygram.python_grammar_no_print_statement
 def lib2to3_parse(src_txt: str) -> Node:
     """Given a string with source, return the lib2to3 Node."""
     grammar = pygram.python_grammar_no_print_statement

-    drv = driver.Driver(grammar, pytree.convert)

     if src_txt[-1] != '\n':
         nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n'
         src_txt += nl
     if src_txt[-1] != '\n':
         nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n'
         src_txt += nl

-    try:
-        result = drv.parse_string(src_txt, True)
-    except ParseError as pe:
-        lineno, column = pe.context[1]
-        lines = src_txt.splitlines()
+    for grammar in GRAMMARS:
+        drv = driver.Driver(grammar, pytree.convert)

         try:
         try:

-            faulty_line = lines[lineno - 1]
-        except IndexError:
-            faulty_line = "<line number missing in source>"
-        raise ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") from None
+            result = drv.parse_string(src_txt, True)
+            break
+
+        except ParseError as pe:
+            lineno, column = pe.context[1]
+            lines = src_txt.splitlines()
+            try:
+                faulty_line = lines[lineno - 1]
+            except IndexError:
+                faulty_line = "<line number missing in source>"
+            exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}")
+    else:
+        raise exc from None

 
     if isinstance(result, Leaf):
         result = Node(syms.file_input, [result])
 
     if isinstance(result, Leaf):
         result = Node(syms.file_input, [result])


@@ -903,6 +916,17 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
         ):
             return NO
 
         ):
             return NO
 

+        elif (
+            prevp.type == token.RIGHTSHIFT
+            and prevp.parent
+            and prevp.parent.type == syms.shift_expr
+            and prevp.prev_sibling
+            and prevp.prev_sibling.type == token.NAME
+            and prevp.prev_sibling.value == 'print'
+        ):
+            # Python 2 print chevron
+            return NO
+

     elif prev.type in OPENING_BRACKETS:
         return NO
 
     elif prev.type in OPENING_BRACKETS:
         return NO
 


@@ -1538,7 +1562,12 @@ def assert_equivalent(src: str, dst: str) -> None:
     try:
         src_ast = ast.parse(src)
     except Exception as exc:
     try:
         src_ast = ast.parse(src)
     except Exception as exc:

-        raise AssertionError(f"cannot parse source: {exc}") from None
+        major, minor = sys.version_info[:2]
+        raise AssertionError(
+            f"cannot use --safe with this file; failed to parse source file "
+            f"with Python {major}.{minor}'s builtin AST. Re-run with --fast "
+            f"or stop using deprecated Python 2 syntax. AST error message: {exc}"
+        )

 
     try:
         dst_ast = ast.parse(dst)
 
     try:
         dst_ast = ast.parse(dst)

diff --git a/blib2to3/pygram.py b/blib2to3/pygram.py
index c4ff9d15733a06787b30d40f4441596578b2c137..bf55ab4babc1f29500fd1880c65702b95207ce70 100644 (file)
--- a/blib2to3/pygram.py
+++ b/blib2to3/pygram.py
@@ -36,5 +36,12 @@ python_symbols = Symbols(python_grammar)
 python_grammar_no_print_statement = python_grammar.copy()
 del python_grammar_no_print_statement.keywords["print"]
 
 python_grammar_no_print_statement = python_grammar.copy()
 del python_grammar_no_print_statement.keywords["print"]
 

+python_grammar_no_exec_statement = python_grammar.copy()
+del python_grammar_no_exec_statement.keywords["exec"]
+
+python_grammar_no_print_statement_no_exec_statement = python_grammar.copy()
+del python_grammar_no_print_statement_no_exec_statement.keywords["print"]
+del python_grammar_no_print_statement_no_exec_statement.keywords["exec"]
+

 pattern_grammar = driver.load_packaged_grammar("blib2to3", _PATTERN_GRAMMAR_FILE)
 pattern_symbols = Symbols(pattern_grammar)
 pattern_grammar = driver.load_packaged_grammar("blib2to3", _PATTERN_GRAMMAR_FILE)
 pattern_symbols = Symbols(pattern_grammar)

diff --git a/blib2to3/pygram.pyi b/blib2to3/pygram.pyi
index 3dbc6489de31ad841068e784d6a11b409e731ed2..5f134d57ad77616dab936da5cc6563309052065c 100644 (file)
--- a/blib2to3/pygram.pyi
+++ b/blib2to3/pygram.pyi
@@ -116,4 +116,6 @@ class pattern_symbols(Symbols):
 
 python_grammar: Grammar
 python_grammar_no_print_statement: Grammar
 
 python_grammar: Grammar
 python_grammar_no_print_statement: Grammar

+python_grammar_no_print_statement_no_exec_statement: Grammar
+python_grammar_no_exec_statement: Grammar

 pattern_grammar: Grammar
 pattern_grammar: Grammar

diff --git a/tests/function.py b/tests/function.py
index 7fa6866bdd70fb3e8297a1ab77ec9d36689a57c5..888ef9fc5096b680ba0b18dd5632ded669bf3fae 100644 (file)
--- a/tests/function.py
+++ b/tests/function.py
@@ -14,8 +14,9 @@ def func_no_args():
   for i in range(10):
     print(i)
     continue
   for i in range(10):
     print(i)
     continue

+  exec("new-style exec", {}, {})

   return None
   return None

-async def coroutine(arg):
+async def coroutine(arg, exec=False):

  "Single-line docstring. Multiline is harder to reformat."
  async with some_connection() as conn:
      await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)
  "Single-line docstring. Multiline is harder to reformat."
  async with some_connection() as conn:
      await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)


@@ -93,10 +94,11 @@ def func_no_args():
         print(i)
         continue
 
         print(i)
         continue
 

+    exec("new-style exec", {}, {})

     return None
 
 
     return None
 
 

-async def coroutine(arg):
+async def coroutine(arg, exec=False):

     "Single-line docstring. Multiline is harder to reformat."
     async with some_connection() as conn:
         await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)
     "Single-line docstring. Multiline is harder to reformat."
     async with some_connection() as conn:
         await conn.do_what_i_mean('SELECT bobby, tables FROM xkcd', timeout=2)

diff --git a/tests/python2.py b/tests/python2.py
new file mode 100644 (file)
index 0000000..5214add
--- /dev/null
+++ b/tests/python2.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python2
+
+import sys
+
+print >> sys.stderr , "Warning:" ,
+print >> sys.stderr , "this is a blast from the past."
+print >> sys.stderr , "Look, a repr:", `sys`
+
+
+def function((_globals, _locals)):
+    exec "print 'hi from exec!'" in _globals, _locals
+
+
+function((globals(), locals()))
+
+
+# output
+
+
+#!/usr/bin/env python2
+
+import sys
+
+print >>sys.stderr, "Warning:",
+print >>sys.stderr, "this is a blast from the past."
+print >>sys.stderr, "Look, a repr:", ` sys `
+
+
+def function((_globals, _locals)):
+    exec "print 'hi from exec!'" in _globals, _locals
+
+
+function((globals(), locals()))

diff --git a/tests/test_black.py b/tests/test_black.py
index 69746d12f0c117def22796f4050eef55d47304f7..34155495371c759ae2976dab0937990fb4af01f8 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -180,6 +180,14 @@ class BlackTestCase(unittest.TestCase):
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, line_length=ll)
 
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, line_length=ll)
 

+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_python2(self) -> None:
+        source, expected = read_data('python2')
+        actual = fs(source)
+        self.assertFormatEqual(expected, actual)
+        # black.assert_equivalent(source, actual)
+        black.assert_stable(source, actual, line_length=ll)
+

     def test_report(self) -> None:
         report = black.Report()
         out_lines = []
     def test_report(self) -> None:
         report = black.Report()
         out_lines = []