Support parsing of async generators in non-async functions (#165)

This is a new syntax added in python3.7, so black can't verify that reformatting will not change the ast unless black itself is run with 3.7. We'll need to change the error message black gives in this case. @ambv any ideas?

Fixes #125.

diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 9a7664bbf95bf81d1809922dcaf53433bae61454..1f51ff0039fd0227df0a5d7f691bbcd684fea795 100644 (file)
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -516,13 +516,14 @@ def generate_tokens(readline):
                         stashed = tok
                         continue
 
-                    if token == 'def':
+                    if token in ('def', 'for'):
                         if (stashed
                                 and stashed[0] == NAME
                                 and stashed[1] == 'async'):
 
-                            async_def = True
-                            async_def_indent = indents[-1]
+                            if token == 'def':
+                                async_def = True
+                                async_def_indent = indents[-1]
 
                             yield (ASYNC, stashed[1],
                                    stashed[2], stashed[3],

diff --git a/tests/data/python37.py b/tests/data/python37.py
new file mode 100644 (file)
index 0000000..ae20ade
--- /dev/null
+++ b/tests/data/python37.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3.7
+
+def f():
+    return (i*2 async for i in arange(42))
+
+# output
+
+
+#!/usr/bin/env python3.7
+
+
+def f():
+    return (i * 2 async for i in arange(42))

diff --git a/tests/test_black.py b/tests/test_black.py
index 10d7f28990d436beaaf5b5faa7f2ce66e6dd0bbb..9c798caa1c3169b6792f8d663ff41a97df345e10 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -411,6 +411,16 @@ class BlackTestCase(unittest.TestCase):
         self.assertFormatEqual(expected, actual)
         black.assert_stable(source, actual, line_length=ll, mode=mode)
 
+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_python37(self) -> None:
+        source, expected = read_data("python37")
+        actual = fs(source)
+        self.assertFormatEqual(expected, actual)
+        major, minor = sys.version_info[:2]
+        if major > 3 or (major == 3 and minor >= 7):
+            black.assert_equivalent(source, actual)
+        black.assert_stable(source, actual, line_length=ll)
+
     @patch("black.dump_to_file", dump_to_stderr)
     def test_fmtonoff(self) -> None:
         source, expected = read_data("fmtonoff")