Move INDENT value to the postponed prefix

This makes blib2to3's tree output valid again (which was broken by the previous
fiddling with INDENT and DEDENT nodes).

Fixes #334

diff --git a/README.md b/README.md
index 716a14edfb07921586998d711c9d35b5fc8cb104..39510390acfcb07523ef3551c1c2152dbc071cbb 100644 (file)
--- a/README.md
+++ b/README.md
@@ -830,6 +830,9 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
   * they now correctly work across function/class boundaries (#335)
 
 
   * they now correctly work across function/class boundaries (#335)
 

+  * they now work when an indentation block starts with empty lines or misaligned
+    comments (#334)
+

 * fixed improper formatting of f-strings with quotes inside interpolated
   expressions (#322)
 
 * fixed improper formatting of f-strings with quotes inside interpolated
   expressions (#322)
 

diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index af7ca1799ca8dc733871755115c5d4ad89c5692a..a51ffc30b9fe9195c1ff936d403387e7fe8b4110 100644 (file)
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -70,24 +70,19 @@ class Driver(object):
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)

-            if type in {token.INDENT, token.DEDENT}:
-                _prefix = prefix
+            if type == token.INDENT:
+                indent_columns.append(len(value))
+                _prefix = prefix + value

                 prefix = ""
                 prefix = ""

-            if type == token.DEDENT:
+                value = ""
+            elif type == token.DEDENT:

                 _indent_col = indent_columns.pop()
                 _indent_col = indent_columns.pop()

-                prefix, _prefix = self._partially_consume_prefix(_prefix, _indent_col)
+                prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col)

             if p.addtoken(type, value, (prefix, start)):
                 if debug:
                     self.logger.debug("Stop.")
                 break
             prefix = ""
             if p.addtoken(type, value, (prefix, start)):
                 if debug:
                     self.logger.debug("Stop.")
                 break
             prefix = ""

-            if type == token.INDENT:
-                indent_columns.append(len(value))
-                if _prefix.startswith(value):
-                    # Don't double-indent.  Since we're delaying the prefix that
-                    # would normally belong to INDENT, we need to put the value
-                    # at the end versus at the beginning.
-                    _prefix = _prefix[len(value):] + value

             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end
             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end

diff --git a/tests/data/debug_visitor.out b/tests/data/debug_visitor.out
index 45027243bc19851ced34d417b63fe595df8e8e44..fa60010d421da59e26337be14285411fee73b8a7 100644 (file)
--- a/tests/data/debug_visitor.out
+++ b/tests/data/debug_visitor.out
@@ -36,10 +36,11 @@ file_input
         NEWLINE
  '\n'
         INDENT
         NEWLINE
  '\n'
         INDENT

- '    '
+ ''

         simple_stmt
           expr_stmt
             NAME
         simple_stmt
           expr_stmt
             NAME

+ '    '

  'tree_depth'
             annassign
               COLON
  'tree_depth'
             annassign
               COLON


@@ -109,10 +110,11 @@ file_input
             NEWLINE
  '\n'
             INDENT
             NEWLINE
  '\n'
             INDENT

- '        '
+ ''

             simple_stmt
               expr_stmt
                 NAME
             simple_stmt
               expr_stmt
                 NAME

+ '        '

  'indent'
                 EQUAL
  ' '
  'indent'
                 EQUAL
  ' '


@@ -184,10 +186,11 @@ file_input
                 NEWLINE
  '\n'
                 INDENT
                 NEWLINE
  '\n'
                 INDENT

- '            '
+ ''

                 simple_stmt
                   expr_stmt
                     NAME
                 simple_stmt
                   expr_stmt
                     NAME

+ '            '

  '_type'
                     EQUAL
  ' '
  '_type'
                     EQUAL
  ' '


@@ -297,10 +300,11 @@ file_input
                     NEWLINE
  '\n'
                     INDENT
                     NEWLINE
  '\n'
                     INDENT

- '                '
+ ''

                     simple_stmt
                       yield_expr
                         NAME
                     simple_stmt
                       yield_expr
                         NAME

+ '                '

  'yield'
                         yield_arg
                           NAME
  'yield'
                         yield_arg
                           NAME


@@ -410,10 +414,11 @@ file_input
                 NEWLINE
  '\n'
                 INDENT
                 NEWLINE
  '\n'
                 INDENT

- '            '
+ ''

                 simple_stmt
                   expr_stmt
                     NAME
                 simple_stmt
                   expr_stmt
                     NAME

+ '            '

  '_type'
                     EQUAL
  ' '
  '_type'
                     EQUAL
  ' '


@@ -542,11 +547,11 @@ file_input
                     NEWLINE
  '\n'
                     INDENT
                     NEWLINE
  '\n'
                     INDENT

- '                '
+ ''

                     simple_stmt
                       power
                         NAME
                     simple_stmt
                       power
                         NAME

- "# We don't have to handle prefixes for `Node` objects since\n                # that delegates to the first child anyway.\n                "
+ "                # We don't have to handle prefixes for `Node` objects since\n                # that delegates to the first child anyway.\n                "

  'out'
                         trailer
                           LPAR
  'out'
                         trailer
                           LPAR


@@ -699,9 +704,10 @@ file_input
               NEWLINE
  '\n'
               INDENT
               NEWLINE
  '\n'
               INDENT

- '        '
+ ''

               simple_stmt
                 STRING
               simple_stmt
                 STRING

+ '        '

  '"""Pretty-prints a given string of `code`.\n\n        Convenience method for debugging.\n        """'
                 NEWLINE
  '\n'
  '"""Pretty-prints a given string of `code`.\n\n        Convenience method for debugging.\n        """'
                 NEWLINE
  '\n'

diff --git a/tests/data/fmtonoff2.py b/tests/data/fmtonoff2.py
index 43095060a2417a1fccce55942dd1a1221e4ac288..e8657c749b3e69e909b51978d22f443e617e5122 100644 (file)
--- a/tests/data/fmtonoff2.py
+++ b/tests/data/fmtonoff2.py
@@ -19,8 +19,17 @@ def test_fader(test):
     pass
 
 def check_fader(test):
     pass
 
 def check_fader(test):

+
+    pass
+
+def verify_fader(test):
+  # misaligned comment

     pass
 
     pass
 

+def verify_fader(test):
+    """Hey, ho."""
+    assert test.passed()
+

 def test_calculate_fades():
     calcs = [
         # one is zero/none
 def test_calculate_fades():
     calcs = [
         # one is zero/none