X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/c891c65b6b2de7c257244cc409276b485130c275..3bfb66971f03da39ae1f4c98c30d55e60f63d33b:/blib2to3/pgen2/driver.py?ds=inline diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py index af7ca17..6452c57 100644 --- a/blib2to3/pgen2/driver.py +++ b/blib2to3/pgen2/driver.py @@ -29,10 +29,15 @@ from . import grammar, parse, token, tokenize, pgen class Driver(object): - def __init__(self, grammar, convert=None, logger=None): + def __init__( + self, + grammar, + convert=None, + logger=None, + ): self.grammar = grammar if logger is None: - logger = logging.getLogger() + logger = logging.getLogger(__name__) self.logger = logger self.convert = convert @@ -70,24 +75,19 @@ class Driver(object): if debug: self.logger.debug("%s %r (prefix=%r)", token.tok_name[type], value, prefix) - if type in {token.INDENT, token.DEDENT}: - _prefix = prefix + if type == token.INDENT: + indent_columns.append(len(value)) + _prefix = prefix + value prefix = "" - if type == token.DEDENT: + value = "" + elif type == token.DEDENT: _indent_col = indent_columns.pop() - prefix, _prefix = self._partially_consume_prefix(_prefix, _indent_col) + prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col) if p.addtoken(type, value, (prefix, start)): if debug: self.logger.debug("Stop.") break prefix = "" - if type == token.INDENT: - indent_columns.append(len(value)) - if _prefix.startswith(value): - # Don't double-indent. Since we're delaying the prefix that - # would normally belong to INDENT, we need to put the value - # at the end versus at the beginning. - _prefix = _prefix[len(value):] + value if type in {token.INDENT, token.DEDENT}: prefix = _prefix lineno, column = end @@ -102,7 +102,7 @@ class Driver(object): def parse_stream_raw(self, stream, debug=False): """Parse a stream and return the syntax tree.""" - tokens = tokenize.generate_tokens(stream.readline) + tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar) return self.parse_tokens(tokens, debug) def parse_stream(self, stream, debug=False): @@ -116,7 +116,10 @@ class Driver(object): def parse_string(self, text, debug=False): """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens(io.StringIO(text).readline) + tokens = tokenize.generate_tokens( + io.StringIO(text).readline, + grammar=self.grammar + ) return self.parse_tokens(tokens, debug) def _partially_consume_prefix(self, prefix, column): @@ -136,12 +139,10 @@ class Driver(object): current_line = "" current_column = 0 wait_for_nl = False - elif char == ' ': + elif char in ' \t': current_column += 1 - elif char == '\t': - current_column += 4 elif char == '\n': - # enexpected empty line + # unexpected empty line current_column = 0 else: # indent is finished @@ -164,7 +165,7 @@ def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None): """Load the grammar (maybe from a pickle).""" if logger is None: - logger = logging.getLogger() + logger = logging.getLogger(__name__) gp = _generate_pickle_name(gt) if gp is None else gp if force or not _newer(gp, gt): logger.info("Generating grammar tables from %s", gt)