]>
git.madduck.net Git - etc/vim.git/blobdiff - blib2to3/pgen2/driver.py
madduck's git repository
Every one of the projects in this repository is available at the canonical
URL git://git.madduck.net/madduck/pub/<projectpath> — see
each project's metadata for the exact URL.
All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@ git. madduck. net .
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
SSH access, as well as push access can be individually
arranged .
If you use my repositories frequently, consider adding the following
snippet to ~/.gitconfig and using the third clone URL listed for each
project:
[url "git://git.madduck.net/madduck/"]
insteadOf = madduck:
- def __init__(self, grammar, convert=None, logger=None):
+ def __init__(
+ self,
+ grammar,
+ convert=None,
+ logger=None,
+ ):
self.grammar = grammar
if logger is None:
self.grammar = grammar
if logger is None:
- logger = logging.getLogger()
+ logger = logging.getLogger(__name__ )
self.logger = logger
self.convert = convert
self.logger = logger
self.convert = convert
if debug:
self.logger.debug("%s %r (prefix=%r)",
token.tok_name[type], value, prefix)
if debug:
self.logger.debug("%s %r (prefix=%r)",
token.tok_name[type], value, prefix)
- if type in {token.INDENT, token.DEDENT}:
- _prefix = prefix
+ if type == token.INDENT:
+ indent_columns.append(len(value))
+ _prefix = prefix + value
- if type == token.DEDENT:
+ value = ""
+ elif type == token.DEDENT:
_indent_col = indent_columns.pop()
_indent_col = indent_columns.pop()
- prefix, _prefix = self._partially_consume_prefix(_ prefix, _indent_col)
+ prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col)
if p.addtoken(type, value, (prefix, start)):
if debug:
self.logger.debug("Stop.")
break
prefix = ""
if p.addtoken(type, value, (prefix, start)):
if debug:
self.logger.debug("Stop.")
break
prefix = ""
- if type == token.INDENT:
- indent_columns.append(len(value))
- if _prefix.startswith(value):
- # Don't double-indent. Since we're delaying the prefix that
- # would normally belong to INDENT, we need to put the value
- # at the end versus at the beginning.
- _prefix = _prefix[len(value):] + value
if type in {token.INDENT, token.DEDENT}:
prefix = _prefix
lineno, column = end
if type in {token.INDENT, token.DEDENT}:
prefix = _prefix
lineno, column = end
def parse_stream_raw(self, stream, debug=False):
"""Parse a stream and return the syntax tree."""
def parse_stream_raw(self, stream, debug=False):
"""Parse a stream and return the syntax tree."""
- tokens = tokenize.generate_tokens(stream.readline)
+ tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar )
return self.parse_tokens(tokens, debug)
def parse_stream(self, stream, debug=False):
return self.parse_tokens(tokens, debug)
def parse_stream(self, stream, debug=False):
def parse_string(self, text, debug=False):
"""Parse a string and return the syntax tree."""
def parse_string(self, text, debug=False):
"""Parse a string and return the syntax tree."""
- tokens = tokenize.generate_tokens(io.StringIO(text).readline)
+ tokens = tokenize.generate_tokens(
+ io.StringIO(text).readline,
+ grammar=self.grammar
+ )
return self.parse_tokens(tokens, debug)
def _partially_consume_prefix(self, prefix, column):
return self.parse_tokens(tokens, debug)
def _partially_consume_prefix(self, prefix, column):
current_line = ""
current_column = 0
wait_for_nl = False
current_line = ""
current_column = 0
wait_for_nl = False
- elif char == '\t':
- current_column += 4
- # e nexpected empty line
+ # u nexpected empty line
current_column = 0
else:
# indent is finished
current_column = 0
else:
# indent is finished
return ''.join(lines), current_line
return ''.join(lines), current_line
-def _generate_pickle_name(gt):
+def _generate_pickle_name(gt, cache_dir=None ):
head, tail = os.path.splitext(gt)
if tail == ".txt":
tail = ""
head, tail = os.path.splitext(gt)
if tail == ".txt":
tail = ""
- return head + tail + ".".join(map(str, sys.version_info)) + ".pickle"
+ name = head + tail + ".".join(map(str, sys.version_info)) + ".pickle"
+ if cache_dir:
+ return os.path.join(cache_dir, os.path.basename(name))
+ else:
+ return name
def load_grammar(gt="Grammar.txt", gp=None,
save=True, force=False, logger=None):
"""Load the grammar (maybe from a pickle)."""
if logger is None:
def load_grammar(gt="Grammar.txt", gp=None,
save=True, force=False, logger=None):
"""Load the grammar (maybe from a pickle)."""
if logger is None:
- logger = logging.getLogger()
+ logger = logging.getLogger(__name__ )
gp = _generate_pickle_name(gt) if gp is None else gp
if force or not _newer(gp, gt):
logger.info("Generating grammar tables from %s", gt)
gp = _generate_pickle_name(gt) if gp is None else gp
if force or not _newer(gp, gt):
logger.info("Generating grammar tables from %s", gt)
return os.path.getmtime(a) >= os.path.getmtime(b)
return os.path.getmtime(a) >= os.path.getmtime(b)
-def load_packaged_grammar(package, grammar_source):
+def load_packaged_grammar(package, grammar_source, cache_dir=None ):
"""Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
"""Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
"""
if os.path.isfile(grammar_source):
"""
if os.path.isfile(grammar_source):
- return load_grammar(grammar_source)
- pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
+ gp = _generate_pickle_name(grammar_source, cache_dir) if cache_dir else None
+ return load_grammar(grammar_source, gp=gp)
+ pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir)
data = pkgutil.get_data(package, pickled_name)
g = grammar.Grammar()
g.loads(data)
data = pkgutil.get_data(package, pickled_name)
g = grammar.Grammar()
g.loads(data)
projects / etc / vim.git / blobdiff