FM / lib2to3

/usr/lib64/python2.6/lib2to3

Name	Size	Mode	Actions
fixes/	-	0755	rm
pgen2/	-	0755	rm
fixer_base.py	6377	0644	edit dl rm
fixer_base.pyc	7135	0644	edit dl rm
fixer_base.pyo	7135	0644	edit dl rm
fixer_util.py	14225	0644	edit dl rm
fixer_util.pyc	14387	0644	edit dl rm
fixer_util.pyo	14281	0644	edit dl rm
Grammar.txt	6536	0644	edit dl rm
Grammar2.6.6.final.0.pickle	20214	0644	edit dl rm
main.py	6868	0644	edit dl rm
main.pyc	6629	0644	edit dl rm
main.pyo	6585	0644	edit dl rm
patcomp.py	6967	0644	edit dl rm
patcomp.pyc	6737	0644	edit dl rm
patcomp.pyo	6410	0644	edit dl rm
PatternGrammar.txt	793	0644	edit dl rm
PatternGrammar2.6.6.final.0.pickle	1383	0644	edit dl rm
pygram.py	890	0644	edit dl rm
pygram.pyc	1259	0644	edit dl rm
pygram.pyo	1259	0644	edit dl rm
pytree.py	28107	0644	edit dl rm
pytree.pyc	29776	0644	edit dl rm
pytree.pyo	28902	0644	edit dl rm
refactor.py	24179	0644	edit dl rm
refactor.pyc	22689	0644	edit dl rm
refactor.pyo	22648	0644	edit dl rm
__init__.py	7	0644	edit dl rm
__init__.pyc	127	0644	edit dl rm
__init__.pyo	127	0644	edit dl rm

Edit: /usr/lib64/python2.6/lib2to3/patcomp.py (6967B)

# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.

"""Pattern compiler.

The grammer is taken from PatternGrammar.txt.

The compiler compiles a pattern to a pytree.*Pattern instance.
"""

__author__ = "Guido van Rossum <guido@python.org>"

# Python imports
import os

# Fairly local imports
from .pgen2 import driver, literals, token, tokenize, parse, grammar

# Really local imports
from . import pytree
from . import pygram

# The pattern grammar file
_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
                                     "PatternGrammar.txt")

class PatternSyntaxError(Exception):
    pass

def tokenize_wrapper(input):
    """Tokenizes a string suppressing significant whitespace."""
    skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
    tokens = tokenize.generate_tokens(driver.generate_lines(input).next)
    for quintuple in tokens:
        type, value, start, end, line_text = quintuple
        if type not in skip:
            yield quintuple

class PatternCompiler(object):

def __init__(self, grammar_file=_PATTERN_GRAMMAR_FILE):
        """Initializer.

Takes an optional alternative filename for the pattern grammar.
        """
        self.grammar = driver.load_grammar(grammar_file)
        self.syms = pygram.Symbols(self.grammar)
        self.pygrammar = pygram.python_grammar
        self.pysyms = pygram.python_symbols
        self.driver = driver.Driver(self.grammar, convert=pattern_convert)

def compile_pattern(self, input, debug=False):
        """Compiles a pattern string to a nested pytree.*Pattern object."""
        tokens = tokenize_wrapper(input)
        try:
            root = self.driver.parse_tokens(tokens, debug=debug)
        except parse.ParseError, e:
            raise PatternSyntaxError(str(e))
        return self.compile_node(root)

def compile_node(self, node):
        """Compiles a node, recursively.

This is one big switch on the node type.
        """
        # XXX Optimize certain Wildcard-containing-Wildcard patterns
        # that can be merged
        if node.type == self.syms.Matcher:
            node = node.children[0] # Avoid unneeded recursion

if node.type == self.syms.Alternatives:
            # Skip the odd children since they are just '|' tokens
            alts = [self.compile_node(ch) for ch in node.children[::2]]
            if len(alts) == 1:
                return alts[0]
            p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1)
            return p.optimize()

if node.type == self.syms.Alternative:
            units = [self.compile_node(ch) for ch in node.children]
            if len(units) == 1:
                return units[0]
            p = pytree.WildcardPattern([units], min=1, max=1)
            return p.optimize()

if node.type == self.syms.NegatedUnit:
            pattern = self.compile_basic(node.children[1:])
            p = pytree.NegatedPattern(pattern)
            return p.optimize()

assert node.type == self.syms.Unit

name = None
        nodes = node.children
        if len(nodes) >= 3 and nodes[1].type == token.EQUAL:
            name = nodes[0].value
            nodes = nodes[2:]
        repeat = None
        if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater:
            repeat = nodes[-1]
            nodes = nodes[:-1]

# Now we've reduced it to: STRING | NAME [Details] | (...) | [...]
        pattern = self.compile_basic(nodes, repeat)

if repeat is not None:
            assert repeat.type == self.syms.Repeater
            children = repeat.children
            child = children[0]
            if child.type == token.STAR:
                min = 0
                max = pytree.HUGE
            elif child.type == token.PLUS:
                min = 1
                max = pytree.HUGE
            elif child.type == token.LBRACE:
                assert children[-1].type == token.RBRACE
                assert  len(children) in (3, 5)
                min = max = self.get_int(children[1])
                if len(children) == 5:
                    max = self.get_int(children[3])
            else:
                assert False
            if min != 1 or max != 1:
                pattern = pattern.optimize()
                pattern = pytree.WildcardPattern([[pattern]], min=min, max=max)

if name is not None:
            pattern.name = name
        return pattern.optimize()

def compile_basic(self, nodes, repeat=None):
        # Compile STRING | NAME [Details] | (...) | [...]
        assert len(nodes) >= 1
        node = nodes[0]
        if node.type == token.STRING:
            value = unicode(literals.evalString(node.value))
            return pytree.LeafPattern(_type_of_literal(value), value)
        elif node.type == token.NAME:
            value = node.value
            if value.isupper():
                if value not in TOKEN_MAP:
                    raise PatternSyntaxError("Invalid token: %r" % value)
                if nodes[1:]:
                    raise PatternSyntaxError("Can't have details for token")
                return pytree.LeafPattern(TOKEN_MAP[value])
            else:
                if value == "any":
                    type = None
                elif not value.startswith("_"):
                    type = getattr(self.pysyms, value, None)
                    if type is None:
                        raise PatternSyntaxError("Invalid symbol: %r" % value)
                if nodes[1:]: # Details present
                    content = [self.compile_node(nodes[1].children[1])]
                else:
                    content = None
                return pytree.NodePattern(type, content)
        elif node.value == "(":
            return self.compile_node(nodes[1])
        elif node.value == "[":
            assert repeat is None
            subpattern = self.compile_node(nodes[1])
            return pytree.WildcardPattern([[subpattern]], min=0, max=1)
        assert False, node

def get_int(self, node):
        assert node.type == token.NUMBER
        return int(node.value)

# Map named tokens to the type value for a LeafPattern
TOKEN_MAP = {"NAME": token.NAME,
             "STRING": token.STRING,
             "NUMBER": token.NUMBER,
             "TOKEN": None}

def _type_of_literal(value):
    if value[0].isalpha():
        return token.NAME
    elif value in grammar.opmap:
        return grammar.opmap[value]
    else:
        return None

def pattern_convert(grammar, raw_node_info):
    """Converts raw node information to a Node or Leaf instance."""
    type, value, context, children = raw_node_info
    if children or type in grammar.number2symbol:
        return pytree.Node(type, children, context=context)
    else:
        return pytree.Leaf(type, value, context=context)

def compile_pattern(pattern):
    return PatternCompiler().compile_pattern(pattern)

cmd: