1. python_mirrors
  2. 2to3

Commits

Guido van Rossum  committed 100de04

Refactored the refactoring -- fixes now live in subpackage "fixes"
and are defined by a class with a specific API. Not all fixes
have been converted yet.

  • Participants
  • Parent commits 3f2d67e
  • Branches default

Comments (0)

Files changed (7)

File fix_apply.py

  • Ignore whitespace
-#!/usr/bin/env python2.5
-# Copyright 2006 Google Inc. All Rights Reserved.
-# Licensed to PSF under a Contributor Agreement.
-
-"""Refactoring tool: change apply(f, a, kw) into f(*a, **kw)."""
-
-__author__ = "Guido van Rossum <guido@python.org>"
-
-# Python imports
-import os
-import sys
-import token
-import logging
-
-import pgen2
-from pgen2 import driver
-
-import pytree
-import patcomp
-
-logging.basicConfig(level=logging.DEBUG)
-
-gr = driver.load_grammar("Grammar.txt") # used by node initializers
-
-
-class Symbols(object):
-
-    def __init__(self, gr):
-        for name, symbol in gr.symbol2number.iteritems():
-            setattr(self, name, symbol)
-
-
-# XXX Make this importable as a module
-syms = Symbols(gr)
-
-
-# XXX Write a single driver script that can do any (or all) refactorings
-def main():
-    args = sys.argv[1:] or ["example.py"]
-
-    dr = driver.Driver(gr, convert=pytree.convert)
-
-    for fn in args:
-        print "Parsing", fn
-        tree = dr.parse_file(fn)
-        refactor(tree)
-        diff(fn, tree)
-
-
-def refactor(tree):
-    visit(tree, fix_apply)
-
-
-def visit(node, func):
-    func(node)
-    for child in node.children:
-        visit(child, func)
-
-
-# Constant nodes used for matching
-n_comma = pytree.Leaf(token.COMMA, ",")
-n_star = pytree.Leaf(token.STAR, "*")
-n_doublestar = pytree.Leaf(token.DOUBLESTAR, "**")
-
-# Tree matching patterns
-pat_compile = patcomp.PatternCompiler().compile_pattern
-p_apply = pat_compile("""
-power< 'apply'
-    trailer<
-        '('
-        arglist<
-            (not argument<NAME '=' any>) func=any ','
-            (not argument<NAME '=' any>) args=any [','
-            (not argument<NAME '=' any>) kwds=any] [',']
-        >
-        ')'
-    >
->
-"""
-    )
-
-
-def fix_apply(node):
-    results = {}
-    if not p_apply.match(node, results):
-        return
-    n_arglist = node.children[1].children[1]
-    assert n_arglist.type
-    func = results["func"]
-    args = results["args"]
-    kwds = results.get("kwds")
-    prefix = node.get_prefix()
-    func.replace(None)
-    if (func.type not in (token.NAME, syms.atom) and
-        (func.type != syms.power or
-         func.children[-2].type == token.DOUBLESTAR)):
-        # Need to parenthesize
-        func = pytree.Node(syms.atom,
-                           (pytree.Leaf(token.LPAR, "("),
-                            func,
-                            pytree.Leaf(token.RPAR, ")")))
-    func.set_prefix("")
-    args.replace(None)
-    args.set_prefix("")
-    if kwds is not None:
-        kwds.replace(None)
-        kwds.set_prefix("")
-    node.children[0].replace(func)
-    node.set_prefix(prefix)
-    l_newargs = [pytree.Leaf(token.STAR, "*"), args]
-    if kwds is not None:
-        l_newargs.extend([pytree.Leaf(token.COMMA, ","),
-                          pytree.Leaf(token.DOUBLESTAR, "**"),
-                          kwds])
-        l_newargs[-2].set_prefix(" ") # that's the ** token
-    for n in l_newargs:
-        if n.parent is not None:
-            n.replace(None) # Force parent to None
-    n_arglist.replace(pytree.Node(syms.arglist, l_newargs))
-    # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t)
-    # can be translated into f(x, y, *t) instead of f(*(x, y) + t)
-
-
-def diff(fn, tree):
-    f = open("@", "w")
-    try:
-        f.write(str(tree))
-    finally:
-        f.close()
-    try:
-        return os.system("diff -u %s @" % fn)
-    finally:
-        os.remove("@")
-
-
-if __name__ == "__main__":
-    main()

File fixes/__init__.py

View file
  • Ignore whitespace
+# Dummy file to make this directory a package.

File fixes/fix_apply.py

View file
  • Ignore whitespace
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Fixer for apply()."""
+
+# Python imports
+import token
+
+# Local imports
+import pytree
+import patcomp
+from pygram import python_symbols as syms
+
+pat_compile = patcomp.PatternCompiler().compile_pattern
+
+PATTERN = """
+power< 'apply'
+    trailer<
+        '('
+        arglist<
+            (not argument<NAME '=' any>) func=any ','
+            (not argument<NAME '=' any>) args=any [','
+            (not argument<NAME '=' any>) kwds=any] [',']
+        >
+        ')'
+    >
+>
+"""
+
+
+class FixApply(object):
+
+    def __init__(self, options):
+        self.options = options
+        self.pattern = pat_compile(PATTERN)
+
+    def match(self, node):
+        results = {}
+        return self.pattern.match(node, results) and results
+
+    def transform(self, node):
+        results = self.match(node)
+        assert results
+        if not results:
+            return
+        assert node.children[1].children[1].type == syms.arglist
+        func = results["func"]
+        args = results["args"]
+        kwds = results.get("kwds")
+        prefix = node.get_prefix()
+        func = func.clone()
+        if (func.type not in (token.NAME, syms.atom) and
+            (func.type != syms.power or
+             func.children[-2].type == token.DOUBLESTAR)):
+            # Need to parenthesize
+            func = pytree.Node(syms.atom,
+                               (pytree.Leaf(token.LPAR, "("),
+                                func,
+                                pytree.Leaf(token.RPAR, ")")))
+        func.set_prefix("")
+        args = args.clone()
+        args.set_prefix("")
+        if kwds is not None:
+            kwds = kwds.clone()
+            kwds.set_prefix("")
+        l_newargs = [pytree.Leaf(token.STAR, "*"), args]
+        if kwds is not None:
+            l_newargs.extend([pytree.Leaf(token.COMMA, ","),
+                              pytree.Leaf(token.DOUBLESTAR, "**"),
+                              kwds])
+            l_newargs[-2].set_prefix(" ") # that's the ** token
+        # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t)
+        # can be translated into f(x, y, *t) instead of f(*(x, y) + t)
+        new = pytree.Node(syms.power,
+                          (func,
+                           pytree.Node(syms.trailer,
+                                       (pytree.Leaf(token.LPAR, "("),
+                                        pytree.Node(syms.arglist, l_newargs),
+                                        pytree.Leaf(token.RPAR, ")")))))
+        new.set_prefix(prefix)
+        return new

File fixes/fix_null.py

View file
  • Ignore whitespace
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Null fixer.  Use as a template."""
+
+
+class FixNull(object):
+
+    """Fixer class.
+
+    The class name must be FixFooBar where FooBar is the result of
+    removing underscores and capitalizing the words of the fix name.
+    For example, the class name for a fixer named 'has_key' should be
+    FixHasKey.
+    """
+
+    def __init__(self, options):
+        """Initializer.
+
+        The argument is an optparse.Values instance which can be used
+        to inspect the command line options.
+        """
+        self.options = options
+
+    def match(self, node):
+        """Matcher.
+
+        Should return a true or false object (not necessarily a bool).
+        It may return a non-empty dict of matching sub-nodes as
+        returned by a matching pattern.
+        """
+        return None
+
+    def transform(self, node):
+        """Transformer.
+
+        Should return None, or a node that is a modified copy of the
+        argument node.  The argument should not be modified in place.
+        """
+        return None

File pygram.py

View file
  • Ignore whitespace
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Export the Python grammar and symbols."""
+
+from pgen2 import driver
+
+
+class Symbols(object):
+
+    def __init__(self, gr):
+        for name, symbol in gr.symbol2number.iteritems():
+            setattr(self, name, symbol)
+
+
+python_grammar = driver.load_grammar("Grammar.txt")
+python_symbols = Symbols(python_grammar)

File pytree.py

View file
  • Ignore whitespace
         """
         raise NotImplementedError
 
+    def clone(self):
+        """Returns a cloned (deep) copy of self.
+
+        This must be implemented by the concrete subclass.
+        """
+        raise NotImplementedError
+
     def post_order(self):
         """Returns a post-order iterator for the tree.
 
         self.type = type
         self.children = tuple(children)
         for ch in self.children:
-            assert ch.parent is None, str(ch)
+            assert ch.parent is None, repr(ch)
             ch.parent = self
 
     def __repr__(self):
         """Compares two nodes for equality."""
         return (self.type, self.children) == (other.type, other.children)
 
+    def clone(self):
+        """Returns a cloned (deep) copy of self."""
+        return Node(self.type, (ch.clone() for ch in self.children))
+
     def post_order(self):
         """Returns a post-order iterator for the tree."""
         for child in self.children:
         """Compares two nodes for equality."""
         return (self.type, self.value) == (other.type, other.value)
 
+    def clone(self):
+        """Returns a cloned (deep) copy of self."""
+        return Leaf(self.type, self.value,
+                    (self.prefix, (self.lineno, self.column)))
+
     def post_order(self):
         """Returns a post-order iterator for the tree."""
         yield self

File refactor.py

View file
  • Ignore whitespace
 import pytree
 import patcomp
 from pgen2 import driver
+import fixes
+import pygram
 
 
 def main(args=None):
     parser.add_option("-f", "--fix", action="append", default=[],
                       help="Each FIX specifies a transformation; default all")
     parser.add_option("-l", "--list-fixes", action="store_true",
-                      help="List available transformations")
+                      help="List available transformations (fixes/fix_*.py)")
     parser.add_option("-v", "--verbose", action="store_true",
                       help="More verbose logging")
 
     options, args = parser.parse_args(args)
     if options.list_fixes:
         print "Available transformations for the -f/--fix option:"
-        for fixname in get_all_fixes():
+        for fixname in get_all_fix_names():
             print fixname
         if not args:
             return 0
     return int(bool(rt.errors))
 
 
-def get_all_fixes():
-    """Return a sorted list of all available fixes."""
-    fixes = []
-    for name in os.listdir(os.path.dirname(__file__)):
+def get_all_fix_names():
+    """Return a sorted list of all available fix names."""
+    fix_names = []
+    for name in os.listdir(os.path.dirname(fixes.__file__)):
         if name.startswith("fix_") and name.endswith(".py"):
-            fixes.append(name[4:-3])
-    fixes.sort()
-    return fixes
+            fix_names.append(name[4:-3])
+    fix_names.sort()
+    return fix_names
 
 
 class RefactoringTool(object):
         """
         self.options = options
         self.errors = 0
-        self.gr = driver.load_grammar("Grammar.txt")
-        self.dr = dr = driver.Driver(self.gr, convert=pytree.convert)
-        self.pairs = self.get_refactoring_pairs()
+        self.driver = driver.Driver(pygram.python_grammar,
+                                    convert=pytree.convert)
+        self.fixers = self.get_fixers()
 
-    def get_refactoring_pairs(self):
+    def get_fixers(self):
         """Inspects the options to load the requested patterns and handlers."""
-        pairs = []
-        fixes = self.options.fix
-        if not fixes or "all" in fixes:
-            fixes = get_all_fixes()
-        for fixname in fixes:
+        fixers = []
+        fix_names = self.options.fix
+        if not fix_names or "all" in fix_names:
+            fixes = get_all_fix_names()
+        for fix_name in fix_names:
             try:
-                mod = __import__("fix_" + fixname)
-            except (ImportError, AttributeError):
-                self.log_error("Can't find transformation %s", fixname)
-            else:
-                name = "?"
-                try:
-                    name = "p_" + fixname
-                    pattern = getattr(mod, name)
-                    name = "fix_" + fixname
-                    handler = getattr(mod, name)
-                except AttributeError:
-                    self.log_error("Can't find fix_%s.%s", fixname, name)
-                else:
-                    if self.options.verbose:
-                        self.log_message("adding transformation: %s", fixname)
-                    pairs.append((pattern, handler))
-        return pairs
+                mod = __import__("fixes.fix_" + fix_name, {}, {}, ["*"])
+            except ImportError:
+                self.log_error("Can't find transformation %s", fix_name)
+                continue
+            parts = fix_name.split("_")
+            class_name = "Fix" + "".join(p.title() for p in parts)
+            try:
+                fix_class = getattr(mod, class_name)
+            except AttributeError:
+                self.log_error("Can't find fixes.fix_%s.%s",
+                               fix_name, class_name)
+                continue
+            try:
+                fixer = fix_class(self.options)
+            except Exception, err:
+                self.log_error("Can't instantiate fixes.fix_%s.%s()",
+                               fix_name, class_name)
+                continue
+            if self.options.verbose:
+                self.log_message("Adding transformation: %s", fix_name)
+            fixers.append(fixer)
+        return fixers
 
     def log_error(self, msg, *args):
         """Increment error count and log a message."""
             return
         try:
             try:
-                tree = self.dr.parse_file(filename)
+                tree = self.driver.parse_file(filename)
             except Exception, err:
                 self.log_error("Can't parse %s: %s: %s",
                                filename, err.__class__.__name__, err)
     def refactor_tree(self, tree):
         changes = 0
         for node in tree.post_order():
-            for pattern, handler in self.pairs:
-                if pattern.match(node):
-                    # XXX Change handler API to return a replacement node
-                    handler(node)
-                    changes += 1
+            for fixer in self.fixers:
+                if fixer.match(node):
+                    new = fixer.transform(node)
+                    if new is not None and new != node:
+                        node.replace(new)
+                        changes += 1
         return changes
 
     def save_tree(self, tree, filename):
                 # XXX Actually save it
                 pass
             else:
-                self.log_error("diff %s returned exit (%s,%s)",
+                self.log_error("Diff %s returned exit (%s,%s)",
                                filename, sts>>8, sts&0xFF)
         finally:
             os.remove(tfn)