Commits

Anonymous committed 2a31b15

Matching similar statements

  • Participants
  • Parent commits 0f3399b

Comments (0)

Files changed (9)

docs/dev/done.txt

 ===========
 
 
+- Patching ASTs to include formatting information : May 9, 2007
+
+
 > Public Release 0.5 : May 6, 2007
 
 

docs/dev/issues.txt

 Patched AST
 ===========
 
-I propose to construct the patched tree from input and at the same
-time insert textual info into it.  In this phase we might add new
-nodes that contain textual information about the source.
-
-Issues:
-
-* How to handle parenthesis? like ``a + ((b + c) + d)``
-* Nodes are not in order; like function args and defaults
-* How to save comments, parenthesis, like breaks and ...
-* Many commands in the same line; like ``a = 1; b = 2``
-* Where do textual data belong to?  To the parent probably
-
-
-Monkey-patching Old Nodes
--------------------------
-
-We can add information to the old tree.  The new ast will look like
-the old one and provides the same methods but adds two new fields:
-`region` and `sorted_children`.
-
-`sorted_children` field is a list of AST nodes and strings.  The
-strings contain textual information and the nodes are the place to
-insert the children.
-
-
-Construction
-------------
-
-The construction is recursive.  When constructing the child nodes we
-should give them the start offset.  They can calculate their own start
-and end offsets.
-
+I propose to construct a patched AST from code that holds formatting
+info.
 
 Supporting Formatting
 ---------------------
 can do this by searching over the parents nodes and `sorted_children`.
 Whenever we encounter the old AST node we replace it with the new one.
 
-There are a few other questions.  Should we store offsets or lengths
-in AST nodes.  Should formatting information be updated after changes.
+There are two approaches here.
+
+* We can change ASTs to construct a new AST
+* We can derived change source from the main AST
+
+The second approach seems more practical
 
 
 Finding Similar Pieces

docs/dev/stories.txt

 * Extract superclass
 * Collapse hierarchy
 * Python C extensions
-* User specified source folders
 * Zip imports
 * Pull up method
 * Push down method
 * Supporting modules without source code
 
 
-* Finding similar statements when extracting variable/method
-
-
 * Moving `staticmethod`\s
 
 
 
 
 > Public Release 0.6m1 : May 20, 2007
+
+
+* Finding similar statements when extracting variable/method
+
+
+* Adding custom source folders in ``config.py``

docs/dev/workingon.txt

-Patched AST
-===========
+Similar Finder
+==============
 
-- tuple parameter unpacking
-- handle Sliceobj
-- handle long slices
+- Matching patterns with multiple statements
+- Check multiple matches
+- Only matches in selected region
 
-* profiling
-* py3k: How to handle both versions?
+* Only scanning selected region for matches
+* Changing patchedast only to patch a branch?
+
+* Handling '<>' in patchedast
+
+* py3k: How to handle both branches?
 
   * metadata=? and classes
   * function annotations
   * // and /
+  * decorated class
   * ...
 
-* adding custom source folders in ``config.py``
-
 
 Remaining Small Stories
 =======================

rope/refactor/__init__.py

         parent = self.resource.parent
         name = self.resource.name[:-3]
         changes.add_change(CreateFolder(parent, name))
-        new_path = parent.path + '/%s/__init__.py' % name
+        parent_path = parent.path + '/'
+        if not parent.path:
+            parent_path = ''
+        new_path = parent_path + '%s/__init__.py' % name
         changes.add_change(MoveResource(self.resource, new_path))
         return changes
 

rope/refactor/patchedast.py

 
 def get_patched_ast(source):
     """Adds `region` and `sorted_children` fields to nodes"""
-    ast = compiler.parse(source)
+    return patch_ast(compiler.parse(source), source)
+
+
+def patch_ast(ast, source):
+    """Patches the given ast"""
     call_for_nodes(ast, _PatchingASTWalker(source))
     return ast
 

rope/refactor/similarfinder.py

 """This module can be used for finding similar code"""
+import compiler.ast
+
+from rope.refactor import patchedast
 
 
 class SimilarFinder(object):
         self.end = len(self.source)
         if end is not None:
             self.end = end
+        self.ast = patchedast.get_patched_ast(self.source)
 
-    def find(self, expression):
-        index = self.start
-        while index < self.end:
-            try:
-                index = self.source.index(expression, index, self.end)
-                yield (index, index + len(expression))
-                index += len(expression)
-            except ValueError:
-                break
+    def get_match_regions(self, code):
+        wanted = self._create_pattern(code)
+        matches = _ASTMatcher(self.ast, wanted).find_matches()
+        for match in matches:
+            start, end = match.get_region()
+            if self.start <= start < end <= self.end:
+                yield match.get_region()
+
+    def _create_pattern(self, expression):
+        ast = compiler.parse(expression)
+        # Module.Stmt
+        nodes = ast.node.nodes
+        if len(nodes) == 1 and isinstance(nodes[0], compiler.ast.Discard):
+            # Discard
+            wanted = nodes[0].expr
+        else:
+            wanted = nodes
+        return wanted
+
+    def _does_match(self, node1, node2):
+        if node1.__class__ != node2.__class__:
+            return False
+        children1 = node1.getChildren()
+        children2 = node2.getChildren()
+        if len(children1) != len(children2):
+            return False
+        for child1, child2 in zip(children1, children2):
+            if isinstance(child1, compiler.ast.Node):
+                if not self._does_match(child1, child2):
+                    return False
+            else:
+                if child1 != child2:
+                    return False
+        return True
+
+
+class _ASTMatcher(object):
+
+    def __init__(self, body, pattern):
+        """Searches the given pattern in the body AST.
+
+        body is an AST node and pattern can be either an AST node or
+        a list of ASTs nodes
+        """
+        self.body = body
+        self.pattern = pattern
+        self.matches = None
+
+    def find_matches(self):
+        if self.matches is None:
+            self.matches = []
+            patchedast.call_for_nodes(self.body, self._check_node,
+                                      recursive=True)
+        return self.matches
+
+    def _check_node(self, node):
+        if isinstance(self.pattern, list):
+            self._match_statements(node)
+        elif self._does_match(self.pattern, node):
+            self.matches.append(ExpressionMatch(node))
+
+    def _does_match(self, node1, node2):
+        if node1.__class__ != node2.__class__:
+            return False
+        children1 = node1.getChildren()
+        children2 = node2.getChildren()
+        if len(children1) != len(children2):
+            return False
+        for child1, child2 in zip(children1, children2):
+            if isinstance(child1, compiler.ast.Node):
+                if not self._does_match(child1, child2):
+                    return False
+            else:
+                if child1 != child2:
+                    return False
+        return True
+
+    def _match_statements(self, node):
+        if not isinstance(node, compiler.ast.Stmt):
+            return
+        for index in range(len(node.nodes)):
+            if len(node.nodes) - index >= len(self.pattern):
+                current_stmts = node.nodes[index:index + len(self.pattern)]
+                if self._does_stmts_match(current_stmts):
+                    self.matches.append(StatementMatch(current_stmts))
+
+    def _does_stmts_match(self, current_stmts):
+        if len(current_stmts) != len(self.pattern):
+            return False
+        for stmt, expected in zip(current_stmts, self.pattern):
+            if not self._does_match(stmt, expected):
+                return False
+        return True
+
+
+class Match(object):
+
+    def get_region(self):
+        """Returns match region"""
+
+class ExpressionMatch(object):
+
+    def __init__(self, ast):
+        self.ast = ast
+
+    def get_region(self):
+        return self.ast.region
+
+
+class StatementMatch(object):
+
+    def __init__(self, ast_list):
+        self.ast_list = ast_list
+
+    def get_region(self):
+        return self.ast_list[0].region[0], self.ast_list[-1].region[1]

rope/ui/refactor.py

         editor = fileeditor.get_editor()
         changes = rope.refactor.ModuleToPackage(
             context.project, resource).get_changes()
-        self.project.do(changes)
+        context.project.do(changes)
 
 
 class ExtractDialog(RefactoringDialog):

ropetest/refactor/similarfindertest.py

 
     def test_trivial_case(self):
         finder = similarfinder.SimilarFinder('')
-        self.assertEquals([], list(finder.find('10')))
+        self.assertEquals([], list(finder.get_match_regions('10')))
 
     def test_constant_integer(self):
         source = 'a = 10\n'
         finder = similarfinder.SimilarFinder(source)
         result = [(source.index('10'), source.index('10') + 2)]
-        self.assertEquals(result, list(finder.find('10')))
+        self.assertEquals(result, list(finder.get_match_regions('10')))
+
+    def test_simple_addition(self):
+        source = 'a = 1 + 2\n'
+        finder = similarfinder.SimilarFinder(source)
+        result = [(source.index('1'), source.index('2') + 1)]
+        self.assertEquals(result, list(finder.get_match_regions('1 + 2')))
+
+    def test_simple_addition2(self):
+        source = 'a = 1 +2\n'
+        finder = similarfinder.SimilarFinder(source)
+        result = [(source.index('1'), source.index('2') + 1)]
+        self.assertEquals(result, list(finder.get_match_regions('1 + 2')))
+
+    def test_simple_assign_statements(self):
+        source = 'a = 1 + 2\n'
+        finder = similarfinder.SimilarFinder(source)
+        self.assertEquals([(0, len(source) - 1)],
+                          list(finder.get_match_regions('a = 1 + 2')))
+
+    def test_simple_multiline_statements(self):
+        source = 'a = 1\nb = 2\n'
+        finder = similarfinder.SimilarFinder(source)
+        self.assertEquals([(0, len(source) - 1)],
+                          list(finder.get_match_regions('a = 1\nb = 2')))
+
+    def test_multiple_matches(self):
+        source = 'a = 1 + 1\n'
+        finder = similarfinder.SimilarFinder(source)
+        result = list(finder.get_match_regions('1'))
+        self.assertEquals(2, len(result))
+        start1 = source.index('1')
+        self.assertEquals((start1, start1 + 1) , result[0])
+        start2 = source.rindex('1')
+        self.assertEquals((start2, start2 + 1) , result[1])
+
+    def test_multiple_matches2(self):
+        source = 'a = 1\nb = 2\n\na = 1\nb = 2\n'
+        finder = similarfinder.SimilarFinder(source)
+        self.assertEquals(
+            2, len(list(finder.get_match_regions('a = 1\nb = 2'))))
+
+    def test_restricting_the_region_to_search(self):
+        source = '1\n\n1\n'
+        finder = similarfinder.SimilarFinder(source, start=2)
+        result = list(finder.get_match_regions('1'))
+        start = source.rfind('1')
+        self.assertEquals([(start, start + 1)], result)
 
 
 if __name__ == '__main__':