Commits

Anonymous committed 92dd38b

Not saving changes to ignored files in project history
Adding sync project to disk action
Incremental objectdb validation

Comments (0)

Files changed (13)

docs/dev/issues.txt

       def is_scope_valid(self, path, key):
           pass
 
-      def is_callinfo_valid(self, callinfo):
+      def is_callinfo_valid(self, path, key, callinfo):
           pass
 
+  class ObjectDB(object):
+
+      def get_scope_info(self):
+          pass
+
+      def sync(self):
+          pass
+
+      def get_files(self):
+          pass
+
+      def validate_files(self):
+          pass
+
+      def validate_file(self, path):
+          pass
+
+      def file_moved(self, path, new_path):
+          pass
+
+
 * Should we use `FilteredResourceObserver`?
 * Changes caused by move refactoring
 

docs/dev/workingon.txt

-Small Stories
-=============
+Incremental ObjectDB Validation
+===============================
 
-- Adding time to `ChangeSet`\s
+- Not saving changes about ignored resources in undo list
+- What if a resource in objectdb is invalid?
+- Adding *sync project to disk* command
+- Add ``enable_objectdb_validation`` config
+
+* A common base class for dbs
 
 * Adding *clear objectdb* command
-* Adding *sync project data* command
 * Renaming `Project.close()` to `Project.sync()`
+* Adding ``Project`` menu entry?
 * Defaults and documents for project ``config.py`` options
 
 * Inline fails when there is an arg mismatch

rope/base/change.py

         destination = _get_destination_for_move(resource, new_location)
         fscommands = self._get_fscommands(resource)
         fscommands.move(resource.real_path,
-                             self.project._get_resource_path(destination))
+                        self.project._get_resource_path(destination))
         new_resource = self.project.get_resource(destination)
         for observer in list(self.project.observers):
             observer.resource_removed(resource, new_resource)

rope/base/history.py

     history_file = property(_get_history_file)
 
     def do(self, changes):
-        self.undo_list.append(changes)
-        if len(self.undo_list) > self.max_undo_count:
-            del self.undo_list[0]
+        if self._is_change_interesting(changes):
+            self.undo_list.append(changes)
+            if len(self.undo_list) > self.max_undo_count:
+                del self.undo_list[0]
         changes.do()
 
+    def _is_change_interesting(self, changes):
+        for resource in changes.get_changed_resources():
+            if not self.project.is_ignored(resource):
+                return True
+        return False
+
     def undo(self, change=None):
         if not self._undo_list:
             raise exceptions.HistoryError('Undo list is empty')

rope/base/oi/memorydb.py

     def __init__(self, validation):
         self.files = {}
         self.validation = validation
+        self.observers = []
 
     def get_scope_info(self, path, key, readonly=True):
         if path not in self.files:
             if readonly:
                 return NullScopeInfo()
-            self.files[path] = {}
+            self._add_file(path)
         if key not in self.files[path]:
             if readonly:
                 return NullScopeInfo()
             self.files[path][key]._set_validation(self.validation)
         return self.files[path][key]
 
+    def _add_file(self, path):
+        self.files[path] = {}
+        for observer in self.observers:
+            observer.added(path)
+
+    def get_files(self):
+        return self.files.keys()
+
+    def validate_files(self):
+        for file in list(self.get_files()):
+            if not self.validation.is_file_valid(file):
+                self._remove_file(file)
+
+    def validate_file(self, file):
+        if file not in self.files:
+            return
+        for key in list(self.files[file]):
+            if not self.validation.is_scope_valid(file, key):
+                del self.files[file][key]
+
+    def file_moved(self, file, newfile):
+        if file not in self.files:
+            return
+        self.files[newfile] = self.files[file]
+        self._remove_file(file)
+
+    def _remove_file(self, file):
+        del self.files[file]
+        for observer in self.observers:
+            observer.removed(file)
+
+    def add_file_list_observer(self, observer):
+        self.observers.append(observer)
+
     def sync(self):
         pass
 
 
     def get_returned(self):
         return self.returned
+
+
+class FileListObserver(object):
+
+    def added(self, path):
+        pass
+
+    def removed(self, path):
+        pass

rope/base/oi/objectinfo.py

             self.objectdb = memorydb.MemoryObjectDB(validation)
         else:
             self.objectdb = shelvedb.ShelveObjectDB(project, validation)
+        if project.get_prefs().get('validate_objectdb', True):
+            self._init_validation()
+
+    def _init_validation(self):
+        self.objectdb.validate_files()
+        observer = rope.base.project.ResourceObserver(self._resource_changed,
+                                                      self._resource_removed)
+        files = []
+        for path in self.objectdb.get_files():
+            resource = self.to_pyobject.file_to_resource(path)
+            if resource is not None and resource.project == self.project:
+                files.append(resource)
+        self.observer = rope.base.project.FilteredResourceObserver(observer,
+                                                                   files)
+        self.objectdb.add_file_list_observer(_FileListObserver(self))
+        self.project.add_observer(self.observer)
+
+    def _resource_changed(self, resource):
+        self.objectdb.validate_file(resource.real_path)
+
+    def _resource_removed(self, resource, new_resource=None):
+        self.observer.remove_resource(resource)
+        if new_resource is not None:
+            self.objectdb.file_moved(resource.real_path, new_resource.real_path)
+            self.observer.add_resource(new_resource)
 
     def get_returned(self, pyobject, args):
         result = self.get_exact_returned(pyobject, args)
 
     def is_more_valid(self, new, old):
         return new[0] not in ('unknown', 'none')
+
+    def is_file_valid(self, path):
+        return self.to_pyobject.file_to_resource(path) is not None
+
+    def is_scope_valid(self, path, key):
+        return self.to_pyobject.transform(('defined', path, key)) is not None
+
+
+class _FileListObserver(object):
+
+    def __init__(self, object_info):
+        self.object_info = object_info
+        self.observer = self.object_info.observer
+        self.to_pyobject = self.object_info.to_pyobject
+
+    def removed(self, path):
+        resource = self.to_pyobject.file_to_resource(path)
+        if resource is not None:
+            self.observer.remove_resource(resource)
+
+    def added(self, path):
+        resource = self.to_pyobject.file_to_resource(path)
+        if resource is not None:
+            self.observer.add_resource(resource)

rope/base/oi/shelvedb.py

         self._index = None
         self.cache = {}
         self.random = random.Random()
+        self.observers = []
 
     def _get_root(self):
         if self._root is None:
             if readonly and not resource.exists():
                 return
             self.cache[path] = shelve.open(resource.real_path, writeback=True)
+            for observer in self.observers:
+                observer.added(path)
         return self.cache[path]
 
     def get_scope_info(self, path, key, readonly=True):
             file_dict.close()
         self._index = None
         self.cache.clear()
+
+    def get_files(self):
+        return self.index.keys()
+
+    def validate_files(self):
+        for file in list(self.get_files()):
+            if not self.validation.is_file_valid(file):
+                self._remove_file(file)
+
+    def validate_file(self, file):
+        if file not in self.index:
+            return
+        file_dict = self._get_file_dict(file)
+        for key in list(file_dict):
+            if not self.validation.is_scope_valid(file, key):
+                del file_dict[key]
+
+    def file_moved(self, file, newfile):
+        if file not in self.index:
+            return
+        self.index[newfile] = self.index[file]
+        self._remove_file(file, on_disk=False)
+
+    def _remove_file(self, file, on_disk=True):
+        if file not in self.index:
+            return
+        if file in self.cache:
+            self.cache[file].close()
+            del self.cache[file]
+        mapping = self.index[file]
+        del self.index[file]
+        if on_disk:
+            self.root.get_child(mapping).remove()
+        for observer in self.observers:
+            observer.removed(file)
+
+    def add_file_list_observer(self, observer):
+        self.observers.append(observer)

rope/base/oi/transform.py

 import re
 
 import rope.base.project
+from rope.base import exceptions
 
 
 class PyObjectToTextual(object):
         except AttributeError:
             return None
 
-    def module_to_pyobject(self, textual):
-        path = textual[1]
-        return self._get_pymodule(path)
-
     def builtin_to_pyobject(self, textual):
         name = textual[1]
         method = getattr(self, 'builtin_%s_to_pyobject' % textual[1], None)
     def none_to_pyobject(self, textual):
         return None
 
-    def function_to_pyobject(self, textual):
-        return self._get_pyobject_at(textual[1], int(textual[2]))
+    def _module_to_pyobject(self, textual):
+        path = textual[1]
+        return self._get_pymodule(path)
 
-    def class_to_pyobject(self, textual):
+    def _function_to_pyobject(self, textual):
+        path = textual[1]
+        lineno = int(textual[2])
+        pymodule = self._get_pymodule(path)
+        if pymodule is not None:
+            scope = pymodule.get_scope()
+            inner_scope = scope.get_inner_scope_for_line(lineno)
+            return inner_scope.pyobject
+
+    def _class_to_pyobject(self, textual):
         path, name = textual[1:]
         pymodule = self._get_pymodule(path)
+        if pymodule is None:
+            return None
         module_scope = pymodule.get_scope()
         suspected = None
         if name in module_scope.get_names():
 
     def defined_to_pyobject(self, textual):
         if len(textual) == 2:
-            return self.module_to_pyobject(textual)
-        elif textual[2].isdigit():
-            return self.function_to_pyobject(textual)
+            return self._module_to_pyobject(textual)
         else:
-            return self.class_to_pyobject(textual)
+            if textual[2].isdigit():
+                result = self._function_to_pyobject(textual)
+            else:
+                result = self._class_to_pyobject(textual)
+            if not isinstance(result, rope.base.pyobjects.PyModule):
+                return result
 
     def instance_to_pyobject(self, textual):
         type = self.transform(textual[1])
                 return i + 1
 
     def _get_pymodule(self, path):
-        root = os.path.abspath(self.project.address)
-        if path.startswith(root):
-            relative_path = path[len(root):]
-            if relative_path.startswith('/') or relative_path.startswith(os.sep):
-                relative_path = relative_path[1:]
-            resource = self.project.get_resource(relative_path)
-        else:
-            resource = rope.base.project.get_no_project().get_resource(path)
-        return self.project.get_pycore().resource_to_pyobject(resource)
+        resource = self.file_to_resource(path)
+        if resource is not None:
+            return self.project.get_pycore().resource_to_pyobject(resource)
 
-    def _get_pyobject_at(self, path, lineno):
-        scope = self._get_pymodule(path).get_scope()
-        inner_scope = scope.get_inner_scope_for_line(lineno)
-        return inner_scope.pyobject
+    def file_to_resource(self, path):
+        try:
+            root = os.path.abspath(self.project.address)
+            if path.startswith(root):
+                relative_path = path[len(root):]
+                if relative_path.startswith('/') or relative_path.startswith(os.sep):
+                    relative_path = relative_path[1:]
+                resource = self.project.get_resource(relative_path)
+            else:
+                resource = rope.base.project.get_no_project().get_resource(path)
+            return resource
+        except exceptions.RopeError:
+            return None

rope/base/project.py

         self.prefs.add_callback('ignored_resources', self.ignored.set_ignored)
         self.set('ignored_resources', ['*.pyc', '.svn', '*~', '.ropeproject'])
         self._init_rope_folder(ropefolder)
+        # Forcing the creation of `self.pycore` to register observers
+        self.pycore
 
     def get_files(self):
         return self._get_files_recursively(self.root)
     #project.set('ignored_resources', ['*.pyc', '.svn', '*~', '.ropeproject'])
     #project.set('objectdb_type', 'shelve')
     #project.set('save_history', True)
-    #project.set('enable_doi', True)
+    #project.set('perform_doi', True)
+    #project.set('validate_objectdb', True)
 '''

rope/base/pycore.py

 
         """
         receiver = self.call_info.doi_data_received
-        if not self.project.get_prefs().get('enable_doi', True):
+        if not self.project.get_prefs().get('perform_doi', True):
             receiver = None
         runner = dynamicoi.PythonFileRunner(self, resource, args, stdin,
                                             stdout, receiver)

rope/ui/fileactions.py

 def validate_project(context):
     context.project.validate(context.project.root)
 
+def sync_project(context):
+    context.project.sync()
+
 def save_editor(context):
     context.get_core().save_active_editor()
 
                             MenuAddress(['File', 'Project Tree'], 't', 2)))
 actions.append(SimpleAction('validate_project', validate_project, 'C-x p v',
                             MenuAddress(['File', 'Validate Project Files'], 'v', 2)))
+actions.append(SimpleAction('sync_project', sync_project, 'C-x p s',
+                            MenuAddress(['File', 'Sync Project To Disk'], None, 2)))
 
 actions.append(SimpleAction('change_buffer', change_editor, 'C-x b',
                             MenuAddress(['File', 'Change Editor...'], 'c', 3)))

ropetest/historytest.py

         change = ChangeContents(self.file1, '1')
         self.history.undo(change)
 
+    def test_ignoring_ignored_resources(self):
+        self.project.set('ignored_resources', ['ignored*'])
+        ignored = self.project.get_file('ignored.txt')
+        change = CreateResource(ignored)
+        self.history.do(change)
+        self.assertTrue(ignored.exists())
+        self.assertEquals(0, len(self.history.undo_list))
+
 
 class SavingHistoryTest(unittest.TestCase):
 

ropetest/objectinfertest.py

 
     def test_considering_nones_to_be_unknowns(self):
         mod = self.pycore.get_string_module(
-            'class C(object):\n    pass\na_var = None\na_var = C()\na_var = None\n')
+            'class C(object):\n    pass\n'
+            'a_var = None\na_var = C()\na_var = None\n')
         c_class = mod.get_attribute('C').get_object()
         a_var = mod.get_attribute('a_var').get_object()
         self.assertEquals(c_class, a_var.get_type())
         self.assertEquals(c1_class, a_var.get_type())
         self.assertEquals(c2_class, b_var.get_type())
 
+    def test_invalidating_data_after_changing(self):
+        mod = self.pycore.create_module(self.project.root, 'mod')
+        code = 'def a_func(arg):\n    return eval("arg")\n' \
+               'a_var = a_func(a_func)\n'
+        mod.write(code)
+        self.pycore.run_module(mod).wait_process()
+        mod.write('\n' + code)
+        mod.write(code)
+        pymod = self.pycore.resource_to_pyobject(mod)
+        self.assertNotEquals(pymod.get_attribute('a_func').get_object(),
+                             pymod.get_attribute('a_var').get_object())
+
+    def test_invalidating_data_after_moving(self):
+        mod2 = self.pycore.create_module(self.project.root, 'mod2')
+        mod2.write('class C(object):\n    pass\n')
+        mod = self.pycore.create_module(self.project.root, 'mod')
+        code = 'import mod2\ndef a_func(arg):\n    return eval(arg)\n' \
+               'a_var = a_func("mod2.C")\n'
+        mod.write(code)
+        self.pycore.run_module(mod).wait_process()
+        mod.move('newmod.py')
+        pymod = self.pycore.get_module('newmod')
+        pymod2 = self.pycore.resource_to_pyobject(mod2)
+        self.assertEquals(pymod2.get_attribute('C').get_object(),
+                          pymod.get_attribute('a_var').get_object())
+
 
 def _do_for_all_dbs(function):
     def called(self):
     def is_more_valid(self, new, old):
         return new != -1
 
+    def is_file_valid(self, path):
+        return path != 'invalid'
+
+    def is_scope_valid(self, path, key):
+        return path != 'invalid' and key != 'invalid'
+
+
+class _MockFileListObserver(object):
+
+    log = ''
+
+    def added(self, path):
+        self.log += 'added %s ' % path
+
+    def removed(self, path):
+        self.log += 'removed %s ' % path
 
 class ObjectDBTest(unittest.TestCase):
 
     @_do_for_all_dbs
     def test_simple_per_name_does_not_exist(self, db):
         scope_info = db.get_scope_info('file', 'key')
-        self.assertEqual(None, scope_info.get_per_name('name'))
+        self.assertEquals(None, scope_info.get_per_name('name'))
 
     @_do_for_all_dbs
     def test_simple_per_name_after_syncing(self, db):
         db.sync()
 
         scope_info = db.get_scope_info('file', 'key')
-        self.assertEqual(1, scope_info.get_per_name('name'))
+        self.assertEquals(1, scope_info.get_per_name('name'))
 
     @_do_for_all_dbs
     def test_getting_returned(self, db):
         scope_info.add_call((1, 2), 3)
 
         scope_info = db.get_scope_info('file', 'key')
-        self.assertEqual(3, scope_info.get_returned((1, 2)))
+        self.assertEquals(3, scope_info.get_returned((1, 2)))
 
     @_do_for_all_dbs
     def test_getting_returned_when_does_not_match(self, db):
         scope_info.add_call((1, 2), 3)
 
         scope_info = db.get_scope_info('file', 'key')
-        self.assertEqual(None, scope_info.get_returned((1, 1)))
+        self.assertEquals(None, scope_info.get_returned((1, 1)))
 
     @_do_for_all_dbs
     def test_getting_call_info(self, db):
 
         scope_info = db.get_scope_info('file', 'key')
         call_infos = list(scope_info.get_call_infos())
-        self.assertEqual(1, len(call_infos))
-        self.assertEqual((1, 2), call_infos[0].get_parameters())
-        self.assertEqual(3, call_infos[0].get_returned())
+        self.assertEquals(1, len(call_infos))
+        self.assertEquals((1, 2), call_infos[0].get_parameters())
+        self.assertEquals(3, call_infos[0].get_returned())
 
     @_do_for_all_dbs
     def test_invalid_per_name(self, db):
         scope_info = db.get_scope_info('file', 'key', readonly=False)
         scope_info.save_per_name('name', -1)
 
-        self.assertEqual(None, scope_info.get_per_name('name'))
+        self.assertEquals(None, scope_info.get_per_name('name'))
 
     @_do_for_all_dbs
     def test_overwriting_per_name(self, db):
         scope_info = db.get_scope_info('file', 'key', readonly=False)
         scope_info.save_per_name('name', 1)
         scope_info.save_per_name('name', 2)
-        self.assertEqual(2, scope_info.get_per_name('name'))
+        self.assertEquals(2, scope_info.get_per_name('name'))
 
     @_do_for_all_dbs
     def test_not_overwriting_with_invalid_per_name(self, db):
         scope_info = db.get_scope_info('file', 'key', readonly=False)
         scope_info.save_per_name('name', 1)
         scope_info.save_per_name('name', -1)
-        self.assertEqual(1, scope_info.get_per_name('name'))
+        self.assertEquals(1, scope_info.get_per_name('name'))
 
     @_do_for_all_dbs
     def test_getting_invalid_returned(self, db):
         scope_info = db.get_scope_info('file', 'key', readonly=False)
         scope_info.add_call((1, 2), -1)
 
-        self.assertEqual(None, scope_info.get_returned((1, 2)))
+        self.assertEquals(None, scope_info.get_returned((1, 2)))
 
     @_do_for_all_dbs
     def test_not_overwriting_with_invalid_returned(self, db):
         scope_info.add_call((1, 2), 3)
         scope_info.add_call((1, 2), -1)
 
-        self.assertEqual(3, scope_info.get_returned((1, 2)))
+        self.assertEquals(3, scope_info.get_returned((1, 2)))
+
+    @_do_for_all_dbs
+    def test_get_files(self, db):
+        scope_info = db.get_scope_info('file1', 'key', readonly=False)
+        scope_info.add_call((1, 2), 3)
+        scope_info = db.get_scope_info('file2', 'key', readonly=False)
+        scope_info.add_call((1, 2), 3)
+
+        self.assertEquals(set(['file1', 'file2']), set(db.get_files()))
+
+    @_do_for_all_dbs
+    def test_validating_files(self, db):
+        scope_info = db.get_scope_info('invalid', 'key', readonly=False)
+        scope_info.add_call((1, 2), 3)
+        db.validate_files()
+        self.assertEquals(0, len(db.get_files()))
+
+    @_do_for_all_dbs
+    def test_validating_file_for_scopes(self, db):
+        scope_info = db.get_scope_info('file', 'invalid', readonly=False)
+        scope_info.add_call((1, 2), 3)
+        db.validate_file('file')
+        self.assertEquals(1, len(db.get_files()))
+        scope_info = db.get_scope_info('file', 'invalid', readonly=True)
+        self.assertEquals(0, len(list(scope_info.get_call_infos())))
+
+    @_do_for_all_dbs
+    def test_validating_file_moved(self, db):
+        scope_info = db.get_scope_info('file', 'key', readonly=False)
+        scope_info.add_call((1, 2), 3)
+
+        db.file_moved('file', 'newfile')
+        self.assertEquals(1, len(db.get_files()))
+        scope_info = db.get_scope_info('newfile', 'key', readonly=True)
+        self.assertEquals(1, len(list(scope_info.get_call_infos())))
+
+    @_do_for_all_dbs
+    def test_using_file_list_observer(self, db):
+        scope_info = db.get_scope_info('invalid', 'key', readonly=False)
+        scope_info.add_call((1, 2), 3)
+        observer = _MockFileListObserver()
+        db.add_file_list_observer(observer)
+        db.validate_files()
+        self.assertEquals('removed invalid ', observer.log)
 
 
 def suite():
     result.addTests(unittest.makeSuite(ObjectInferTest))
     result.addTests(unittest.makeSuite(DynamicOITest))
     result.addTests(unittest.makeSuite(NewStaticOITest))
+    result.addTests(unittest.makeSuite(ObjectDBTest))
     return result
 
 if __name__ == '__main__':