Commits

markolopa  committed 77c5443

flexible persister added; pickle attribute started

  • Participants
  • Parent commits 29acfb6

Comments (0)

Files changed (10)

File persister/container.py

 class PersistentObject(object):
     # needed by the container to tell that an attribute has to be
     # persisted
-    def __init__(self, default_value=None):
+    def __init__(self, default_value=None, pickle=False):
         self.default_value = default_value
+        self.pickle = pickle
 
 class PersistentListOrDict(PersistentObject):
-    def __init__(self, default_value):
+    def __init__(self, default_value, pickle):
         PersistentObject.__init__(self, default_value)
 
     # for pylint
 class PersistentList(PersistentListOrDict):
     """persistent lists need a special class so that things like __setitem__
     call the persister (see also ListReference and PersistentDict)"""
-    def __init__(self):
+    def __init__(self, pickle=False):
         # the default value is the empty list
-        PersistentListOrDict.__init__(self, [])
+        PersistentListOrDict.__init__(self, [], pickle)
 
     # for pylint
     def __iter__(self):
 class PersistentDict(PersistentListOrDict):
     """persistent dicts need a special class so that things like __setitem__
     call the persister (see also DictReference and PersistentList)"""
-    def __init__(self):
+    def __init__(self, pickle=False):
         # the default value is the empty dict
-        PersistentListOrDict.__init__(self, {})
+        PersistentListOrDict.__init__(self, {}, pickle)
 
 
 class ListOrDictReference(object):
     - if they were treated like as normal objects they would
       not be persisted for instance after setitem operations
     """
-    def __init__(self, name, persister, default_value):
+    def __init__(self, name, persister, default_value, pickle):
         self._name = name
         self._persister = persister
         self.default_value = default_value
+        self.pickle = pickle
 
     def __getitem__(self, key):
-        obj = self._persister.load(self._name, self.default_value)
+        obj = self._persister.load(self._name, self.default_value,
+                                   pickle=self.pickle)
         return obj[key]
 
     def __setitem__(self, key, value):
-        obj = self._persister.load(self._name, self.default_value)
+        obj = self._persister.load(self._name, self.default_value, 
+                                   pickle=self.pickle)
         obj[key] = value
-        self._persister.save(self._name, obj, self.default_value)
+        self._persister.save(self._name, obj, self.default_value, 
+                             pickle=self.pickle)
 
     def __delitem__(self, key):
         obj = self._persister.load(self._name, self.default_value)
         del obj[key]
-        self._persister.save(self._name, obj, self.default_value)
+        self._persister.save(self._name, obj, self.default_value,
+                             pickle=self.pickle)
 
 
 class ListReference(ListOrDictReference):
     # new methods are to be added if the need arises
-    def __init__(self, name, persister):
-        ListOrDictReference.__init__(self, name, persister, [])
+    def __init__(self, name, persister, pickle=False):
+        ListOrDictReference.__init__(self, name, persister, [], pickle)
         
     def __iter__(self):
-        _list = self._persister.load(self._name, [])
+        _list = self._persister.load(self._name, [], pickle=self.pickle)
         for x in _list:
             yield x
             
     def insert(self, i, value):
         _list = self._persister.load(self._name, [])
         _list.insert(i, value)
-        self._persister.save(self._name, _list, [])
+        self._persister.save(self._name, _list, [], pickle=self.pickle)
 
     def append(self, value):
         _list = self._persister.load(self._name, [])
         _list.append(value)
-        self._persister.save(self._name, _list, [])
+        self._persister.save(self._name, _list, [], pickle=self.pickle)
 
 class DictReference(ListOrDictReference):
     # new methods are to be added if the need arises
-    def __init__(self, name, persister):
-        ListOrDictReference.__init__(self, name, persister, {})
+    def __init__(self, name, persister, pickle=False):
+        ListOrDictReference.__init__(self, name, persister, {}, pickle)
 
     def get(self, key):
-        _dict = self._persister.load(self._name, {})
+        _dict = self._persister.load(self._name, {}, pickle=self.pickle)
         return _dict.get(key)
 
     def items(self):
-        _dict = self._persister.load(self._name, {})
+        _dict = self._persister.load(self._name, {}, pickle=self.pickle)
         return _dict.items()
         
 
             # for the lists/dicts, create a reference object
             # - this is the object that will be able to do __getitem__ etc
             if isinstance(value, PersistentList):
-                ref = ListReference(name, self._persister)
+                ref = ListReference(name, self._persister, pickle=value.pickle)
                 self._list_or_dict_references[name] = ref
             elif isinstance(value, PersistentDict):
-                ref = DictReference(name, self._persister)
+                ref = DictReference(name, self._persister, pickle=value.pickle)
                 self._list_or_dict_references[name] = ref
 
             return
         # if the value is a list or not?
         obj = self.__dict__.get(name)
         if isinstance(obj, PersistentObject):
-            self._persister.save(name, value, obj.default_value)
+            self._persister.save(name, value, obj.default_value, 
+                                 pickle=obj.pickle)
             return
             
         # case 3: setting a value to a non-persistent attribute (perhaps

File persister/flexible_persister.py

+"""
+Principles
+- We have 3 types of persisters:
+  - dir (is_dir == True): a directory
+  - file (if_file == True): a yaml file
+  - default: 
+
+- the parent of a file persister is a dir persister.
+- the parent of a default persister is a default persister or a file persister.
+- EMPTY PERSISTER: the empty persister is represented by:
+  - for the dir persister: an empty directory
+  - for the yaml persister: an empty file
+  - for the default persister: an empty dict belonging to the parent dict
+- the attributes can be in a file or in the yaml structure
+  - for the dir persister: a file must be used (pickle)
+  - for the file persister:
+    - option 1: yaml structure
+    - option 2: file name in the yaml structure + pickle file in the directory
+      of the first ancestror which is a dir persister
+- *default value*: the attribute is at its default value iff
+  - for the dir persister; the file does not exist
+  - for other persisters: the entry does not exist in the persister
+  (the idea is that at the creation of the persister all attributes are at the
+  default value, without any manipulation needed for the attributes)
+- the cache/dict issue:
+  - the dir persister has a cache to avoid rereading the attributes in the file
+  - the yaml persister does not have a cache: all yaml data has to be continously
+    mirrored in the dict. So when the root yaml persister is created the whole
+    subpersister and all the attibutes go into memory
+"""
+import os
+import shutil
+import cPickle
+import yaml
+
+from limma.config.logger import get_module_logger
+from limma.persister.persister import Persister
+
+def write_to_yaml_file(value, path):
+    st = yaml.dump(value, default_flow_style=False)
+    with open(path, 'w') as fobj:
+        fobj.write(st)
+
+def read_from_yaml_file(path):
+    with open(path) as fobj:
+        st = fobj.read()
+    return yaml.load(st)
+
+class FlexiblePersister(Persister):
+    """Persist using a mix of directory tree and yaml files. For the attributes
+    there is also two choices: either put them into the yaml file or in a
+    file using pickle"""
+    def __init__(self, name, parent=None, data_dir=None, create=False, 
+                 reset=False, is_dir=False):
+        "check comments in FileSystemPersister"
+        Persister.__init__(self)
+        self.name = name
+        self.parent = parent
+        self.data_dir = data_dir
+
+        self.logger = get_module_logger()
+        
+        # data_dir implies is_dir (but is_dir can be True and no data_dir None)
+        if data_dir is not None:
+            is_dir = True
+
+        # compute the type
+        # - is_dir: only if said to be so
+        # - is_yaml: the first persister which is not dir
+        # - inside_yaml: descendents of a yaml persister
+        # XXX: it is probably a good idea to refactor this class into 3 or 4
+        self.is_dir = False
+        self.is_yaml = False
+        self.inside_yaml = False
+        if is_dir:
+            self.is_dir = True
+        elif parent is None or parent.is_dir:
+            self.is_yaml = True
+        else:
+            self.inside_yaml = True
+
+        # compute self.data_dir: only for dir persister and yaml persisters
+        if self.is_dir or self.is_yaml:
+            if parent is None:
+                if data_dir is None:
+                    raise ValueError("data_dir must be given when there is no "
+                                     "parent container")
+            else:
+                data_dir = os.path.join(parent.data_dir, name)
+            self.data_dir = data_dir
+        else:
+            self.data_dir = None
+
+        # compute self.yaml_path (only for yaml persister)
+        if self.is_yaml:
+            if parent is None or not parent.is_dir:
+                raise ValueError("yaml persister must be a children of a "
+                                 "directory persisters (%s)" % self.name)
+            self.yaml_path = os.path.join(parent.data_dir, self.name) + '.yaml'
+        else:
+            self.yaml_path = None
+
+        # reset
+        if reset:
+            self.remove_data()
+
+        # create the directory if requested
+        #   - dir persiter: create empty dir
+        #   - yaml persister: create yaml file with empty dict + the empty dir
+        #     for pickle objects
+        #   - inside_yaml persister: create empty dict in the parent dict
+        if create:
+            if self.is_dir:
+                if os.path.exists(self.data_dir):
+                    raise ValueError("persister %s (directory %s) "
+                                     "already exists"
+                                     % (self.name, self.data_dir))
+                os.mkdir(self.data_dir)
+            elif self.is_yaml:
+                if os.path.exists(self.yaml_path):
+                    raise ValueError("persister %s (file %s) "
+                                     "already exists"
+                                     % (self.name, self.yaml_path))
+                write_to_yaml_file({}, self.yaml_path)
+                if os.path.exists(self.data_dir):
+                    raise ValueError("persister %s (directory %s) "
+                                     "already exists"
+                                     % (self.name, self.data_dir))
+                os.mkdir(self.data_dir)
+            else:
+                if name in parent.dict:
+                    raise ValueError("persister %s already exists" % self.name)
+                parent.dict[self.name] = {}
+
+        # create load and dict
+        if self.is_dir:
+            self._cache = {}
+            self.dict = None
+        elif self.is_yaml:
+            self._cache = None
+            self.dict = self.read_yaml()
+        else:
+            self._cache = None
+            self.dict = self.parent.dict[self.name]
+
+    def get_subpersister(self, name, create=False):
+        return FlexiblePersister(name, self, create=create)
+
+    def _compute_attribute_path_yaml(self, name):
+        return os.path.join(self.data_dir, name)
+
+    def _compute_attribute_path_pickle(self, name):
+        """the pickle file must stay in the directory of the first ancestor which
+        is not an inside-yaml persister"""
+        persister = self
+        path_items = [name]
+        count = 0
+        while True:
+            if persister.data_dir is not None:
+                fname = '--'.join(path_items)
+                return os.path.join(persister.data_dir, fname)
+            path_items.append(persister.name)
+            count += 1
+            if count > 100:
+                raise RuntimeError("infinite loop avoided")
+
+    def _compute_default(self, default_value):
+        #return default_value
+        # XXX: paranoid attempt to avoid initialising to objects with the
+        # same mutable structure (needed?, good approach?)
+        if default_value == []:
+            default_value = []
+        elif default_value == {}:
+            default_value = {}
+        obj = default_value
+        return obj
+
+    def load(self, name, default_value, pickle=False):
+
+        # special case: pickle file
+        if pickle:
+            path = self._compute_attribute_path_pickle(name)
+            with open(path, 'r') as fobj:
+                return cPickle.load(fobj)
+
+        # is_dir : attributes in files
+        if self.is_dir:
+
+            if self._cache.has_key(name):
+                return self._cache[name]
+
+            path = self._compute_attribute_path_yaml(name)
+            self.logger.debug("getting persistent object '%s' from an "
+                              "existing file" % name)
+            if os.path.exists(path):
+                return read_from_yaml_file(path)
+            else:
+                return self._compute_default(default_value)
+
+        # otherwise: attribute in the dict
+        else:
+            if name in self.dict:
+                return self.dict[name]
+            else:
+                return self._compute_default(default_value)
+
+    def save(self, name, value, default_value, pickle=False):
+        "persist an attribute of the persister"
+
+        # special case: pickle file
+        if pickle:
+            path = self._compute_attribute_path_pickle(name)
+            with open(path, 'w') as fobj:
+                cPickle.dump(value, fobj)
+                return
+
+        # normal case: yaml file
+        if self.is_dir:
+            path = self._compute_attribute_path_yaml(name)
+            if value == default_value and os.path.exists(path):
+                os.remove(path)
+            else:
+                write_to_yaml_file(value, path)
+            self._cache[name] = value
+
+        else:
+            self.dict[name] = value
+            self.write_yaml()
+
+    def read_yaml(self):
+        if self.is_dir:
+            raise ValueError("can't read yaml for a directory persister "
+                             "(named %s)" % self.name)
+        with open(self.yaml_path) as fobj:
+            st = fobj.read()
+        return yaml.load(st)
+
+    def write_yaml(self):
+        "persist the persiter itself by writing the yaml file"
+
+        if self.is_dir:
+            raise ValueError("can't write a yaml directory persister (named %s)" 
+                             % self.name)
+        elif self.is_yaml:
+            write_to_yaml_file(self.dict, self.yaml_path)
+        else:
+            self.parent.write_yaml()
+            
+    def remove_data(self):
+        "remove from the persistence"
+        # don't put this in __delete__, which runs each time the code stops!
+        if self.is_dir:
+            shutil.rmtree(self.data_dir)
+        elif self.is_yaml:
+            os.remove(self.yaml_path)
+        else:
+            del self.parent.dict[self.name]
+            self.parent.write()
+

File persister/persister.py

     #   - cons:
     #     -  for flag it is nicer to tell which default value it has (True or
     #        False) then getting bool()
-    def load(self, name, default_value):
+    def load(self, name, default_value, pickle=False):
         raise NotImplementedError()
 
-    def save(self, name, value, default_value):
+    def save(self, name, value, default_value, pickle=False):
         raise NotImplementedError()
 
 class DictPersister(Persister):
             _dict = {}
         self._dict = _dict
 
-    def load(self, name, default_value):
+    def load(self, name, default_value, pickle=False):
         return self._dict.get(name, default_value)
 
-    def save(self, name, value, default_value):
+    def save(self, name, value, default_value, pickle=False):
         if value == default_value and self._dict.has_key(name):
             del self._dict[name]
         else:
                                  "does not exist"
                                  % (self.name, self.dir))
 
-        self.object_names = set()
-
         self._cache = {}
 
     def get_subpersister(self, name, create=False):
     def _compute_path(self, name):
         return os.path.join(self.dir, name)
 
-    def load(self, name, default_value):
+    def load(self, name, default_value, pickle=False):
 
         if self._cache.has_key(name):
             return self._cache[name]
         self._cache[name] = obj
         return obj
 
-    def save(self, name, obj, default_value):
+    def save(self, name, obj, default_value, pickle=False):
 
         # this would probably do more harm than good
         #if self._cache.has_key(name) and self._cache[name] == obj:
         # don't put this in __delete__, which runs each time the code stops!
         shutil.rmtree(self.dir)
 
-class YamlPersister(Persister):
-    def __init__(self, name, parent=None, data_dir=None, create=False, 
-                 reset=False):
-        "check comments in FileSystemPersister"
-        Persister.__init__(self)
-        self.name = name
-        self.parent = parent
-        self.data_dir = data_dir
 
-        self.logger = get_module_logger()
-
-        # important flags for the yaml persister
-        # - for the moment put into files only the attributes of the root
-        if parent is None:
-            self.is_root = True
-            self.store_in_file = True
-        else:
-            self.is_root = False
-            self.store_in_file = False
-        if parent.is_root:
-            self.is_file = True
-            self.path = os.path.join(parent.data_dir, self.name)
-        else:
-            self.is_file = False
-            self.path = None
-
-        # check if data_dir is needed
-        if self.is_root:
-            if data_dir is None:
-                raise ValueError("data_dir must be given when there is no "
-                                 "parent container")
-            self.logger.info("Creating root persister %s in directory %s"
-                             % (name, self.data_dir))
-
-        # reset
-        if reset:
-            self.remove_data()
-
-        # create the directory if requested
-        # - 3 types
-        #   - is_root: nothing to do (only checks). No dir iff empty dict
-        #   - is_file: nothing to do (only checks). No file iff empty dict
-        #   - other: add the dict entry in the parent
-        if create:
-            if self.is_root:
-                if os.path.exists(self.data_dir):
-                    raise ValueError("persistent container %s (directory %s) "
-                                     "already exists"
-                                     % (self.name, self.data_dir))
-            elif self.is_file:
-                if os.path.exists(self.path):
-                    raise ValueError("persistent container %s (directory %s) "
-                                     "already exists"
-                                     % (self.name, self.data_dir))
-            else:
-                self._save_dict()
-
-        # load dict
-        self._dict = {}
-
-    def get_subpersister(self, name, create=False):
-        return YamlPersister(name, self, create=create)
-
-    def _compute_path(self, name):
-        return os.path.join(self.dir, name)
-
-    def load(self, name, default_value):
-
-        if self._cache.has_key(name):
-            return self._cache[name]
-
-        if self.use_file:
-        
-        path = self._compute_path(name)
-        self.logger.debug("getting persistent object '%s' from an "
-                          "existing file" % name)
-        if os.path.exists(path):
-            with open(path, 'r') as fobj:
-                obj = cPickle.load(fobj)
-        else:
-            
-            # XXX: paranoid attempt to avoid initialising to objects with the
-            # same mutable structure (needed?, good approach?)
-            if default_value == []:
-                default_value = []
-            elif default_value == {}:
-                default_value = {}
-                
-            obj = default_value
-
-        self._cache[name] = obj
-        return obj
-
-    def save(self, name, obj, default_value):
-
-        # this would probably do more harm than good
-        #if self._cache.has_key(name) and self._cache[name] == obj:
-        #    return
-
-        path = self._compute_path(name)
-        if obj == default_value and os.path.exists(path):
-            os.remove(path)
-        else:
-            with open(path, 'w') as fobj:
-                cPickle.dump(obj, fobj)
-        self._cache[name] = obj
-
-    def remove_data(self):
-        "remove from the persistence"
-        # don't put this in __delete__, which runs each time the code stops!
-        shutil.rmtree(self.dir)
-

File persister/yaml_persister.py

+"""
+Principles
+- We have 3 types of persisters:
+  - dir (is_dir == True): a directory
+  - file (if_file == True): a yaml file
+  - default: 
+
+- the parent of a file persister is a dir persister.
+- the parent of a default persister is a default persister or a file persister.
+- EMPTY PERSISTER: the empty persister is represented by:
+  - for dir persister: an empty directory
+  - for the file persister: an empty file
+  - for the default persister: an empty dict belonging to the parent dict
+- the attributes can be in a file or in the yaml structure
+  - for the dir persister: a file must be used (pickle)
+  - for the file persister:
+    - option 1: yaml structure
+    - option 2: file name in the yaml structure + pickle file in the directory
+      of the first ancestror which is a dir persister
+- *default value*: the attribute is at its default value iff
+  - for the dir persister; the file does not exist
+  - for other persisters: the entry does not exist in the persister
+  (the idea is that at the creation of the persister all attributes are at the
+  default value, without any manipulation needed for the attributes)
+
+"""
+import os
+import shutil
+import cPickle
+
+from limma.config.logger import get_module_logger
+from limma.persister.persister import Persister
+
+class YamlPersister(Persister):
+    def __init__(self, name, parent=None, data_dir=None, create=False, 
+                 reset=False, is_dir=None):
+        "check comments in FileSystemPersister"
+        Persister.__init__(self)
+        self.name = name
+        self.parent = parent
+        self.data_dir = data_dir
+
+        # decide for the moment which persister is dir or not (without
+        # forcing the caller to say that)
+        # - for the moment: 
+        #   - the book tree is a dir persister
+        #   - the book is a yaml_file persiter
+        if is_dir is None
+
+
+
+
+
+        self.logger = get_module_logger()
+
+        # important flags for the yaml persister
+        # - for the moment put into files only the attributes of the root
+        if parent is None:
+            self.is_dir = True
+            self.is_file = False
+        elif parent.parent is None:
+            self.is_dir = False
+            self.is_file = True
+        else:
+            self.is_dir = False
+            self.is_file = False
+
+        # compute self.dir
+        if self.is_dir:
+            if data_dir is None:
+                raise ValueError("data_dir must be given when there is no "
+                                 "parent container")
+            self.logger.info("Creating root persister %s in directory %s"
+                             % (name, self.data_dir))
+            self.dir = data_dir
+        else:
+            self.dir = None
+
+        # compute self.path
+        if self.is_file:
+            if not parent.is_dir:
+                raise ValueError("file persister must be children of directory "
+                                 "persisters (%s)" % self.name)
+            self.path = os.path.join(parent.dir, self.name)
+
+        # reset
+        if reset:
+            self.remove_data()
+
+        # create the directory if requested
+        # - 3 types
+        #   - is_root: nothing to do (only checks). No dir iff empty dict
+        #   - is_file: nothing to do (only checks). No file iff empty dict
+        #   - other: nothing to do (only checks). No yaml entry iff empty dict
+        if create:
+            if self.is_dir:
+                if os.path.exists(self.data_dir):
+                    raise ValueError("persister %s (directory %s) "
+                                     "already exists"
+                                     % (self.name, self.data_dir))
+            elif self.is_file:
+                if os.path.exists(self.path):
+                    raise ValueError("persister %s (file %s) "
+                                     "already exists"
+                                     % (self.name, self.path))
+            else:
+                if name in parent.dict:
+                    raise ValueError("persister %s already exists" % self.name)
+
+        # load dict
+        self._dict = {}
+
+    def get_subpersister(self, name, create=False):
+        return YamlPersister(name, self, create=create)
+
+    def load(self, name, default_value):
+
+        if self.is_dir:
+            raise ValueError("can't get an attribute %s from a yaml directory "
+                             "persister (named %s)" % (name, self.name))
+
+        if name in self._dict:
+            return self._dict[name]
+        else:
+            #return default_value
+            # XXX: paranoid attempt to avoid initialising to objects with the
+            # same mutable structure (needed?, good approach?)
+            if default_value == []:
+                default_value = []
+            elif default_value == {}:
+                default_value = {}
+            obj = default_value
+            return obj
+
+    def save(self, name, obj, default_value):
+        "persist an attribute of the persister"
+
+        if self.is_dir:
+            raise ValueError("can't save an attribute %s into a yaml directory "
+                             "persister (named %s)" % (name, self.name))
+
+        # put into the dict and write the dict into the disk
+        self._dict[name] = obj
+        self.write()
+
+    def write(self):
+        "persist the persiter itself"
+
+        if self.is_dir:
+            raise ValueError("can't write a yaml directory persister (named %s)" 
+                             % self.name)
+        elif self.is_file:
+            #XXX: yaml stuff
+            pass
+        else:
+            self.parent.write()
+            
+    def remove_data(self):
+        "remove from the persistence"
+        # don't put this in __delete__, which runs each time the code stops!
+        if self.is_dir:
+            shutil.rmtree(self.dir)
+        elif self.is_file:
+            os.remove(self.path)
+        else:
+            del self.parent.dict[self.name]
+            self.parent.write()

File plugins/plantrisk_change.py

                                            self._output_fname)
 
         # fifth element: subprocess
-        self._output = PersistentObject()
+        self._output = PersistentObject(pickle=True)
         self._subprocess_box = SubprocessBox('subprocess', "Running console",
                                              None,
                                              self._output,

File plugins/shell_no_cancel.py

         self._cmd_box = TextInput('command', "Command", self._cmd_str)
 
         # second element is the output
-        self._output = PersistentObject()
+        self._output = PersistentObject(pickle=True)
         self._output_box = TextBox('command_output', "Output", 
                                    self._output,
                                    visible=(self._output is not None))

File plugins/shell_with_cancel.py

 
         # second element is the running window (it needs to know
         # the command
-        self._output = PersistentObject()
+        self._output = PersistentObject(pickle=True)
         self._subprocess_box = SubprocessBox('subprocess', "Running console",
                                              [self._cmd_str], 
                                              self._output,

File structure/book_tree.py

 from limma.persister.container import Container, PersistentList
+from limma.persister.flexible_persister import FlexiblePersister
 from limma.util.compute_random_name import compute_random_name
 from limma.config.logger import get_module_logger
 from limma.structure.book import Book
 BOOK_NAME_PREFIX = 'book_'
 
 class BookTree(Container):
-    def __init__(self, name, parent_persister, create=False):
+    def __init__(self, name, parent_persister, create=False, data_dir=None):
         self.name = name
-        self._persister = parent_persister.get_subpersister(name, create=create)
+        if parent_persister is None:
+            if data_dir is None:
+                raise ValueError("a data_dir must be given if no parent "
+                                 "persister is used (name = %s)" % name)
+            self._persister = FlexiblePersister(name, create=create, 
+                                                data_dir=data_dir)
+        else:
+            self._persister = parent_persister.get_subpersister(name, create=create)
         Container.__init__(self, self._persister)
         self.logger = get_module_logger()
 

File structure/structure.py

 import os
 
-from limma.persister.persister import FileSystemPersister
 from limma.structure.book_tree import BookTree
 
 class Structure(object):
             create = False
         else:
             create = True
-        persister = FileSystemPersister('struct', None, data_dir=data_dir, 
-                                        create=create)
-        self.book_tree = BookTree('book_tree', persister, create=create)
+        self.book_tree = BookTree('book_tree', None, create=create, 
+                                  data_dir=data_dir)

File tests/test_flexible_persister.py

+#pylint: disable=C0103
+# C0103 - Invalid name (allow using small names here)
+
+from random import randint
+
+from limma.persister.flexible_persister import FlexiblePersister
+from limma.persister.container import (
+    Container, PersistentObject, PersistentList, PersistentDict)
+from limma.tests.helpers import DataDirTestCase
+
+# XX using different classes for the tests so that the directory
+# are different (better for debugging)
+
+class TestFlexiblePersister1(DataDirTestCase):
+
+    def test_no_parent_no_root(self):
+        self.assertRaises(ValueError, FlexiblePersister, 'something', None, 
+                          data_dir=None)
+
+class TestFlexiblePersister2(DataDirTestCase):
+    def test_data_dir_given_and_parent(self):
+        # test creating a new dir for the data_dir
+        _ = FlexiblePersister('c', None, data_dir=self.data_dir, 
+                                create=True, reset=True)
+        # test reusing an existing data_dir
+        c = FlexiblePersister('c', None, data_dir=self.data_dir)
+        # test creating a subpersister
+        _ = c.get_subpersister('cc', create=True)
+
+class TestFlexiblePersister3(DataDirTestCase):
+    def test_obj_list_dict(self):
+
+        # test level-0 persister
+        p1 = FlexiblePersister('p1', None, data_dir=self.data_dir,
+                               create=True, reset=True)
+        self._objects(p1)
+        self._list_reference(p1)
+        self._dict_reference(p1)
+        self._subpersister(p1)
+
+        # test level-1 persister
+        p2 = p1.get_subpersister('p2', create=True)
+        self._objects(p2)
+        self._list_reference(p2)
+        self._dict_reference(p2)
+        self._subpersister(p2)
+
+        # test level-2 persister
+        p3 = p2.get_subpersister('p3', create=True)
+        self._objects(p3)
+        self._list_reference(p3)
+        self._dict_reference(p3)
+        self._subpersister(p3)
+
+    def _objects(self, persister):
+        
+        class MyContainer(Container):
+            def __init__(self, persister):
+                Container.__init__(self, persister)
+                self.a_int = PersistentObject()
+                self.a_list = PersistentList()
+                self.a_dict = PersistentDict()
+                
+        x = MyContainer(persister)
+        self.assertEqual(x.a_int, None)
+        assert_equal_lists(self, x.a_list, [])
+        assert_equal_dicts(self, x.a_dict, {})
+
+        # use random values so that data changes from one test to another
+        a_int = randint(0, 100)
+        a_list = range(randint(0, 10))
+        a_dict = {randint(0, 10): randint(0, 100)}
+
+        x.a_int = a_int
+        x.a_list = a_list[:]
+        x.a_dict = a_dict.copy()
+        self.assertEqual(x.a_int, a_int)
+        assert_equal_lists(self, x.a_list, a_list)
+        assert_equal_dicts(self, x.a_dict, a_dict)
+
+    def _list_reference(self, persister):
+
+        class MyContainer(Container):
+            def __init__(self, persister):
+                Container.__init__(self, persister)
+                self.l = PersistentList()
+        x = MyContainer(persister)
+
+        # test default value
+        l = []
+        assert_equal_lists(self, x.l, l)
+
+        # test set
+        l = range(4)
+        x.l = range(4)  # using l here the list remains the same
+        assert_equal_lists(self, x.l, l)
+
+        # test setitem
+        l[2] = 'spam'
+        x.l[2] = 'spam'
+        assert_equal_lists(self, x.l, l)
+
+        # test getitem
+        self.assertEqual(x.l[2], l[2])
+
+        # test delete
+        del l[3]
+        del x.l[3]
+        assert_equal_lists(self, x.l, l)
+
+        # test contains
+        self.assertTrue('spam' in x.l)
+        self.assertFalse('eggs' in x.l)
+
+        # test iter
+        for a, b in zip(x.l, l):
+            self.assertEqual(a, b)
+            
+    def _dict_reference(self, persister):
+
+        class MyContainer(Container):
+            def __init__(self, persister):
+                Container.__init__(self, persister)
+                self.d = PersistentDict()
+        x = MyContainer(persister)
+
+        # test default value
+        d = {}
+        assert_equal_dicts(self, x.d, d)
+
+        # test setattr
+        d = {'a': 'spam'}
+        x.d = {'a': 'spam'}
+        assert_equal_dicts(self, x.d, d)
+
+        # test setitem
+        d['b'] = 'eggs'
+        x.d['b'] = 'eggs'
+        assert_equal_dicts(self, x.d, d)
+
+        # test getitem
+        self.assertEqual(x.d['a'], d['a'])
+
+        # test delete
+        del d['a']
+        del x.d['a']
+        assert_equal_dicts(self, x.d, d)
+
+        # test get
+        self.assertEqual(x.d.get('a'), d.get('a'))
+        self.assertEqual(x.d.get('bad key'), d.get('bad key'))
+
+    def _subpersister(self, persister):
+
+        p = persister.get_subpersister('p', create=True)
+
+        class MyContainer(Container):
+            def __init__(self, persister):
+                Container.__init__(self, persister)
+                self.a_int = PersistentObject()
+                self.a_list = PersistentList()
+                self.a_dict = PersistentDict()
+                
+        x = MyContainer(p)
+        self.assertEqual(x.a_int, None)
+        assert_equal_lists(self, x.a_list, [])
+        assert_equal_dicts(self, x.a_dict, {})
+
+class TestFlexiblePersister4(DataDirTestCase):
+    "the most important test, except for list/dict specific operations"
+
+    def test_persistence(self):
+        p0 = FlexiblePersister('p0', None, data_dir=self.data_dir,
+                               create=True, reset=True)
+        class MyContainer(Container):
+            def __init__(self, persister):
+                Container.__init__(self, persister)
+                self.int_0 = PersistentObject()
+                self.int_1 = PersistentObject(pickle=True)
+                self.list_0 = PersistentList()
+                self.list_1 = PersistentList(pickle=True)
+                self.dict_0 = PersistentDict()
+                self.dict_1 = PersistentDict(pickle=True)
+                self.a_dict = PersistentDict()
+                
+        # use different values so that data changes from one test to another
+        int_00 = 41
+        int_01 = 42
+        int_10 = 43
+        int_11 = 44
+        int_20 = 45
+        int_21 = 46
+        list_00 = range(3)
+        list_01 = range(4)
+        list_10 = range(5)
+        list_11 = range(6)
+        list_20 = range(7)
+        list_21 = range(8)
+        dict_00 = {0: 1}
+        dict_01 = {0: 2}
+        dict_10 = {1: 11}
+        dict_11 = {1: 12}
+        dict_20 = {2: 21}
+        dict_21 = {2: 22}
+
+        # p0 is a directory persister
+        self.assertTrue(p0.is_dir and not p0.is_yaml and not p0.inside_yaml)
+        c0 = MyContainer(p0)
+        c0.int_0 = int_00
+        c0.int_1 = int_01
+        c0.list_0 = list_00[:]
+        c0.list_1 = list_01[:]
+        c0.dict_0 = dict_00.copy()
+        c0.dict_1 = dict_01.copy()
+
+        # p1 is a yaml persister
+        p1 = p0.get_subpersister('p1', create=True)
+        self.assertTrue(not p1.is_dir and p1.is_yaml and not p1.inside_yaml)
+        c1 = MyContainer(p1)
+        c1.int_0 = int_10
+        c1.int_1 = int_11
+        c1.list_0 = list_10[:]
+        c1.list_1 = list_11[:]
+        c1.dict_0 = dict_10.copy()
+        c1.dict_1 = dict_11.copy()
+
+        # p2 is a inside-yaml persister
+        p2 = p1.get_subpersister('p2', create=True)
+        self.assertTrue(not p2.is_dir and not p2.is_yaml and p2.inside_yaml)
+        c2 = MyContainer(p2)
+        c2.int_0 = int_20
+        c2.int_1 = int_21
+        c2.list_0 = list_20[:]
+        c2.list_1 = list_21[:]
+        c2.dict_0 = dict_20.copy()
+        c2.dict_1 = dict_21.copy()
+
+        # check values
+        self.assertEqual(c0.int_0, int_00)
+        self.assertEqual(c0.int_1, int_01)
+        assert_equal_lists(self, c0.list_0, list_00)
+        assert_equal_lists(self, c0.list_1, list_01)
+        assert_equal_dicts(self, c0.dict_0, dict_00)
+        assert_equal_dicts(self, c0.dict_1, dict_01)
+        self.assertEqual(c1.int_0, int_10)
+        self.assertEqual(c1.int_1, int_11)
+        assert_equal_lists(self, c1.list_0, list_10)
+        assert_equal_lists(self, c1.list_1, list_11)
+        assert_equal_dicts(self, c1.dict_0, dict_10)
+        assert_equal_dicts(self, c1.dict_1, dict_11)
+        self.assertEqual(c2.int_0, int_20)
+        self.assertEqual(c2.int_1, int_21)
+        assert_equal_lists(self, c2.list_0, list_20)
+        assert_equal_lists(self, c2.list_1, list_21)
+        assert_equal_dicts(self, c2.dict_0, dict_20)
+        assert_equal_dicts(self, c2.dict_1, dict_21)
+
+        # remove persisters (files should remain)
+        del p2, p1, p0, c2, c1, c0
+
+        # test persistence
+        # - note that the name of the first persister is irrelevant (since no
+        #   subdir is created with its name 
+        q0 = FlexiblePersister('spam', None, data_dir=self.data_dir)
+        d0 = MyContainer(q0)
+        q1 = q0.get_subpersister('p1')
+        d1 = MyContainer(q1)
+        q2 = q1.get_subpersister('p2')
+        d2 = MyContainer(q2)
+        self.assertEqual(d0.int_0, int_00)
+        self.assertEqual(d0.int_1, int_01)
+        assert_equal_lists(self, d0.list_0, list_00)
+        assert_equal_lists(self, d0.list_1, list_01)
+        assert_equal_dicts(self, d0.dict_0, dict_00)
+        assert_equal_dicts(self, d0.dict_1, dict_01)
+        self.assertEqual(d1.int_0, int_10)
+        self.assertEqual(d1.int_1, int_11)
+        assert_equal_lists(self, d1.list_0, list_10)
+        assert_equal_lists(self, d1.list_1, list_11)
+        assert_equal_dicts(self, d1.dict_0, dict_10)
+        assert_equal_dicts(self, d1.dict_1, dict_11)
+        self.assertEqual(d2.int_0, int_20)
+        self.assertEqual(d2.int_1, int_21)
+        assert_equal_lists(self, d2.list_0, list_20)
+        assert_equal_lists(self, d2.list_1, list_21)
+        assert_equal_dicts(self, d2.dict_0, dict_20)
+        assert_equal_dicts(self, d2.dict_1, dict_21)
+
+def assert_equal_lists(self, pseudo_list, _list):
+    self.assertEqual([e for e in pseudo_list], _list)
+    
+def assert_equal_dicts(self, pseudo_dict, _dict):
+    for k, v in pseudo_dict.items():
+        self.assertEqual(v, _dict[k])
+    for k, v in _dict.items():
+        self.assertEqual(v, pseudo_dict[k])
+