1. Chris Mutel
  2. brightway2-data

Commits

Chris Mutel  committed 72e9fc9

Almost finished with testing

  • Participants
  • Parent commits e6e8cf8
  • Branches bw2package

Comments (0)

Files changed (7)

File bw2data/data_store.py

View file
  • Ignore whitespace
 # -*- coding: utf-8 -*
+from . import config
 from .errors import UnknownObject
-from . import config
 import numpy as np
 import os
 import warnings
         new_obj.process()
         return new_obj
 
+    def backup(self):
+        """Save a backup to ``backups`` folder.
+
+        Returns:
+            File path of backup.
+
+        """
+        from .io import BW2Package
+        return BW2Package.export_obj(self)
+
     def write(self, data):
         """Serialize intermediate data to disk.
 
         Need some metaprogramming because class methods have `self` injected automatically."""
         self.validator.__func__(data)
         return True
-
-    def backup(self):
-        """Backup data to compressed JSON file"""
-        raise NotImplementedError
-

File bw2data/database.py

View file
  • Ignore whitespace
         ('col', np.uint32),
     ]
 
-
-    def backup(self):
-        """Save a backup to ``backups`` folder.
-
-        Returns:
-            File path of backup.
-
-        """
-        pass
-        # from .io import BW2PackageExporter
-        # return BW2PackageExporter.export_database(self.name,
-        #     folder="backups", extra_string="." + str(int(time()))
-        #     )
-
     def copy(self, name):
         """Make a copy of the database.
 

File bw2data/io/bw2package.py

View file
  • Ignore whitespace
 # -*- coding: utf-8 -*
-from .. import Database, databases, config, JsonWrapper, methods, Method
+from .. import config, JsonWrapper
 from ..logs import get_logger
-from ..utils import database_hash, download_file
+from ..utils import download_file
 from ..errors import UnsafeData, InvalidPackage
 from ..validate import bw2package_validator
 from voluptuous import Invalid
 from time import time
 import os
-import warnings
 
 
 class BW2Package(object):
+    """This is a format for saving objects which implement the :ref:`datastore` API. Data is stored as a BZip2-compressed file of JSON data. This archive format is compatible across Python versions, and is, at least in theory, programming-language agnostic.
+
+    Validation is done with ``bw2data.validate.bw2package_validator``.
+
+    The data format is:
+
+    .. code-block:: python
+
+        {
+            'metadata': {},  # Dictionary of metadata to be written to metadata-store.
+            'name': basestring,  # Name of object
+            'class': {  # Data on the undying class. A new class is instantiated base on these strings. See _create_class.
+                'module': basestring,  # e.g. "bw2data.database"
+                'name': basestring  # e.g. "Database"
+            },
+            'unrolled_dict': bool,  # Flag indicating if dictionary keys needed to be modified for JSON
+            'data': object  # Object data, e.g. LCIA method or LCI database
+        }
+
+    .. note:: This class does not need to be instantiated, as all its methods are ``classmethods``, i.e. do ``BW2Package.import_obj("foo")`` instead of ``BW2Package().import_obj("foo")``
+
+    """
     APPROVED = {
         'bw2data',
         'bw2regional',
 
     @classmethod
     def _is_valid_package(cls, data):
-        """Valid packages must have the following structure:
-
-        {
-            'metadata': {
-                'module': str,
-                'name': str
-            },
-            'data': object
-        }
-
-        """
         try:
             bw2package_validator(data)
             return True
         return locals()[metadata['name']]
 
     @classmethod
-    def export_objs(cls, objs, folder="export"):
-        for obj in objs:
-            cls.export_obj(obj, folder)
-
-    @classmethod
-    def export_obj(cls, obj, folder="export"):
+    def _prepare_obj(cls, obj):
         data = obj.load()
         ready_data, unrolled = cls._unroll_dict(data)
-        to_export = {
+        return {
             'metadata': obj.metadata[obj.name],
             'name': obj.name,
             'class': cls._get_class_metadata(obj),
             'unrolled_dict': unrolled,
             'data': ready_data,
         }
+
+    @classmethod
+    def _load_object(cls, data, whitelist=True):
+        if not cls._is_valid_package(data):
+            raise InvalidPackage
+        data['class'] = cls._create_class(data['class'], whitelist)
+
+        if isinstance(data['name'], list):
+            data['name'] = tuple(data['name'])
+
+        if data['unrolled_dict']:
+            data['data'] = cls._reroll_dict(data['data'])
+
+        return data
+
+    @classmethod
+    def _create_obj(cls, data):
+        instance = data['class'](data['name'])
+
+        if data['name'] not in instance.metadata:
+            instance.register(**data['metadata'])
+        else:
+            instance.backup()
+            instance.metadata[data['name']] = data['metadata']
+
+        instance.write(data['data'])
+        instance.process()
+        return instance
+
+    @classmethod
+    def export_objs(cls, objs, filename, folder="export"):
+        """Export a list of objects. Can have heterogeneous types.
+
+        Args:
+            * *objs* (list): List of objects to export.
+            * *filename* (str): Name of file to create.
+            * *folder* (str, optional): Folder to create file in. Default is ``export``.
+
+        Returns:
+            Filepath of created file.
+
+        """
         filepath = os.path.join(
             config.request_dir(folder),
-            obj.filename + u".bw2package"
+            filename + u".bw2package"
         )
-        JsonWrapper.dump_bz2(to_export, filepath)
+        JsonWrapper.dump_bz2(
+            [cls._prepare_obj(o) for o in objs],
+            filepath
+        )
+        return filepath
 
     @classmethod
-    def import_objs(cls, filepath, whitelist=True):
-        unprocessed = JsonWrapper.load_bz2(filepath)
-        if isinstance(unprocessed, list):
-            for obj in list:
-                cls.import_obj(obj)
-        else:
-            cls.import_obj(unprocessed)
+    def export_obj(cls, obj, filename=None, folder="export"):
+        """Export an object.
+
+        Args:
+            * *obj* (object): Object to export.
+            * *filename* (str, optional): Name of file to create. Default is ``obj.name``.
+            * *folder* (str, optional): Folder to create file in. Default is ``export``.
+
+        Returns:
+            Filepath of created file.
+
+        """
+        if filename is None:
+            filename = obj.name
+        filepath = os.path.join(
+            config.request_dir(folder),
+            filename + u".bw2package"
+        )
+        JsonWrapper.dump_bz2(cls._prepare_obj(obj), filepath)
+        return filepath
 
     @classmethod
-    def import_obj(cls, data, whitelist=True):
-        if not cls._is_valid_package(data):
-            raise InvalidPackage
-        obj = cls._create_class(data['class'], whitelist)
+    def load_file(cls, filepath, whitelist=True):
+        """Load a bw2package file with one or more objects. Does not create new objects.
 
-        if isinstance(data['name'], list):
-            name = tuple(data['name'])
+        Args:
+            * *filepath* (str): Path of file to import
+            * *whitelist* (bool): Apply whitelist to allowed types. Default is ``True``.
+
+        Returns the loaded data in the bw2package dict data format, with the following changes:
+            * ``"class"`` is an actual class.
+            * dictionaries are rerolled, if necessary.
+            * if ``"name"`` was a list, it is converted to a tuple.
+
+        """
+        raw_data = JsonWrapper.load_bz2(filepath)
+        if isinstance(raw_data, dict):
+            return cls._load_object(raw_data)
         else:
-            name = data['name']
+            return [cls._load_object(o) for o in raw_data]
 
-        instance = obj(name)
+    @classmethod
+    def import_file(cls, filepath, whitelist=True):
+        """Import bw2package file, and create the loaded objects, including registering, writing, and processing the created objects.
 
-        if name not in instance.metadata:
-            instance.register(**data['metadata'])
+        Args:
+            * *filepath* (str): Path of file to import
+            * *whitelist* (bool): Apply whitelist to allowed types. Default is ``True``.
+
+        Returns:
+            Created object or list of created objects.
+
+        """
+        loaded = cls.load_file(filepath, whitelist)
+        if isinstance(loaded, dict):
+            return cls._import_obj(loaded)
         else:
-            obj(name).backup()
-            instance.metadata[name] = data['metadata']
-            # instance.metadata.flush()
-
-        json_data = data['data']
-        if instance['unrolled_dict']:
-            json_data = cls._reroll_dict(json_data)
-
-        instance.write(json_data)
-        instance.process()
-        return instance
+            return [cls._import_obj(o) for o in loaded]
 
 
 def download_biosphere():
     logger = get_logger("io-performance.log")
     start = time()
-    filepath = download_file("biosphere.bw2package")
+    filepath = download_file("biosphere-new.bw2package")
     logger.info("Downloading biosphere package: %.4g" % (time() - start))
     start = time()
-    BW2PackageImporter.importer(filepath)
+    BW2Package.import_objs(filepath)
     logger.info("Importing biosphere package: %.4g" % (time() - start))
 
 
 def download_methods():
     logger = get_logger("io-performance.log")
     start = time()
-    filepath = download_file("methods.bw2iapackage")
+    filepath = download_file("methods-new.bw2iapackage")
     logger.info("Downloading methods package: %.4g" % (time() - start))
     start = time()
-    BW2PackageImporter.importer(filepath)
+    BW2Package.import_objs(filepath)
     logger.info("Importing methods package: %.4g" % (time() - start))

File bw2data/tests/packaging.py

View file
  • Ignore whitespace
 from ..io import BW2Package
 from ..serialization import SerializedDict
 from fixtures import food, biosphere
-from voluptuous import Invalid
+import copy
+import fractions
 
 
 class MockMetadata(SerializedDict):
 
 
 class BW2PackageTest(BW2DataTest):
-    def test_foo(self):
-        pass
-
     def test_class_metadata(self):
         class_metadata = {
             'module': 'bw2data.tests.packaging',
         )
 
     def test_validation(self):
-       pass
+        good_dict = {
+            'metadata': {'foo': 'bar'},
+            'name': 'Johnny',
+            'class': {
+                'module': 'some',
+                'name': 'thing'
+            },
+            'unrolled_dict': False,
+            'data': {}
+        }
+        self.assertTrue(BW2Package._is_valid_package(good_dict))
+        d = copy.deepcopy(good_dict)
+        del d['unrolled_dict']
+        self.assertTrue(BW2Package._is_valid_package(d))
+        d = copy.deepcopy(good_dict)
+        d['name'] = ()
+        self.assertTrue(BW2Package._is_valid_package(d))
+        for key in ['metadata', 'name', 'data']:
+            d = copy.deepcopy(good_dict)
+            del d[key]
+            self.assertFalse(BW2Package._is_valid_package(d))
 
     def test_whitelist(self):
         good_class_metadata = {
         cls = BW2Package._create_class(class_metadata, False)
         self.assertEqual(cls, Database)
 
-    def test_io(self):
+    def test_load_object(self):
+        test_data = {
+            'metadata': {'foo': 'bar'},
+            'name': ['Johnny', 'B', 'Good'],
+            'class': {
+                'module': 'fractions',
+                'name': 'Fraction'
+            },
+            'unrolled_dict': False,
+            'data': {}
+        }
+        after = BW2Package._load_object(copy.deepcopy(test_data), False)
+        for key in test_data:
+            self.assertTrue(key in after)
+        with self.assertRaises(InvalidPackage):
+            BW2Package._load_object({})
+        self.assertEqual(after['class'], fractions.Fraction)
+        self.assertEqual(after['name'], ('Johnny', 'B', 'Good'))
+        self.assertTrue(isinstance(after, dict))
+
+    def test_create_obj(self):
         pass
+
+    def test_export_filenames(self):
+        pass
+
+    def test_load_file(self):
+        pass
+
+    def test_roundtrip(self):
+        pass
+
+    def test_import_file(self):
+        pass

File bw2data/validate.py

View file
  • Ignore whitespace
 
 bw2package_validator = Schema({
     Required('metadata'): {basestring: object},
-    Required('name'): Any(basestring, tuple),
+    Required('name'): Any(basestring, tuple, list),
     'class': {
         Required('module'): basestring,
         Required('name'): basestring,

File docs/index.rst

View file
  • Ignore whitespace
 
 Both the data and metadata objects *store* data, and provide easy ways to save and load data.
 
+.. _metadata-store:
+
 Metadata stores
 ---------------
 

File docs/io.rst

View file
  • Ignore whitespace
 BW2Package
 ==========
 
-Brightway2 has its own data format for efficient saving, loading, and transfer. Read more at the `Brightway2 documentation <http://brightway2.readthedocs.org/en/latest/key-concepts.html#data-interchange>`_.
+Brightway2 has its own data format for archiving data which is both efficient and compatible across operating systems and programming languages. This is the default backup format for Brightway2 :ref:`datastore` objects.
 
 .. note:: **imports** and **exports** are supported.
 
-
-.. autoclass:: bw2data.io.BW2PackageImporter
-    :members:
-
-.. autoclass:: bw2data.io.BW2PackageExporter
+.. autoclass:: bw2data.io.BW2Package
     :members:
 
 Ecospold1