Commits

Chris Mutel  committed 578bc9f

0.8.4: Include locations in inventory and IA arrays, plus tests

  • Participants
  • Parent commits 64202ef

Comments (0)

Files changed (12)

File bw2data/__init__.py

 # -*- coding: utf-8 -*
 from _config import config
-from meta import databases, methods, mapping, reset_meta
+from meta import databases, methods, mapping, reset_meta, geomapping
 from serialization import JsonWrapper
 from database import Database
 from method import Method

File bw2data/_config.py

         except OSError:
             self.dir = tempfile.mkdtemp()
             self.is_temp_dir = True
-            warnings.warn("\n\tYour changes will not be saved!\n"
-                "\tSet a writeable directory!\n"
-                "\tCurrent data directory is:\n"
-                "\t%s" % self.dir, UserWarning)
+            if not getattr(self, "dont_warn", False):
+                warnings.warn("\n\tYour changes will not be saved!\n"
+                    "\tSet a writeable directory!\n"
+                    "\tCurrent data directory is:\n"
+                    "\t%s" % self.dir, UserWarning)
         self.load_preferences()
 
     def load_preferences(self):

File bw2data/database.py

 # -*- coding: utf-8 -*-
-from . import databases, config, mapping
+from . import databases, config, mapping, geomapping
 from errors import MissingIntermediateData, UnknownObject
 from query import Query
 from time import time
 
         """
         self.database = database
-        if self.database not in databases:
+        if self.database not in databases and not \
+                getattr(config, "dont_warn", False):
             warnings.warn("\n\t%s not a currently installed database" % \
                 database, UserWarning)
 
             raise UnknownObject("This database is not yet registered")
         databases.increment_version(self.database, len(data))
         mapping.add(data.keys())
+        geomapping.add([x["location"] for x in data.values() if \
+            x.get("location", False)])
         if config.p.get("use_cache", False) and self.database in config.cache:
             config.cache[self.database] = data
         filepath = os.path.join(config.dir, "intermediate", self.filename())
         dtype = [('uncertainty_type', np.uint8),
             ('input', np.uint32),
             ('output', np.uint32),
+            ('geo', np.uint32),
             ('row', np.uint32),
             ('col', np.uint32),
             ('technosphere', np.bool),
                     exc["uncertainty type"],
                     mapping[exc["input"]],
                     mapping[key],
+                    geomapping[data[key].get("location", "GLO")],
                     MAX_INT_32,
                     MAX_INT_32,
                     exc["technosphere"],

File bw2data/meta.py

         return len(self.data)
 
 
+class GeoMapping(Mapping):
+    """A dictionary that maps location codes to integers. Needed because parameter arrays have integer ``geo`` fields.
+
+    File data is stored in ``geomapping.pickle``.
+
+    This dictionary does not support setting items directly; instead, use the ``add`` method to add multiple keys."""
+    _filename = "geomapping.pickle"
+
+    def __init__(self, *args, **kwargs):
+        super(GeoMapping, self).__init__(*args, **kwargs)
+        # At a minimum, "GLO" should always be present
+        self.add(["GLO"])
+
+    def __unicode__(self):
+        return u"Mapping from locations to parameter indices."
+
+
 class Databases(SerializedDict):
     """A dictionary for database metadata. This class includes methods to manage database versions. File data is saved in ``databases.json``."""
     _filename = "databases.json"
 mapping = Mapping()
 databases = Databases()
 methods = Methods()
+geomapping = GeoMapping()
 
 
 def reset_meta():
     mapping.__init__()
     databases.__init__()
     methods.__init__()
+    geomapping.__init__()

File bw2data/method.py

 # -*- coding: utf-8 -*-
-from . import config, mapping, methods
+from . import config, mapping, methods, geomapping
 from copy import copy
 from errors import UnknownObject, MissingIntermediateData
 from utils import MAX_INT_32, random_string
     """
     def __init__(self, method, *args, **kwargs):
         self.method = method
-        if self.method not in methods:
+        if self.method not in methods and not \
+                getattr(config, "dont_warn", False):
             warnings.warn("\n\t%s not a currently installed method" % (
                 " : ".join(method)), UserWarning)
 
         if self.method not in methods:
             raise UnknownObject("This database is not yet registered")
         mapping.add(data.keys())
+        geo_codes = [x[1] for x in data.values() if isinstance(x, (tuple, list))]
+        if geo_codes:
+            geomapping.add(geo_codes)
         filepath = os.path.join(config.dir, "intermediate",
             "%s.pickle" % self.get_abbreviation())
         with open(filepath, "wb") as f:
         dtype = [('uncertainty_type', np.uint8),
             ('flow', np.uint32),
             ('index', np.uint32),
+            ('geo', np.uint32),
             ('amount', np.float32),
             ('sigma', np.float32),
             ('minimum', np.float32),
         arr = np.zeros((len(data), ), dtype=dtype)
         arr['minimum'] = arr['maximum'] = arr['sigma'] = np.NaN
         for i, (key, value) in enumerate(data.iteritems()):
+            if isinstance(value, (tuple, list)):
+                value, geo = value
+            else:
+                geo = "GLO"
+
             arr[i] = (
                 0,
                 mapping[key],
                 MAX_INT_32,
+                geomapping[geo],
                 value,
                 np.NaN,
                 np.NaN,

File bw2data/tests/__init__.py

 # -*- coding: utf-8 -*-
-from base import BW2Test
-from database import DatabaseTest
+from .base import BW2DataTest
+from .database import DatabaseTest
+from .geo import GeoTest

File bw2data/tests/base.py

 # -*- coding: utf-8 -*-
-from .. import config
+from .. import config, reset_meta
 import tempfile
 import unittest
 
 
-class BW2Test(unittest.TestCase):
-    def __setup__(self):
+class BW2DataTest(unittest.TestCase):
+    def setUp(self):
+        config.dont_warn = True
         config.dir = tempfile.mkdtemp()
         config.create_basic_directories()
+        reset_meta()
         self.extra_setup()
 
     def extra_setup(self):

File bw2data/tests/database.py

 # -*- coding: utf-8 -*-
-from . import BW2Test
+from . import BW2DataTest
 from .. import Database, databases
 from fixtures import food, biosphere
 
 
-class DatabaseTest(BW2Test):
+class DatabaseTest(BW2DataTest):
     def extra_setup(self):
         d = Database("biosphere")
         d.register("Tests", [], len(biosphere))
 
     def test_versions(self):
         pass
+
+# Test that database.write revises number of processes correctly

File bw2data/tests/fixtures.py

 biosphere = {
-    ("biopshere", 1): {
+    ("biosphere", 1): {
         'categories': ['things'],
         'code': 1,
         'exchanges': [],
         'code': 1,
         'exchanges': [{
             'amount': 0.5,
-            'input': ('Test', 2),
+            'input': ('food', 2),
             'technosphere': True,
             'uncertainty type': 0},
             {'amount': 0.05,
             'input': ('biosphere', 1),
             'technosphere': False,
             'uncertainty type': 0}],
-        'location': 'GLO',
+        'location': 'CA',
         'name': 'lunch',
         'type': 'process',
         'unit': 'kg'
         'code': 2,
         'exchanges': [{
             'amount': 0.25,
-            'input': ('Test', 1),
+            'input': ('food', 1),
             'technosphere': True,
             'uncertainty type': 0},
             {'amount': 0.15,
             'input': ('biosphere', 2),
             'technosphere': False,
             'uncertainty type': 0}],
-        'location': 'GLO',
+        'location': 'CH',
         'name': 'dinner',
         'type': 'process',
         'unit': 'kg'

File bw2data/tests/geo.py

+# -*- coding: utf-8 -*-
+from . import BW2DataTest
+from .. import Database, databases, geomapping, reset_meta, config, Method
+from fixtures import food, biosphere
+import copy
+import numpy as np
+import os
+import pickle
+
+
+class GeoTest(BW2DataTest):
+    def extra_setup(self):
+        geomapping.data = {}
+        geomapping.flush()
+        geomapping.__init__()
+
+    def add_biosphere(self):
+        d = Database("biosphere")
+        d.register("biosphere", [], len(biosphere))
+        d.write(biosphere)
+
+    def add_method(self, simple=True):
+        self.add_biosphere()
+        method = Method(("test method",))
+        method.register("kg")
+        if simple:
+            method.write({
+                ("biosphere", 1): 6,
+                ("biosphere", 2): 5
+                })
+        else:
+            method.write({
+                ("biosphere", 1): (6, "foo"),
+                ("biosphere", 2): (5, "bar")
+                })
+        return method
+
+    def test_geomapping_retrieval(self):
+        geomapping.add(["foobar"])
+        self.assertTrue("foobar" in geomapping)
+        geomapping.__init__()
+        self.assertTrue("foobar" in geomapping)
+
+    def test_glo_always_present(self):
+        print geomapping.data
+        self.assertTrue("GLO" in geomapping)
+
+    def test_method_adds_default_geo(self):
+        method = self.add_method()
+        method.process()
+        pickled = pickle.load(open(os.path.join(config.dir, "processed",
+            method.get_abbreviation() + ".pickle"), "rb"))
+        global_index = geomapping["GLO"]
+        self.assertEqual(global_index, int(pickled[0]["geo"]))
+        self.assertEqual(global_index, int(pickled[1]["geo"]))
+        self.assertEquals(pickled.shape, (2,))
+
+    def test_method_adds_correct_geo(self):
+        method = self.add_method(False)
+        method.process()
+        pickled = pickle.load(open(os.path.join(config.dir, "processed",
+            method.get_abbreviation() + ".pickle"), "rb"))
+        self.assertEqual(geomapping["foo"], int(pickled[0]["geo"]))
+        self.assertEqual(geomapping["bar"], int(pickled[1]["geo"]))
+        self.assertEquals(pickled.shape, (2,))
+
+    def test_database_adds_correct_geo(self):
+        self.add_biosphere()
+        database = Database("food")
+        database.register("food", ["biosphere"], len(food))
+        database.write(food)
+        database.process()
+        pickled = pickle.load(open(os.path.join(config.dir, "processed",
+            database.database + ".pickle"), "rb"))
+        self.assertTrue(geomapping["CA"] in pickled["geo"].tolist())
+        self.assertTrue(geomapping["CH"] in pickled["geo"].tolist())
+
+    def test_database_adds_default_geo(self):
+        self.add_biosphere()
+        database = Database("food")
+        database.register("food", ["biosphere"], len(food))
+        new_food = copy.deepcopy(food)
+        for v in new_food.values():
+            del v["location"]
+        database.write(new_food)
+        database.process()
+        pickled = pickle.load(open(os.path.join(config.dir, "processed",
+            database.database + ".pickle"), "rb"))
+        self.assertTrue(np.allclose(pickled["geo"],
+            geomapping["GLO"] * np.ones((4,))))
+
+    def test_method_write_adds_to_geomapping(self):
+        self.add_method(False)
+        self.assertTrue("foo" in geomapping)
+        self.assertTrue("bar" in geomapping)
+
+    def test_database_write_adds_to_geomapping(self):
+        self.add_biosphere()
+        d = Database("food")
+        d.register("Tests", ["biosphere"], len(food))
+        d.write(food)
+        self.assertTrue("CA" in geomapping)
+        self.assertTrue("CH" in geomapping)

File docs/technical.rst

     :members:
     :inherited-members:
 
+.. autoclass:: bw2data.meta.GeoMapping
+    :members:
+    :inherited-members:
+
+
 Documentation
 =============
 
 
 setup(
   name='bw2data',
-  version="0.8.3",
+  version="0.8.4",
   packages=packages,
   author="Chris Mutel",
   author_email="cmutel@gmail.com",