Commits

Chris Mutel  committed 4e60a5f

0.8; separate data handling to bw2data package

  • Participants
  • Parent commits f5f9641

Comments (0)

Files changed (29)

 include *.txt
 include brightway2/*.py
-include brightway2/io/*.py
-include brightway2/proxies/*.py

File brightway2/__init__.py

 # -*- coding: utf-8 -*
-from _config import config
-from meta import databases, methods, mapping, reset_meta
-from serialization import JsonWrapper
-from database import Database
-from method import Method
-from query import Query, Filter, Result
+from bw2data import *

File brightway2/_config.py

-# -*- coding: utf-8 -*-
-import os
-import json
-import tempfile
-
-
-class Config(object):
-    """A singleton that store configuration settings. Default data directory is ``brightway`` in the user's home directory, and is stored as ``config.dir``. Other configuration settings can also be assigned as needed.
-
-    Args:
-        * *path* (str, optional): The path of the data directory. Must be writeable.
-
-    """
-    version = 0.1
-    basic_directories = ("processed", "intermediate", "backups", "logs")
-
-    # TODO: Create directory if needed (and basic dirs)
-
-    def __init__(self, path=None):
-        self.is_temp_dir = False
-        self.reset(path)
-        self.cache = {}
-
-    def check_dir(self, dir=None):
-        """Check is directory is writeable."""
-        return os.access(dir or self.dir, os.W_OK)
-
-    def reset(self, path=None):
-        """Reset to original configuration. Useful for testing."""
-        # Use _dir instead of dir beacuse need to check dir ourselves
-        self._dir = self.get_home_directory(path)
-        if not self.check_dir():
-            self.dir = tempfile.mkdtemp()
-            self.is_temp_dir = True
-            print "Your changes will not be saved! Set a writeable directory!"
-            print "Current data directory is:"
-            print self.dir
-        self.load_preferences()
-
-    def load_preferences(self):
-        """Load a set of preferences from a file in the home directory.
-
-        Preferences as stored as ``config.p``."""
-        try:
-            self.p = json.load(open(os.path.join(
-                self.dir, "preferences.json")))
-        except:
-            self.p = {"use_cache": True}
-
-    def save_preferences(self):
-        """Serialize preferences to disk."""
-        with open(os.path.join(self.dir, "preferences.json"), "w") as f:
-            json.dump(self.p, f, indent=2)
-
-    def get_home_directory(self, path=None):
-        """Get data directory, trying in order:
-
-        * Provided path (optional)
-        * ``BRIGHTWAY2-DIR`` environment variable
-        * ``brightway2`` in user's home directory
-
-        To set the environment variable:
-
-        * Unix/Mac: ``export BRIGHTWAY2_DIR=/path/to/brightway2/directory``
-        * Windows: ``set BRIGHTWAY2_DIR=\path\to\brightway2\directory``
-
-        """
-        if path:
-            return path
-        envvar = os.getenv("BRIGHTWAY2_DIR")
-        if envvar:
-            return envvar
-        else:
-            return os.path.join(os.path.expanduser("~"), "brightway2")
-
-    def request_dir(self, dirname):
-        """Return ``True`` if directory already exists or can be created."""
-        path = os.path.join(self.dir, dirname)
-        if self.check_dir(path):
-            return path
-        else:
-            try:
-                os.mkdir(path)
-                return path
-            except:
-                return False
-
-    def create_basic_directories(self):
-        """Create basic directory structure.
-
-        Useful when first starting or for tests."""
-        for name in self.basic_directories:
-            os.mkdir(os.path.join(self.dir, name))
-
-    def _get_dir(self):
-        return self._dir
-
-    def _set_dir(self, d):
-        self._dir = d
-        if not self.check_dir():
-            raise OSError("This directory is not writeable")
-
-    dir = property(_get_dir, _set_dir)
-
-
-config = Config()

File brightway2/database.py

-# -*- coding: utf-8 -*-
-from . import databases, config, mapping
-from errors import MissingIntermediateData, UnknownObject
-from query import Query
-from time import time
-from utils import natural_sort, MAX_INT_32
-from validate import db_validator
-import datetime
-import numpy as np
-import os
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-
-class Database(object):
-    """A manager for a database. This class can register or deregister databases, write intermediate data, process data to parameter arrays, query, validate, and copy databases.
-
-    Databases are automatically versioned.
-
-    The Database class never holds intermediate data, but it can load or write intermediate data. The only attribute is *database*, which is the name of the database being managed.
-
-    Instantiation does not load any data. If this database is not yet registered in the metadata store, a warning is written to ``stdout``.
-
-    Args:
-        *database* (str): Name of the database to manage.
-
-    """
-    def __init__(self, database):
-        """Instantiate a Database object.
-
-        Does not load any data. If this database is not yet registered in the metadata store, a warning is written to **stdout**.
-
-
-        """
-        self.database = database
-        if self.database not in databases:
-            print "Warning: %s not a currently installed database" % database
-
-    @property
-    def version(self):
-        """The current version number (integer) of this database.
-
-        Returns:
-            Version number
-
-        """
-        return databases.version(self.database)
-
-    def query(self, *queries):
-        """Search through the database. See :class:`query.Query` for details."""
-        return Query(*queries)(self.load())
-
-    def relabel_data(self, data, new_name):
-        """Relabel database keys and exchanges.
-
-        In a database which internally refer to the same database, update to new database name ``new_name``.
-
-        Needed to copy a database completely or cut out a section of a database.
-
-        For example:
-
-        .. code-block:: python
-
-            data = {
-                ("old and boring", 1):
-                    {"exchanges": [
-                        {"input": ("old and boring", 42),
-                        "amount": 1.0},
-                        ]
-                    },
-                ("old and boring", 2):
-                    {"exchanges": [
-                        {"input": ("old and boring", 1),
-                        "amount": 4.0}
-                        ]
-                    }
-                }
-            print relabel_database(data, "shiny new")
-            >> {
-                ("shiny new", 1):
-                    {"exchanges": [
-                        {"input": ("old and boring", 42),
-                        "amount": 1.0},
-                        ]
-                    },
-                ("shiny new", 2):
-                    {"exchanges": [
-                        {"input": ("shiny new", 1),
-                        "amount": 4.0}
-                        ]
-                    }
-                }
-
-        In the example, the exchange to ``("old and boring", 42)`` does not change, as this is not part of the updated data.
-
-        Args:
-            * *data* (dict): The database data to modify
-            * *new_name* (str): The name of the modified database
-
-        Returns:
-            The modified database
-
-        """
-        def relabel_exchanges(obj, new_name):
-            for e in obj['exchanges']:
-                if e["input"] in data:
-                    e["input"] = (new_name, e["input"][1])
-            return obj
-
-        return dict([((new_name, k[1]), self.relabel_exchanges(v, new_name)) \
-            for k, v in data.iteritems()])
-
-    def copy(self, name):
-        """Make a copy of the database.
-
-        Internal links within the database will be updated to match the new database name.
-
-        Args:
-            * *name* (str): Name of the new database.
-
-        """
-        assert name not in databases, ValueError("This database exists")
-        data = self.relabel_data(self.load(), name)
-        new_database = Database(name)
-        new_database.register(
-            format="Brightway2 copy",
-            depends=databases[self.database]["depends"],
-            num_processes=len(data))
-        new_database.write(data)
-
-    def backup(self):
-        """Save a backup to ``backups`` folder.
-
-        Returns:
-            Filepath of backup.
-
-        """
-        filepath = os.path.join(config.request_dir("backups"), self.filename() + \
-            ".%s.backup" % int(time()))
-        with open(filepath, "wb") as f:
-            pickle.dump(self.load(), f, protocol=pickle.HIGHEST_PROTOCOL)
-        return filepath
-
-    def revert(self, version):
-        """Return data to a previous state.
-
-        .. warning:: Reverted changes can be overwritten.
-
-        Args:
-            * *version* (int): Number of the version to revert to.
-
-        """
-        assert version in [x[0] for x in self.versions()], "Version not found"
-        self.backup()
-        databases[self.database]["version"] = version
-        self.process(version)
-
-    def register(self, format, depends, num_processes):
-        """Register a database with the metadata store.
-
-        Databases must be registered before data can be written.
-
-        Args:
-            * *format* (str): Format that the database was converted from, e.g. "Ecospold"
-            * *depends* (list): Names of the databases that this database references, e.g. "biosphere"
-            * *num_processes* (int): Number of processes in this database.
-
-        """
-        assert self.database not in databases
-        databases[self.database] = {
-            "from format": format,
-            "depends": depends,
-            "number": num_processes,
-            "version": 0
-            }
-
-    def deregister(self):
-        """Remove a database from the metadata store. Does not delete any files."""
-        del databases[self.database]
-
-    def validate(self, data):
-        """Validate data. Must be called manually.
-
-        Raises ``voluptuous.Invalid`` if data does not validate.
-
-        Args:
-            * *data* (dict): The data, in its processed form.
-
-        """
-        db_validator(data)
-
-    def filename(self, version=None):
-        """Filename for given version; Default is current.
-
-        Returns:
-            Filename (not path)
-
-        """
-        return "%s.%i.pickle" % (self.database,
-            version or self.version)
-
-    def write(self, data):
-        """Serialize data to disk.
-
-        Args:
-            * *data* (dict): Inventory data
-
-        """
-        if self.database not in databases:
-            raise UnknownObject("This database is not yet registered")
-        databases.increment_version(self.database)
-        mapping.add(data.keys())
-        filepath = os.path.join(config.dir, "intermediate", self.filename())
-        with open(filepath, "wb") as f:
-            pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
-
-    def load(self, version=None):
-        """Load the intermediate data for this database.
-
-        Can also load previous versions of this database's intermediate data.
-
-        Args:
-            * *version* (int): Version of the database to load. Default is *None*, for the latest version.
-
-        Returns:
-            The intermediate data, a dictionary.
-
-        """
-        if self.database not in databases:
-            raise UnknownObject("This database is not yet registered")
-        if version == None and config.p.get("use_cache", False) and \
-                self.database in config.cache:
-            return config.cache[self.database]
-        try:
-            data = pickle.load(open(os.path.join(config.dir, "intermediate",
-                self.filename(version)), "rb"))
-            if version == None and config.p.get("use_cache", False):
-                config.cache[self.database] = data
-            return data
-        except OSError:
-            raise MissingIntermediateData("This version (%i) not found" % version)
-
-    def versions(self):
-        """Get a list of available versions of this database.
-
-        Returns:
-            List of (version, datetime created) tuples.
-
-        """
-        directory = os.path.join(config.dir, "intermediate")
-        files = natural_sort(filter(
-            lambda x: ".".join(x.split(".")[:-2]) == self.database,
-            os.listdir(directory)))
-        return sorted([(int(name.split(".")[-2]),
-            datetime.datetime.fromtimestamp(os.stat(os.path.join(
-            config.dir, directory, name)).st_mtime)) for name in files])
-
-    def process(self, version=None):
-        """Process intermediate data from a Python dictionary to a `NumPy <http://numpy.scipy.org/>`_ `Structured <http://docs.scipy.org/doc/numpy/reference/arrays.classes.html#record-arrays-numpy-rec>`_ `Array <http://docs.scipy.org/doc/numpy/user/basics.rec.html>`_. A structured array (also called record arrays) is a heterogeneous array, where each column has a different label and data type. These structured arrays act as a standard data format for LCA and Monte Carlo calculations, and are the native data format for the Stats Arrays package.
-
-        Processed arrays are saved in the ``processed`` directory.
-
-        Args:
-            * *version* (int, optional): The version of the database to process
-
-        """
-        data = self.load(version)
-        num_exchanges = sum([len(obj["exchanges"]) for obj in data.values()])
-        assert data
-        dtype = [('uncertainty_type', np.uint8),
-            ('input', np.uint32),
-            ('output', np.uint32),
-            ('row', np.uint32),
-            ('col', np.uint32),
-            ('technosphere', np.bool),
-            ('amount', np.float32),
-            ('sigma', np.float32),
-            ('minimum', np.float32),
-            ('maximum', np.float32),
-            ('negative', np.bool)]
-        arr = np.zeros((num_exchanges, ), dtype=dtype)
-        arr['minimum'] = arr['maximum'] = arr['sigma'] = np.NaN
-        count = 0
-        for key in data:
-            for exc in data[key]["exchanges"]:
-                arr[count] = (
-                    exc["uncertainty type"],
-                    mapping[exc["input"]],
-                    mapping[key],
-                    MAX_INT_32,
-                    MAX_INT_32,
-                    exc["technosphere"],
-                    exc["amount"],
-                    exc.get("sigma", np.NaN),
-                    exc.get("minimum", np.NaN),
-                    exc.get("maximum", np.NaN),
-                    exc["amount"] < 1
-                    )
-                count += 1
-
-        filepath = os.path.join(config.dir, "processed", "%s.pickle" % \
-            self.database)
-        with open(filepath, "wb") as f:
-            pickle.dump(arr, f, protocol=pickle.HIGHEST_PROTOCOL)
-
-    def __unicode__(self):
-        return u"Brightway2 database %s" % self.database
-
-    def __str__(self):
-        return self.__unicode__()

File brightway2/errors.py

-class MissingIntermediateData(StandardError):
-    pass
-
-
-class UnknownExchange(StandardError):
-    pass
-
-
-class UnknownObject(StandardError):
-    pass

File brightway2/io/__init__.py

-from import_ecospold import EcospoldImporter
-from import_method import EcospoldImpactAssessmentImporter

File brightway2/io/export_gexf.py

-# -*- coding: utf-8 -*
-from .. import config, Database
-from lxml.builder import ElementMaker
-from lxml.etree import tostring
-import datetime
-import itertools
-import os
-import progressbar
-
-
-class DatabaseToGEXF(object):
-    def __init__(self, database, include_descendants=False):
-        self.database = database
-        self.descendants = include_descendants
-        if self.descendants:
-            raise NotImplemented
-        filename = database + ("_plus" if include_descendants else "")
-        self.filepath = os.path.join(config.request_dir("output"),
-            filename + ".gexf")
-        self.data = Database(self.database).load()
-        self.id_mapping = dict([(key, str(i)) for i, key in enumerate(
-            self.data)])
-
-    def export(self):
-        E = ElementMaker(namespace="http://www.gexf.net/1.2draft",
-            nsmap={None: "http://www.gexf.net/1.2draft"})
-        meta = E.meta(E.creator("Brightway2"), E.description(self.database),
-            lastmodified=datetime.date.today().strftime("%Y-%m-%d"))
-        attributes = E.attributes(
-            E.attribute(id="0", title="category", type="string"),
-            **{"class": "node"}
-        )
-        nodes, edges = self.get_data(E)
-        graph = E.graph(attributes, nodes, edges, mode="static",
-            defaultedgetype="directed")
-        with open(self.filepath, "w") as f:
-            f.write(tostring(E.gexf(meta, graph, version="1.2"),
-                xml_declaration=True, encoding="utf-8",
-                pretty_print=True))
-
-    def get_data(self, E):
-        count = itertools.count()
-        nodes = []
-        edges = []
-
-        widgets = ['Processes: ', progressbar.Percentage(), ' ',
-            progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
-            progressbar.ETA()]
-        pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(self.data)
-            ).start()
-
-        for i, (key, value) in enumerate(self.data.iteritems()):
-            nodes.append(E.node(
-                E.attvalues(
-                    E.attvalue(
-                        value="-".join(value["categories"]),
-                        **{"for": "0"}
-                    )
-                ),
-                id=self.id_mapping[key],
-                label=value["name"]
-            ))
-            for exc in value["exchanges"]:
-                if exc["input"] not in self.id_mapping:
-                    continue
-                else:
-                    edges.append(E.edge(
-                        id=str(count.next()),
-                        source=self.id_mapping[exc["input"]],
-                        target=self.id_mapping[key]
-                    ))
-            pbar.update(i)
-        pbar.finish()
-
-        return E.nodes(*nodes), E.edges(*edges)

File brightway2/io/import_ecospold.py

-# -*- coding: utf-8 -*
-from __future__ import division
-from .. import Database, mapping
-from ..logs import get_logger
-from ..errors import UnknownExchange
-from lxml import objectify
-import math
-import os
-import progressbar
-from stats_toolkit.distributions import *
-
-BIOSPHERE = ("air", "water", "soil", "resource")
-
-
-class EcospoldImporter(object):
-    """Import inventory datasets from ecospold XML format.
-
-    Does not have any arguments; instead, instantiate the class, and then import using the ``importer`` method, i.e. ``EcospoldImporter().importer(filepath)``."""
-    def importer(self, path, name, depends=["biosphere", ]):
-        """Import an inventory dataset, or a directory of inventory datasets.
-
-        Args:
-            *path* (str): A filepath or directory.
-
-        """
-        data = []
-        log = get_logger(name)
-        log.critical(u"Starting import of %s (from %s)" % (name, path))
-        if os.path.isdir(path):
-            files = [os.path.join(path, filter(lambda x: x[-4:].lower(
-                ) == ".xml", os.listdir(path)))]
-        else:
-            files = [path]
-
-        widgets = ['Files: ', progressbar.Percentage(), ' ',
-            progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
-            progressbar.ETA()]
-        pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(files)
-            ).start()
-
-        for index, filename in enumerate(files):
-            root = objectify.parse(open(filename)).getroot()
-
-            if root.tag != '{http://www.EcoInvent.org/EcoSpold01}ecoSpold':
-                # Unrecognized file type
-                log.critical(u"skipping %s - no ecoSpold element" % filename)
-                continue
-
-            for dataset in root.iterchildren():
-                data.append(self._process_dataset(dataset))
-
-            pbar.update(index)
-
-        # Hackish
-        for o in data:
-            try:
-                o["code"] = int(o["code"])
-            except:
-                pass
-
-        # Fix exchanges
-        codes = set([o["code"] for o in data])
-        for ds in data:
-            for exc in ds["exchanges"]:
-                code = exc["code"]
-                # Hack - not work with others?
-                try:
-                    code = int(code)
-                except:
-                    pass
-                if code in codes:
-                    exc["input"] = (name, code)
-                else:
-                    exc["input"] = self._find_in_dependent_database(code,
-                        exc, depends)
-                exc["technosphere"] = exc["input"][0] != "biosphere"
-
-        data = dict([((name, int(o["code"])), o) for o in data])
-
-        manager = Database(name)
-        manager.register(("Ecospold", 1), depends, len(data))
-        manager.write(data)
-
-        pbar.finish()
-
-    def _find_in_dependent_database(self, code, exc, depends):
-        for db in depends:
-            if (db, code) in mapping:
-                return (db, code)
-
-        # Add new biosphere flow if needed
-        if exc["_matching"].get("categories", [None, ])[0] in BIOSPHERE:
-            data = exc["_matching"]
-
-            # Emission or resource
-            resource = data["categories"][0] == "resource"
-            data["type"] = "resource" if resource else "emission"
-
-            # Biosphere flows don't have locations or exchanges
-            del data["location"]
-            data["exchanges"] = []
-
-            # Write modified biosphere database
-            biosphere = Database("biosphere")
-            bio_data = biosphere.load()
-            bio_data[("biosphere", code)] = data
-            biosphere.write(bio_data)
-            return ("biosphere", code)
-        raise UnknownExchange(("The exchange %s couldn't be " + \
-            "matched to this or a depending database") % code)
-
-    def _process_dataset(self, dataset):
-        data = {}
-        ref_func = dataset.metaInformation.processInformation.\
-            referenceFunction
-
-        data["name"] = ref_func.get("name")
-        data["type"] = "process"  # True for all ecospold?
-        data["categories"] = [ref_func.get("category"), ref_func.get(
-            "subCategory")]
-        # Convert ("foo", "unspecified") to ("foo",)
-        while data["categories"][-1] == "unspecified":
-            data["categories"] = data["categories"][:-1]
-        data["location"] = dataset.metaInformation.processInformation.\
-            geography.get("location")
-        data["code"] = dataset.get("number")
-        data["unit"] = ref_func.get("unit")
-        data["exchanges"] = self._process_exchanges(dataset)
-        return data
-
-    def _process_exchanges(self, dataset):
-        data = []
-        # Skip definitional exchange - we assume this already
-        for exc in dataset.flowData.iterchildren():
-            if exc.get("name") == dataset.metaInformation.processInformation.\
-                    referenceFunction.get("name") != None and float(
-                    exc.get("meanValue", 0.)) == 1.0:
-                continue
-
-            this = {
-                "code": int(exc.get("number")),
-                "_matching": {
-                    "categories": (exc.get("category"), exc.get("subCategory")),
-                    "location": exc.get("location"),
-                    "unit": exc.get("unit"),
-                    "name": exc.get("name")
-                    }
-                }
-
-            if exc.get("generalComment"):
-                this["pedigree matrix"] = exc.get("generalComment")
-
-            uncertainty = int(exc.get("uncertaintyType", 0))
-            mean = exc.get("meanValue")
-            min_ = exc.get("minValue")
-            max_ = exc.get("maxValue")
-            sigma = exc.get("standardDeviation95")
-
-            if uncertainty == 1:
-                # Lognormal
-                this.update({
-                    'uncertainty type': LognormalUncertainty.id,
-                    'amount': float(mean),
-                    'sigma': math.log(math.sqrt(float(sigma)))
-                    })
-                if this['sigma'] == 0:
-                    # Bad ecoinvent data
-                    this['uncertainty type'] = UndefinedUncertainty.id
-                    del this["sigma"]
-            elif uncertainty == 2:
-                # Normal
-                this.update({
-                    'uncertainty type': NormalUncertainty.id,
-                    'amount': float(mean),
-                    'sigma': float(sigma) / 2
-                    })
-            elif uncertainty == 3:
-                # Triangular
-                this.update({
-                    'uncertainty type': TriangularUncertainty.id,
-                    'minimum': float(min_),
-                    'maximum': float(max_)
-                    })
-                # Sometimes this isn't included (though it SHOULD BE)
-                if exc.get("mostLikelyValue"):
-                    this['amount'] = float(exc.get("mostLikelyValue"))
-                else:
-                    this['amount'] = float(mean)
-            elif uncertainty == 4:
-                # Uniform
-                this.update({
-                    'uncertainty type': UniformUncertainty.id,
-                    'amount': float(mean),
-                    'minimum': float(min_),
-                    'maximum': float(max_)
-                    })
-            else:
-                # None
-                this.update({
-                    'uncertainty type': UndefinedUncertainty.id,
-                    'amount': float(mean)
-                })
-
-            data.append(this)
-
-        # Sort for consistent order to make import comparisons easier
-        data.sort(key=lambda x: x["input"])
-        return data

File brightway2/io/import_method.py

-# -*- coding: utf-8 -*
-from .. import Database, mapping, Method, methods
-from lxml import objectify
-import os
-import progressbar
-try:
-    import cPickle as pickle
-except:
-    import pickle
-
-
-class EcospoldImpactAssessmentImporter(object):
-    """Import impact assessment methods and weightings from ecospold XML format.
-
-Does not have any arguments; instead, instantiate the class, and then import using the ``importer`` method, i.e. ``EcospoldImpactAssessmentImporter().importer(filepath)``."""
-    def importer(self, path):
-        """Import an impact assessment method, or a directory of impact assessment methods.
-
-        Args:
-            *path* (str): A filepath or directory.
-
-        """
-        if os.path.isdir(path):
-            files = [os.path.join(path, filter(lambda x: x[-4:].lower(
-                ) == ".xml", os.listdir(path)))]
-        else:
-            files = [path]
-
-        self.biosphere_data = Database("biosphere").load()
-        if progressbar:
-            widgets = ['Files: ', progressbar.Percentage(), ' ',
-                progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
-                progressbar.ETA()]
-            pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(files)
-                ).start()
-
-        for index, filepath in enumerate(files):
-            # Note that this is only used for the first root method found in
-            # the file
-            root = objectify.parse(open(filepath)).getroot()
-            for dataset in root.iterchildren():
-                self.add_method(dataset)
-            pbar.update(index)
-
-        pbar.finish()
-
-    def add_method(self, ds):
-        ref_func = ds.metaInformation.processInformation.referenceFunction
-        name = (ref_func.get("category"), ref_func.get("subCategory"),
-            ref_func.get("name"))
-        description = ref_func.get("generalComment") or ""
-        unit = ref_func.get("unit") or ""
-        data = {}
-        for cf in ds.flowData.iterchildren():
-            if ("biosphere", int(cf.get("number"))) not in mapping:
-                # Add new biosphere flow
-                code = int(cf.get("number"))
-                new_flow = {
-                    "name": cf.get("name"),
-                    "categories": (cf.get("category"),
-                        cf.get("subCategory") or "unspecified"),
-                    "code": code,
-                    "unit": cf.get("unit"),
-                    "exchanges": []
-                }
-
-                # Convert ("foo", "unspecified") to ("foo",)
-                while new_flow["categories"][-1] == "unspecified":
-                    new_flow["categories"] = new_flow["categories"][:-1]
-
-                # Emission or resource
-                resource = new_flow["categories"][0] == "resource"
-                new_flow["type"] = "resource" if resource else "emission"
-
-                # Write modified biosphere database
-                biosphere = Database("biosphere")
-                bio_data = biosphere.load()
-                bio_data[("biosphere", code)] = new_flow
-                biosphere.write(bio_data)
-                return ("biosphere", code)
-            data[("biosphere", int(cf.get("number")))] = float(
-                cf.get("meanValue"))
-        assert name not in methods
-        method = Method(name)
-        method.register(unit, description, len(data))
-        method.write(data)
-        method.process()

File brightway2/logs.py

-# -*- coding: utf-8 -*-
-from . import config
-from logging.handlers import RotatingFileHandler
-from utils import random_string
-import codecs
-import datetime
-import logging
-import os
-
-
-def get_logger(name, add_datetime=True, level=logging.INFO):
-    now = datetime.datetime.now()
-    filename = "%s-%s.log" % (name, now.strftime("%d-%B-%Y-%I-%M%p"))
-    handler = RotatingFileHandler(os.path.join(config.dir, 'logs', filename),
-        maxBytes=50000, encoding='utf-8', backupCount=5)
-    formatter = logging.Formatter(
-        "%(asctime)s %(levelname)s %(lineno)d %(message)s")
-    logger = logging.getLogger("name")
-    logger.setLevel(level)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    return logger
-
-
-def get_io_logger(name):
-    """Build a logger that records only relevent data for display later as HTML."""
-    dirname = config.request_dir("logs")
-    assert dirname, "No logs directory found"
-
-    filepath = os.path.join(dirname, "%s.%s.log" % (name, random_string(6)))
-    handler = logging.StreamHandler(codecs.open(filepath, "w", "utf-8"))
-    logger = logging.getLogger(name)
-    logger.setLevel(logging.INFO)
-    handler.setFormatter(logging.Formatter(u"%(message)s"))
-    logger.addHandler(handler)
-    return logger, filepath

File brightway2/meta.py

-from serialization import SerializedDict, PickledDict
-
-
-class Mapping(PickledDict):
-    """A dictionary that maps object ids, like ``("Ecoinvent 2.2", 42)``, to integers. Needed because parameter arrays have integer ``row`` and ``column`` fields.
-
-    File data is saved in ``mapping.pickle``.
-
-    This dictionary does not support setting items directly; instead, use the ``add`` method to add multiple keys."""
-    _filename = "mapping.pickle"
-
-    def add(self, keys):
-        """Add a set of keys. These keys can already be in the mapping; only new keys will be added.
-
-        Args:
-            * *keys* (list): The keys to add.
-
-        """
-        index = max(self.data.values())
-        for i, key in enumerate(keys):
-            if key not in self.data:
-                self.data[key] = index + i + 1
-        self.flush()
-
-    def delete(self, keys):
-        """Delete a set of keys.
-
-        Args:
-            *keys* (list): The keys to delete.
-
-        """
-        for key in keys:
-            del self.data[key]
-        self.flush()
-
-    def __setitem__(self, key, value):
-        raise NotImplemented
-
-    def __unicode__(self):
-        return u"Mapping from databases and methods to parameter indices."
-
-    def __len__(self):
-        return len(self.data)
-
-
-class Databases(SerializedDict):
-    """A dictionary for database metadata. This class includes methods to manage database versions. File data is saved in ``databases.json``."""
-    _filename = "databases.json"
-
-    def increment_version(self, database):
-        """Increment the ``database`` version. Returns the new version."""
-        self.data[database]["version"] += 1
-        self.flush()
-        return self.data[database]["version"]
-
-    def version(self, database):
-        """Return the ``database`` version"""
-        return self.data[database]["version"]
-
-    def __unicode__(self):
-        return u"Brightway2 databases metadata with %i objects" % len(
-            self.data)
-
-
-class Methods(SerializedDict):
-    """A dictionary for method metadata. File data is saved in ``methods.json``."""
-    _filename = "methods.json"
-
-    def pack(self, data):
-        """Transform the dictionary to a list because JSON can't handle lists as keys"""
-        return [(k, v) for k, v in data.iteritems()]
-
-    def unpack(self, data):
-        """Transform data back to a dictionary"""
-        return dict([(tuple(x[0]), x[1]) for x in data])
-
-    def __unicode__(self):
-        return u"Brightway2 methods metadata with %i objects" % len(
-            self.data)
-
-
-mapping = Mapping()
-databases = Databases()
-methods = Methods()
-
-
-def reset_meta():
-    mapping.__init__()
-    databases.__init__()
-    methods.__init__()

File brightway2/method.py

-# -*- coding: utf-8 -*-
-from . import config, mapping, methods
-from copy import copy
-from errors import UnknownObject, MissingIntermediateData
-from utils import MAX_INT_32, random_string
-import numpy as np
-import os
-import string
-from validate import ia_validator
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-
-def abbreviate(names, length=8):
-    abbrev = lambda x: x if x[0] in string.digits else x[0].lower()
-    name = " ".join(names).split(" ")[0].lower() + \
-        "".join([abbrev(x) for x in " ".join(names).split(" ")[1:]])
-    return name + "-" + random_string(length)
-
-
-class Method(object):
-    """A manager for a method. This class can register or deregister methods, write intermediate data, process data to parameter arrays, validate, and copy methods.
-
-    The Method class never holds intermediate data, but it can load or write intermediate data. The only attribute is *method*, which is the name of the method being managed.
-
-    Instantiation does not load any data. If this method is not yet registered in the metadata store, a warning is written to ``stdout``.
-
-    Methods are hierarchally structured, and this structure is preserved in the method name. It is a tuple of strings, like ``('ecological scarcity 2006', 'total', 'natural resources')``.
-
-    Args:
-        * *method* (tuple): Name of the method to manage. Must be a tuple of strings.
-
-    """
-    def __init__(self, method, *args, **kwargs):
-        self.method = method
-        if self.method not in methods:
-            print "Warning: %s not a currently installed method" % (
-                " : ".join(method))
-
-    def get_abbreviation(self):
-        """Abbreviate a method identifier (a tuple of long strings) for a filename. Random characters are added because some methods have similar names which would overlap when abbreviated."""
-        try:
-            return methods[self.method]["abbreviation"]
-        except KeyError:
-            raise UnknownObject("This method is not yet registered")
-
-    def copy(self, name=None):
-        """Make a copy of the method.
-
-        Args:
-            * *name* (tuple, optional): Name of the new method.
-
-        """
-        name = name or self.method[:-1] + ("Copy of " + self.method[-1],)
-        new_method = Method(name)
-        metadata = copy(methods[self.method])
-        del metadata["abbreviation"]
-        new_method.register(**metadata)
-        new_method.write(self.load())
-
-    def register(self, unit, description="", num_cfs=0):
-        """Register a database with the metadata store.
-
-        Methods must be registered before data can be written.
-
-        Args:
-            * *unit* (str): Unit for impact assessment CFs
-            * *description* (str): Description
-            * *num_cfs* (int): Number of characterization factors
-
-        """
-        assert self.method not in methods
-        methods[self.method] = {
-            "abbreviation": abbreviate(self.method),
-            "unit": unit,
-            "description": description,
-            "num_cfs": num_cfs
-            }
-
-    def deregister(self):
-        """Remove a method from the metadata store. Does not delete any files."""
-        del methods[self.method]
-
-    def validate(self, data):
-        """Validate data. Must be called manually.
-
-        Args:
-            * *data* (dict): The data, in its processed form.
-
-        """
-        ia_validator(data)
-        return True
-
-    def write(self, data):
-        """Serialize data to disk.
-
-        Args:
-            * *data* (dict): Inventory data
-
-        """
-        if self.method not in methods:
-            raise UnknownObject("This database is not yet registered")
-        mapping.add(data.keys())
-        filepath = os.path.join(config.dir, "intermediate",
-            "%s.pickle" % self.get_abbreviation())
-        with open(filepath, "wb") as f:
-            pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
-
-    def load(self):
-        """Load the intermediate data for this method.
-
-        Returns:
-            The intermediate data, a dictionary.
-
-        """
-        try:
-            return pickle.load(open(os.path.join(config.dir, "intermediate",
-                "%s.pickle" % self.get_abbreviation()), "rb"))
-        except OSError:
-            raise MissingIntermediateData("Can't load intermediate data")
-
-    def process(self):
-        """Process intermediate data from a Python dictionary to a `NumPy <http://numpy.scipy.org/>`_ `Structured <http://docs.scipy.org/doc/numpy/reference/arrays.classes.html#record-arrays-numpy-rec>`_ `Array <http://docs.scipy.org/doc/numpy/user/basics.rec.html>`_. A structured array (also called record arrays) is a heterogeneous array, where each column has a different label and data type. These structured arrays act as a standard data format for LCA and Monte Carlo calculations, and are the native data format for the Stats Arrays package.
-
-        Processed arrays are saved in the ``processed`` directory.
-
-        Although it is not standard to provide uncertainty distributions for impact assessment methods, the structured array includes uncertainty fields.
-
-        """
-        data = pickle.load(open(os.path.join(config.dir, "intermediate",
-            "%s.pickle" % self.get_abbreviation()), "rb"))
-        assert data
-        dtype = [('uncertainty_type', np.uint8),
-            ('flow', np.uint32),
-            ('index', np.uint32),
-            ('amount', np.float32),
-            ('sigma', np.float32),
-            ('minimum', np.float32),
-            ('maximum', np.float32),
-            ('negative', np.bool)]
-        arr = np.zeros((len(data), ), dtype=dtype)
-        arr['minimum'] = arr['maximum'] = arr['sigma'] = np.NaN
-        for i, (key, value) in enumerate(data.iteritems()):
-            arr[i] = (
-                0,
-                mapping[key],
-                MAX_INT_32,
-                value,
-                np.NaN,
-                np.NaN,
-                np.NaN,
-                False
-                )
-        filepath = os.path.join(config.dir, "processed", "%s.pickle" % \
-            self.get_abbreviation())
-        with open(filepath, "wb") as f:
-            pickle.dump(arr, f, protocol=pickle.HIGHEST_PROTOCOL)

File brightway2/proxies/__init__.py

-# -*- coding: utf-8 -*
-from array import ArrayProxy, OneDimensionalArrayProxy, ListArrayProxy
-from sparse import CompressedSparseMatrixProxy, SparseMatrixProxy

File brightway2/proxies/array.py

-# -*- coding: utf-8 -*
-import numpy as np
-import unittest
-
-
-class DifferentIndexTypesError(StandardError):
-    pass
-
-
-class InconsistentSlicingError(StandardError):
-    pass
-
-
-class ArrayProxy(object):
-    """
-Provides a dictionary-based interface from database row ids to array indices.
-
-ArrayProxy provides a matrix or array whose indices are translated through a
-lookup dictionary. Slices are not generally supported.
-    """
-    def __init__(self, data, row_dict, col_dict=None):
-        super(ArrayProxy, self).__init__()
-        self.data = data
-        self.row_dict = row_dict
-        if col_dict:
-            self.col_dict = col_dict
-
-    def translate(self, obj, type="row"):
-        if hasattr(obj, "id"):
-            obj = obj.id
-        if isinstance(obj, slice):
-            if obj == slice(None):
-                return obj
-            else:
-                # Could support for slicing using Django's SortedDict
-                # (http://code.djangoproject.com/browser/django/trunk/django/utils/datastructures.py)
-                # or other sorted dictionary implementations, e.g.
-                # http://code.activestate.com/recipes/496761/ or
-                # http://www.voidspace.org.uk/downloads/odict.py
-                # However, this is not desperately needed...
-                raise NotImplementedError("Slices are not supported, " + \
-                    "because their meaning is unclear when translated into" + \
-                    " array indices. If you are confident, you can call " + \
-                    "slices directly on self.data.")
-        elif isinstance(obj, (list, tuple)):
-            return tuple([self.translate(_obj, type) for _obj in obj])
-        else:  # No longer test for integer keys; subclasses can use strings
-            try:
-                if type == "row":
-                    return self.row_dict[obj]
-                elif type == "col":
-                    return self.col_dict[obj]
-            except KeyError:
-                raise KeyError("Provided object %s is not a valid %s key" \
-                    % (obj, type))
-
-    def __getitem__(self, *args):
-        assert len(args) == 1
-        x = self.translate(args[0][0], "row")
-        y = self.translate(args[0][1], "col")
-        return self.data.__getitem__((x, y),)
-
-    def __setitem__(self, *args):
-        assert len(args) == 2
-        x = self.translate(args[0][0], "row")
-        y = self.translate(args[0][1], "col")
-        return self.data.__setitem__((x, y), args[1])
-
-    def __repr__(self):
-        return '<%s for %s>' % (self.__class__.__name__, self.data.__repr__(
-            )[1:-1])
-
-    def __str__(self):
-        return self.data.__str__()
-
-    # Numpy functions
-    def any(self, *args, **kwargs):
-        return self.data.any(*args, **kwargs)
-
-    def all(self, *args, **kwargs):
-        return self.data.all(*args, **kwargs)
-
-    def sum(self, *args, **kwargs):
-        return self.data.sum(*args, **kwargs)
-
-    def min(self, *args, **kwargs):
-        # This should work for both ndarrays and sparse matrices
-        try:
-            m = self.data.min(*args, **kwargs)
-        except AttributeError:
-            m = self.data.data.min(*args, **kwargs)
-        while isinstance(m, np.ndarray):
-            m = m[0]
-        return m
-
-    def max(self, *args, **kwargs):
-        try:
-            m = self.data.max(*args, **kwargs)
-        except AttributeError:
-            m = self.data.data.max(*args, **kwargs)
-        while isinstance(m, np.ndarray):
-            m = m[0]
-        return m
-
-    def cumsum(self, *args, **kwargs):
-        return self.data.cumsum(*args, **kwargs)
-
-    def mean(self, *args, **kwargs):
-        return self.data.mean(*args, **kwargs)
-
-    def row(self, id):
-        return self.row_dict[id]
-
-    def col(self, id):
-        return self.col_dict[id]
-
-    @property
-    def shape(self):
-        return self.data.shape
-
-    @property
-    def row_dict_rev(self):
-        """Build only upon demand"""
-        if not hasattr(self, "_row_dict_rev"):
-            self._row_dict_rev = dict(zip(self.row_dict.values(),
-                self.row_dict.keys()))
-        return self._row_dict_rev
-
-    @property
-    def col_dict_rev(self):
-        if not hasattr(self, "_col_dict_rev"):
-            self._col_dict_rev = dict(zip(self.col_dict.values(),
-                self.col_dict.keys()))
-        return self._col_dict_rev
-
-
-class OneDimensionalArrayProxy(ArrayProxy):
-    """
-A special case of ArrayProxy limited to one-dimensional arrays.
-
-Used for supply and demand arrays in LCA calculations.
-    """
-    def __init__(self, data, row_dict):
-        if not len(data.shape) == 1:
-            raise AttributeError("Must only be used for one-dimensional array")
-        super(OneDimensionalArrayProxy, self).__init__(data, row_dict)
-
-    def __getitem__(self, *args):
-        assert len(args) == 1
-        x = self.translate(args[0], "row")
-        return self.data.__getitem__((x,),)
-
-    def __setitem__(self, *args):
-        assert len(args) == 2
-        x = self.translate(args[0], "row")
-        self.data.__setitem__((x,), args[1])
-
-
-class ListArrayProxy(object):
-    """
-An interface to a list of objects that translates lookeups from foo[bar,baz] to
-foo.indices.index(bar)[baz]. If baz is a slice, returns a generator.
-
-Takes list_, the list of objects, and optionally index_objs, which is an
-iterable of objects used as indices to list_. index_objs must all be of the
-same type.
-    """
-
-    __slots__ = ["indices", "list", "index_type"]
-
-    def __init__(self, list_, index_objs=None):
-        self.list = list_
-        if index_objs:
-            self.check_indice_types(index_objs)
-            if len(index_objs) != len(list_):
-                raise ValueError("index_objs must have same length as list_")
-            self.indices = list(index_objs)
-        else:
-            self.indices = None
-
-    def check_indice_types(self, objs):
-        l = [type(x) for x in objs]
-        if not len(set(l)) == 1:
-            raise DifferentIndexTypesError
-
-    def sum(self):
-        return sum([sum(obj) for obj in self.list])
-
-    def __unicode__(self):
-        if self.indices:
-            return "<ListArrayProxy for %s>" % self.indices[0]
-        else:
-            return "<ListArrayProxy for unknown objects (id %s)>" % id(self)
-
-    def translate_slice(self, sl):
-        """Possible translate slice arguments into self.indices terms"""
-        if sl.start in self.indices:
-            start = self.indices.index(sl.start)
-            start_translated = True
-        else:
-            start = sl.start
-            start_translated = False
-        if sl.stop in self.indices:
-            stop = self.indices.index(sl.stop)
-            stop_translated = True
-        else:
-            stop = sl.stop
-            stop_translated = False
-        if start_translated != stop_translated and start != None and stop != \
-                None:
-            raise InconsistentSlicingError(
-                "Only one slice element could be found in the indices")
-        return slice(start, stop, sl.step)
-
-    def __getitem__(self, args):
-        if args == None:
-            raise SyntaxError
-
-        if isinstance(args, tuple):
-            list_pos = args[0]
-        else:
-            list_pos = args
-
-        if self.indices and isinstance(list_pos, slice):
-            list_pos = self.translate_slice(list_pos)
-        elif self.indices and list_pos in self.indices:
-            list_pos = self.indices.index(list_pos)
-
-        if not isinstance(args, tuple):
-            return self.list[list_pos]
-        elif isinstance(list_pos, slice):
-            return (obj.__getitem__(*args[1:]) for obj in self.list[list_pos])
-        else:
-            return self.list[list_pos].__getitem__(*args[1:])
-
-    def __setitem__(self):
-        raise NotImplementedError
-
-    def __iter__(self):
-        return iter(self.list)
-
-    def __len__(self):
-        return len(self.list)
-
-    def __repr__(self):
-        return self.__unicode__()
-
-
-class ArrayProxyTest(unittest.TestCase):
-    def test_array_proxy(self):
-        a = np.zeros((3, 3))
-        for x in range(3):
-            for y in range(3):
-                a[x, y] = x + 3 * y
-        a = ArrayProxy(a, {10: 0, 11: 1, 12: 2}, {20: 0, 21: 1, 22: 2})
-        self.assertEqual(a[10, 20], 0)
-        self.assertTrue(np.allclose(a[11, (20, 21)], np.array((1, 4))))
-
-    def test_array_proxy_object(self):
-        class Dummy(object):
-            def __init__(self, id):
-                self.id = id
-
-        a = np.zeros((3, 3))
-        for x in range(3):
-            for y in range(3):
-                a[x, y] = x + 3 * y
-        a = ArrayProxy(a, {10: 0, 11: 1, 12: 2}, {20: 0, 21: 1, 22: 2})
-        obj = Dummy(10)
-        self.assertEqual(a[obj, 20], 0)
-
-    def test_array_proxy_key_error(self):
-        a = np.zeros((3, 3))
-        for x in range(3):
-            for y in range(3):
-                a[x, y] = x + 3 * y
-        a = ArrayProxy(a, {10: 0, 11: 1, 12: 2}, {20: 0, 21: 1, 22: 2})
-        self.assertRaises(KeyError, a.__getitem__, (20, 10))
-
-        class Vanilla(object):
-            pass
-        v = Vanilla()
-        self.assertRaises(KeyError, a.__getitem__, (v, 20))
-
-    def test_array_proxy_slices(self):
-        a = np.zeros((3, 3))
-        for x in range(3):
-            for y in range(3):
-                a[x, y] = x + 3 * y
-        a = ArrayProxy(a, {10: 0, 11: 1, 12: 2}, {20: 0, 21: 1, 22: 2})
-        self.assertRaises(NotImplementedError, a.__getitem__, (slice(None, 11,
-            None), 20))
-        # Note: np slices don't preserve shape!
-        self.assertTrue(np.allclose(a[:, 21], np.array((3, 4, 5,))))
-        self.assertTrue(np.allclose(a[11, :], np.array((1, 4, 7))))
-
-    def test_reverse_dict(self):
-        a = np.zeros((3, 3))
-        for x in range(3):
-            for y in range(3):
-                a[x, y] = x + 3 * y
-        a = ArrayProxy(a, {10: 0, 11: 1, 12: 2}, {20: 0, 21: 1, 22: 2})
-        self.assertFalse(hasattr(a, "_row_dict_rev"))
-        self.assertFalse(hasattr(a, "_col_dict_rev"))
-        self.assertTrue(a.row_dict_rev)
-        self.assertTrue(a.col_dict_rev)
-        self.assertEquals(a.row_dict_rev[0], 10)
-        self.assertEquals(a.col_dict_rev[0], 20)
-
-    def test_one_dimensional_proxy(self):
-        b = np.zeros((3,))
-        for x in range(3):
-            b[x] = x + 3
-        b = OneDimensionalArrayProxy(b, {10: 0, 11: 1, 12: 2})
-        self.assertEqual(b[11], 4)
-        self.assertTrue(np.allclose(b[:], np.array((3, 4, 5))))
-        b[11] = 13
-        self.assertEqual(b[11], 13)
-        self.assertTrue(np.allclose(b[:], np.array((3, 13, 5))))
-        b = np.zeros((3, 3))
-        self.assertRaises(AttributeError, OneDimensionalArrayProxy, b, {})
-        self.assertRaises(TypeError, OneDimensionalArrayProxy, b, {}, {})
-
-
-class ListArrayProxyTest(unittest.TestCase):
-    def test_list_array_proxy_without_indices(self):
-        l = ListArrayProxy((range(10), range(10)))
-        self.assertEqual(l[0], range(10))
-        self.assertEqual(list(l[:, 1]), [1, 1])
-        self.assertEqual(list(l[:, :]), [range(10), range(10)])
-        self.assertEqual(list(l[:, 4:6]), [range(10)[4:6], range(10)[4:6]])
-
-    def test_objects_as_indices(self):
-        class A(object):
-            pass
-        m = A()
-        n = A()
-        l = ListArrayProxy((range(5), range(5, 10)), [m, n])
-        self.assertEqual(l[n], range(5, 10))
-        self.assertEqual(l.sum(), sum(range(10)))
-        self.assertEqual(l[n, :2], [5, 6])
-
-    def test_mismatched_index_length(self):
-        self.assertRaises(ValueError, ListArrayProxy, range(5), range(4))
-
-    def test_list_array_proxy_with_indices(self):
-        class A(object):
-            pass
-
-        class B(object):
-            pass
-        m = A()
-        o = B()
-        self.assertRaises(DifferentIndexTypesError, ListArrayProxy, ((), ()),
-            [m, o])
-
-    def test_slices_with_indices(self):
-        l = ListArrayProxy((range(3), range(5), range(7)),
-            (3, 5, 7))
-        self.assertEqual(l[:], (range(3), range(5), range(7)))
-        self.assertRaises(InconsistentSlicingError, l.__getitem__, slice(3, 4))
-        self.assertFalse(l[:3])
-        self.assertEqual(l[:5], (range(3),))
-        self.assertEqual([x for x in l[::2]], [range(3), range(7)])
-        self.assertEqual([x for x in l[5:]], [range(5), range(7)])

File brightway2/proxies/sparse.py

-# -*- coding: utf-8 -*
-import numpy as np
-import scipy.sparse
-import unittest
-from . import ArrayProxy
-
-
-class SparseMatrixProxy(ArrayProxy):
-    """
-Provides a dictionary-based interface from database row ids to array indices 
-for sparse matrices. Does not assume a certain sparsity structure.
-    """
-    def __init__(self, data, row_dict, col_dict, *args, **kwargs):
-        self.row_dict = row_dict
-        self.col_dict = col_dict
-        self.data = data
-        try:
-            self.format = self.data.getformat()
-        except AttributeError:
-            raise TypeError("Must pass a Scipy sparse matrix")
-
-    def __getitem__(self, *args):
-        assert len(args) == 1
-        x = self.translate(args[0][0], "row")
-        y = self.translate(args[0][1], "col")
-
-        if x == slice(None) or y == slice(None):
-            raise NotImplementedError("SparseMatrixProxy doesn't support " + \
-                "slices; use CompressedSparseMatrixProxy for slices")
-        return self.data.__getitem__((x, y),)
-
-    def __setitem__(self, *args):
-        assert len(args) == 2
-        x = self.translate(args[0][0], "row")
-        y = self.translate(args[0][1], "col")
-
-        if x == slice(None) or y == slice(None):
-            raise NotImplementedError, "SparseMatrixProxy doesn't support " +\
-                "assignment by slice"
-        self.data.__setitem__((x,y), args[1])
-
-    def get_row_as_dict(self, row):
-        raise NotImplementedError, "Use CompressedSparseMatrixProxy for " + \
-            "dictionary matrix slices"
-
-    def get_col_as_dict(self, col):
-        raise NotImplementedError, "Use CompressedSparseMatrixProxy for " + \
-            "dictionary matrix slices"
-
-    def toarray(self):
-        return self.data.toarray()
-
-    def todense(self):
-        return self.data.todense()
-
-    @property
-    def nnz(self):
-        # Used to use getnzmax, but that seems to be missing from newest scipy
-        return self.data.getnnz()
-
-
-class CompressedSparseMatrixProxy(SparseMatrixProxy):
-    """
-Subclass of SparseMatrixProxy for CSR (comma separated row) or CSC (comma 
-separated column) matrices.
-
-This class will create, on demand, the complementary matrix type (e.g. CSC for 
-CSR) for efficient slicing and multiple assignment. If a matrix is changed, a 
-*dirty* flag is set that tells the class to re-create the complementary matrix 
-before accessing its information. All updates to complementary matrices are 
-lazy.
-    """
-
-    __slots__ = ['row_dict', 'col_dict', 'format', 'dirty', '_csr', '_csc']
-
-    def __init__(self, data, row_dict, col_dict, *args, **kwargs):
-        self.row_dict = row_dict
-        self.col_dict = col_dict
-        self.dirty = True
-        # try:
-        self.format = data.getformat()
-        if self.format == "csc":
-            self._csc = data
-        else:
-            self._csr = data.tocsr()
-            self.format = "csr"
-
-    def __getitem__(self, *args):
-        assert len(args) == 1
-        x = self.translate(args[0][0], "row")
-        y = self.translate(args[0][1], "col")
-
-        if isinstance(x, slice) and isinstance(y, slice):
-            raise NotImplementedError, \
-                "Convert to dense matrix to do slices on multiple dimensions"
-        elif isinstance(x, slice):
-            if self.dirty and self.format == "csr":
-                self._csc = self._csr.tocsc()
-            return self._csc.__getitem__((x,y),)
-        elif isinstance(y, slice):
-            if self.dirty and self.format == "csc":
-                self._csr = self._csc.tocsr()
-            return self._csr.__getitem__((x,y),)
-        else:
-            return self.data.__getitem__((x,y),)
-
-    def _get_data(self):
-        if self.format == "csr":
-            return self._csr
-        else:
-            return self._csc
-
-    def _set_data(self, data):
-        self.dirty = True
-        if self.format == "csr":
-            self._csr = data
-        else:
-            self._csc = data
-
-    data = property(_get_data, _set_data)
-
-    def __setitem__(self, *args):
-        # Set dirty flag to indicate that matrix has changed, and need to
-        # recompute the partner matrix if is accessed
-        self.dirty = True
-        # TODO: Warn if setting slicing on compressed sparse matrix?
-        super(CompressedSparseMatrixProxy, self).__setitem__(*args)
-
-    def get_row_as_dict(self, row):
-        obj = self.__getitem__((row,slice(None)),)
-        return dict([(self.row_dict_rev[obj.indices[index]], obj.data[index] 
-            ) for index in xrange(obj.indices.shape[0])])
-
-    def get_col_as_dict(self, col):
-        obj = self.__getitem__((slice(None),col),)
-        return dict([(self.col_dict_rev[obj.indices[index]], obj.data[index] 
-            ) for index in xrange(obj.indices.shape[0])])
-
-
-class SparseMatrixProxyTest(unittest.TestCase):
-    def test_sparse_matrix_proxy(self):
-        mat = scipy.sparse.lil_matrix((3,3))
-        for x in range(3):
-            for y in range(3):
-                if x == y: continue
-                mat[x,y] = x+3*y
-        mat = SparseMatrixProxy(mat, {10: 0, 11: 1, 12: 2}, {20: 0, 21: 1, 
-            22: 2})
-        self.assertEqual(mat[10, 20], 0)
-        self.assertEqual(mat[11, 22], 7)
-        self.assertEqual(mat.nnz, 6)
-        mat[11,21] = 3
-        self.assertEqual(mat[11,21], 3)
-        self.assertRaises(NotImplementedError, mat.__getitem__, (slice(None), 
-            21))
-        self.assertRaises(NotImplementedError, mat.__setitem__, (slice(None), 
-            21), 1)
-
-    def test_compressed_sparse_matrix_proxy(self):
-        c = scipy.sparse.lil_matrix((3,3))
-        for x in range(3):
-            for y in range(3):
-                if x == y: continue
-                c[x,y] = x+3*y
-        c = CompressedSparseMatrixProxy(c, {10: 0, 11: 1, 12: 2}, 
-            {20: 0, 21: 1, 22: 2})
-        self.assertEqual(c[10, 20], 0)
-        self.assertEqual(c[11, 22], 7)
-        self.assertEqual(c.nnz, 6)
-        self.assertTrue(isinstance(c.data, scipy.sparse.csr.csr_matrix))
-        self.assertTrue(np.allclose(c[:,21].todense(), np.array(((3,),(
-            0,),(5,)))))
-        self.assertTrue(np.allclose(c[11,:].todense(), np.array((1,0,7))))
-        c[11,21] = 3
-        assert c[11,21] == 3

File brightway2/query.py

-# -*- coding: utf-8 -*
-import collections
-import operator
-import itertools
-
-operators = {
-    "<": operator.lt,
-    "<=": operator.le,
-    "==": operator.eq,
-    "is": operator.eq,
-    "not": operator.ne,
-    "!=": operator.ne,
-    ">=": operator.ge,
-    ">": operator.gt,
-    "in": operator.contains,
-    "notin": lambda x, y: not operator.contains(x, y),
-    "iin": lambda x, y: x.lower() in y.lower(),
-    "inot": lambda x, y: x.lower() != y.lower(),
-    "iis": lambda x, y: x.lower() == y.lower(),
-    "len": lambda x, y: len(x) == y,
-}
-
-
-def try_op(f, x, y):
-    try:
-        return f(x, y)
-    except:
-        return False
-
-
-class Dictionaries(object):
-    """A wrapper for a single call to iteritems() for multiple dictionaries.
-
-    Useful for a ``Query`` across multiple databases."""
-    def __init__(self, *args):
-        self.dicts = args
-
-    def iteritems(self):
-        return itertools.chain(*[x.iteritems() for x in self.dicts])
-
-
-class Result(object):
-    """A container that wraps a filtered dataset. Returned by a calling a ``Query`` object. A result object functions like a read-only dictionary; you can call ``Result[some_key]``, or ``some_key in Result``, or ``len(Result)``.
-
-    The dataset can also be sorted, using ``sort(field)``; the underlying data is then a ``collections.OrderedDict``.
-
-    Args:
-        * *result* (dict): The filtered dataset.
-
-    """
-    def __init__(self, result):
-        self.result = result
-
-    def __str__(self):
-        return u"Query result with %i entries" % len(self.result)
-
-    def __repr__(self):
-        if len(self.result) > 20:
-            data = dict([(key, self.result[key]) for key in \
-                self.result.keys()[:20]])
-        elif not len(self.result):
-            return "Query result:\n\tNo query results found."
-        else:
-            data = self.result
-        return "Query result: (total %i)\n" % len(self.result) \
-            + "\n".join(["%s: %s" % (key, data[key]["name"]) for key in data])
-
-    def sort(self, field, reverse=False):
-        """Sort the filtered dataset. Operates in place; does not return anything.
-
-        Args:
-            * *field* (str): The key used for sorting.
-            * *reverse* (bool, optional): Reverse normal sorting order.
-
-        """
-        self.result = collections.OrderedDict(sorted(self.result.iteritems(),
-            key=lambda t: t[1].get(field, None), reverse=reverse))
-
-    # Generic dictionary methods
-    def __len__(self):
-        return len(self.result)
-
-    def __iter__(self):
-        return iter(self.result)
-
-    def keys(self):
-        return self.result.keys()
-
-    def iteritems(self):
-        return self.result.iteritems()
-
-    def __getitem__(self, key):
-        return self.result[key]
-
-    def __contains__(self, key):
-        return key in self.result
-
-
-class Query(object):
-    """A container for a set of filters applied to a dataset.
-
-    Filters are applied by calling the ``Query`` object, and passing the dataset to filter as the argument. Calling a ``Query`` with some data returns a ``Result`` object with the filtered dataset.
-
-    Args:
-        * *filters* (filters): One or more ``Filter`` objects.
-
-    """
-    def __init__(self, *filters):
-        self.filters = list(filters)
-
-    def add(self, filter_):
-        """Add another filter.
-
-        Args:
-            *filter_* (``Filter``): A Filter object.
-
-        """
-        self.filters.append(filter_)
-
-    def __call__(self, data):
-        for filter_ in self.filters:
-            data = filter_(data)
-        return Result(data)
-
-
-class Filter(object):
-    """A filter on a dataset.
-
-    The following functions are supported:
-
-        * "<", "<=", "==", ">", ">=": Mathematical relations
-        * "is", "not": Identity relations. Work on any Python object.
-        * "in", "notin": List or string relations.
-        * "iin", "iis", "inot": Case-insensitive string relations.
-        * "len": Length relation.
-
-    In addition, any function which defines a relationship between an input and an output can also be used.
-
-    Examples:
-
-        * All ``name`` values are *foo*: ``Filter("name", "is", "foo")``
-        * All ``name`` values include the string *foo*: ``Filter("name", "in", "foo")``
-        * Category (a list of categories and subcategories) includes *foo*: ``Filter("category", "in", "foo")``
-
-    Args:
-        * *key* (str): The field to filter on.
-        * *function* (str or object): One of the pre-defined filters, or a callable object.
-        * *value* (object): The value to test against.
-
-    Returns:
-        A ``Result`` object which wraps a new data dictionary.
-
-    """
-    def __init__(self, key, function, value):
-        self.key = key
-        self.function = function
-        self.value = value
-        if not callable(function):
-            self.function = operators.get(function, None)
-        if not self.function:
-            raise ValueError("No valid function found")
-
-    def __call__(self, data):
-        return dict(((k, v) for k, v in data.iteritems() if try_op(
-            self.function, v.get(self.key, None), self.value)))
-
-
-# class Exchange(object):
-#     def __init__(self, *args):
-#         self.filters = args
-
-#     def __call__(self, data):
-#         """All filters should pass for at least one exchange"""
-#         return dict([
-#             (k, v) for k, v in data.iteritems() if \
-#                 any([
-#                     all([f.filter(e) for f in self.filters]) \
-#                     for e in v["exchanges"]
-#                 ])
-#             ])

File brightway2/serialization.py

-# -*- coding: utf-8 -*-
-import os
-from . import config
-from time import time
-try:
-    import anyjson
-except ImportError:
-    anyjson = None
-    import json
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-
-class JsonWrapper(object):
-    @classmethod
-    def dump(self, data, file):
-        with open(file, "w") as f:
-            if anyjson:
-                f.write(anyjson.serialize(data))
-            else:
-                json.dump(data, f, indent=2)
-
-    @classmethod
-    def load(self, file):
-        if anyjson:
-            return anyjson.deserialize(open(file).read())
-        else:
-            return json.load(open(file))
-
-    @classmethod
-    def dumps(self, data):
-        if anyjson:
-            return anyjson.serialize(data)
-        else:
-            return json.dumps(data)
-
-    @classmethod
-    def loads(self, data):
-        if anyjson:
-            return anyjson.deserialize(data)
-        else:
-            return json.loads(data)
-
-
-class SerializedDict(object):
-    """Base class for dictionary that can be serlialized to of unserialized from disk. Uses JSON as its storage format. Has most of the methods of a dictionary.
-
-    Upon instantiation, the serialized dictionary is read from disk."""
-    def __init__(self):
-        if not getattr(self, "_filename"):
-            raise NotImplemented("SerializedDict must be subclassed, and the filename must be set.")
-        self._filepath = os.path.join(config.dir, self._filename)
-        self.load()
-
-    def load(self):
-        """Load the serialized data. Creates the file if not yet present."""
-        try:
-            self.data = self.deserialize()
-        except IOError:
-            # Create if not present
-            self.data = {}
-            self.flush()
-
-    def flush(self):
-        """Serialize the current data to disk."""
-        self.serialize()
-
-    @property
-    def list(self):
-        """List the keys of the dictionary. This is a property, and does not need to be called."""
-        return sorted(self.data.keys())
-
-    def __getitem__(self, key):
-        return self.data[key]
-
-    def __setitem__(self, key, value):
-        self.data[key] = value
-        self.flush()
-
-    def __contains__(self, key):
-        return key in self.data
-
-    def __str__(self):
-        return self.__unicode__()
-
-    def __delitem__(self, name):
-        del self.data[name]
-        self.flush()
-
-    def __len__(self):
-        return len(self.data)
-
-    def iteritems(self):
-        return self.data.iteritems()
-
-    def keys(self):
-        return self.data.keys()
-
-    def values(self):
-        return self.data.values()
-
-    def serialize(self, filepath=None):
-        """Method to do the actual serialization. Can be replaced with other serialization formats.
-
-        Args:
-            * *filepath* (str, optional): Provide an alternate filepath (e.g. for backup).
-
-        """
-        JsonWrapper.dump(self.pack(self.data), filepath or self._filepath)
-
-    def deserialize(self):
-        """Load the serialized data. Can be replaced with other serialization formats."""
-        return self.unpack(JsonWrapper.load(self._filepath))
-
-    def pack(self, data):
-        """Transform the data, if necessary. Needed because JSON must have strings as dictionary keys."""
-        return data
-
-    def unpack(self, data):
-        """Return serialized data to true form."""
-        return data
-
-    def backup(self):
-        """Write a backup version of the data to the ``backups`` directory."""
-        filepath = os.path.join(config.dir, "backups",
-            self._filename + ".%s.backup" % int(time()))
-        self.serialize(filepath)
-
-
-class PickledDict(SerializedDict):
-    """Subclass of ``SerializedDict`` that uses the pickle format instead of JSON."""
-    def serialize(self):
-        with open(self._filepath, "wb") as f:
-            pickle.dump(self.pack(self.data), f,
-                protocol=pickle.HIGHEST_PROTOCOL)
-
-    def deserialize(self):
-        return self.unpack(pickle.load(open(self._filepath, "rb")))

File brightway2/tests/__init__.py

-# -*- coding: utf-8 -*-
-from base import BW2Test
-from database import DatabaseTest

File brightway2/tests/base.py

-# -*- coding: utf-8 -*-
-from .. import config
-import tempfile
-import unittest
-
-
-class BW2Test(unittest.TestCase):
-    def __setup__(self):
-        config.dir = tempfile.mkdtemp()
-        config.create_basic_directories()
-        self.extra_setup()
-
-    def extra_setup(self):
-        pass

File brightway2/tests/database.py

-# -*- coding: utf-8 -*-
-from . import BW2Test
-from .. import Database, databases
-from fixtures import food, biosphere
-
-
-class DatabaseTest(BW2Test):
-    def extra_setup(self):
-        d = Database("biosphere")
-        d.register("Tests", [], len(biosphere))
-        d.write(biosphere)
-        d = Database("food")
-        d.register("Tests", ["biosphere"], len(food))
-        d.write(food)
-
-    def test_setup(self):
-        self.assertEqual(len(databases), 2)
-
-    def test_copy(self):
-        pass
-
-    def test_relabel_data(self):
-        pass
-
-    def test_revert(self):
-        pass
-
-    def test_register(self):
-        pass
-
-    def test_load(self):
-        pass
-
-    def test_versions(self):
-        pass

File brightway2/tests/fixtures.py

-biosphere = {
-    ("biopshere", 1): {
-        'categories': ['things'],
-        'code': 1,
-        'exchanges': [],