Commits

Chris Mutel committed 5793b0a

0.9.1: Add uploading logs to help debug stuff

Comments (0)

Files changed (6)

bw2data/io/bw2package.py

 # -*- coding: utf-8 -*
 from .. import Database, databases, config, JsonWrapper, methods, Method
+from ..logs import get_logger
 from ..utils import database_hash, download_file
+from time import time
 import bz2
 import os
 import warnings
 
     @classmethod
     def import_database(cls, filepath, overwrite):
+        logger = get_logger("io-performance.log")
+
         if overwrite:
             raise NotImplementedError
         with bz2.BZ2File(filepath) as f:
+            start = time()
+
             package_data = JsonWrapper.loads(f.read())
+
+            logger.info("Loading BW2Package database (len %s): %.4g" % (len(package_data), time() - start))
+
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
             for name, data in package_data.iteritems():
+                start = time()
+
                 db_data = dict([((name, key), value) for key, value in \
                     data["data"].iteritems()])
                 if name in databases:
                 database.write(db_data)
                 database.process()
 
+                logger.info("Processing BW2Package database (len %s): %.4g" % (len(db_data), time() - start))
+
     @classmethod
     def import_method(cls, filepath, overwrite):
+        logger = get_logger("io-performance.log")
+
         if overwrite:
             raise NotImplementedError
         with bz2.BZ2File(filepath) as f:
+            start = time()
+
             package_data = JsonWrapper.loads(f.read())
+
+            logger.info("Loading BW2Package method (len %s): %.4g" % (len(package_data), time() - start))
+
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
+            start = time()
             for data in package_data:
+
                 name = tuple(data["metadata"]["name"])
                 if name in methods:
                     raise ValueError("Duplicate method")
                     unit=data["metadata"]["unit"],
                     description=data["metadata"]["description"],
                     num_cfs=data["metadata"]["num_cfs"]
-                    )
+                )
                 method.write([
                     [(o["database"], o["code"]), o["amount"], o["location"]
-                    ] for o in data["cfs"]])
+                ] for o in data["cfs"]])
                 method.process()
 
+            logger.info("Processing BW2Package methods (len %s): %.4g" % (len(package_data), time() - start))
+
 
 def download_biosphere():
+    logger = get_logger("io-performance.log")
+    start = time()
     filepath = download_file("biosphere.bw2package")
+    logger.info("Downloading biosphere package: %.4g" % (time() - start))
+    start = time()
     BW2PackageImporter.importer(filepath)
+    logger.info("Importing biosphere package: %.4g" % (time() - start))
 
 
 def download_methods():
+    logger = get_logger("io-performance.log")
+    start = time()
     filepath = download_file("methods.bw2iapackage")
+    logger.info("Downloading methods package: %.4g" % (time() - start))
+    start = time()
     BW2PackageImporter.importer(filepath)
+    logger.info("Importing methods package: %.4g" % (time() - start))
 # -*- coding: utf-8 -*-
 from . import config
 from logging.handlers import RotatingFileHandler
-from utils import random_string
+from utils import random_string, create_in_memory_zipfile_from_directory
 import codecs
 import datetime
 import logging
 import os
+import requests
+import uuid
+from serialization import JsonWrapper
+try:
+    import anyjson
+except ImportError:
+    anyjson = None
 
 
 def get_logger(name, level=logging.INFO):
 '''))
     logger.addHandler(handler)
     return logger
+
+
+def upload_logs_to_server(metadata={}):
+    # Hardcoded for now
+    url = "http://reports.brightwaylca.org/logs"
+    dirpath = config.request_dir("logs")
+    zip_fo = create_in_memory_zipfile_from_directory(dirpath)
+    files = {'file': (uuid.uuid4().hex + ".zip", zip_fo.read())}
+    metadata['json'] = 'native' if anyjson is None else \
+        anyjson.implementation.name
+    metadata['windows'] = config._windows
+    return requests.post(
+        url,
+        data=metadata,
+        files=files
+    )
 import re
 import requests
 import string
+import zipfile
+try:
+    import cStringIO as StringIO
+except ImportError:
+    import StringIO
 
 # Maximum value for unsigned integer stored in 4 bytes
 MAX_INT_32 = 4294967295
     config.is_temp_dir = False
     download_biosphere()
     download_methods()
+
+
+def create_in_memory_zipfile_from_directory(path):
+    # Based on http://stackoverflow.com/questions/2463770/python-in-memory-zip-library
+    memory_obj = StringIO.StringIO()
+    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
+    zf = zipfile.ZipFile(memory_obj, "a", zipfile.ZIP_DEFLATED, False)
+    for filename in files:
+        zf.writestr(
+            filename,
+            open(os.path.join(path, filename)).read()
+        )
+    # Mark the files as having been created on Windows so that
+    # Unix permissions are not inferred as 0000
+    for zfile in zf.filelist:
+        zfile.create_system = 0
+    zf.close()
+    memory_obj.seek(0)
+    return memory_obj
 # The short X.Y version.
 version = '0.9'
 # The full version, including alpha/beta/rc tags.
-release = '0.9.0'
+release = '0.9.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 lxml
+nose
 numpy
+progressbar
+requests>=1.1.0
 scipy
-nose
-progressbar
 voluptuous
+brightway2>=0.9.1
 bw-stats-toolkit>=0.7
-requests>=1.1.0
-brightway2>=0.9.0-alpha2
         packages.append(pkg)
 
 setup(
-  name='bw2data',
-  version="0.9.0-alpha3",
-  packages=packages,
-  author="Chris Mutel",
-  author_email="cmutel@gmail.com",
-  license=open('LICENSE.txt').read(),
-  install_requires=["voluptuous", "progressbar", "numpy", "lxml", "scipy", "requests>=1.1.0", "brightway2"],
-  url="https://bitbucket.org/cmutel/brightway2-data",
-  long_description=open('README.rst').read(),
-  classifiers=[
-    'Development Status :: 4 - Beta',
-    'Intended Audience :: End Users/Desktop',
-    'Intended Audience :: Developers',
-    'Intended Audience :: Science/Research',
-    'License :: OSI Approved :: BSD License',
-    'Operating System :: MacOS :: MacOS X',
-    'Operating System :: Microsoft :: Windows',
-    'Operating System :: POSIX',
-    'Programming Language :: Python',
-    'Programming Language :: Python :: 2.7',
-    'Programming Language :: Python :: 2 :: Only',
-    'Topic :: Scientific/Engineering :: Information Analysis',
-    'Topic :: Scientific/Engineering :: Mathematics',
-    'Topic :: Scientific/Engineering :: Visualization',
+    name='bw2data',
+    version="0.9.1",
+    packages=packages,
+    author="Chris Mutel",
+    author_email="cmutel@gmail.com",
+    license=open('LICENSE.txt').read(),
+    install_requires=["voluptuous", "progressbar", "numpy", "lxml", "scipy", "requests>=1.1.0", "brightway2"],
+    url="https://bitbucket.org/cmutel/brightway2-data",
+    long_description=open('README.rst').read(),
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: End Users/Desktop',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'License :: OSI Approved :: BSD License',
+        'Operating System :: MacOS :: MacOS X',
+        'Operating System :: Microsoft :: Windows',
+        'Operating System :: POSIX',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 2 :: Only',
+        'Topic :: Scientific/Engineering :: Information Analysis',
+        'Topic :: Scientific/Engineering :: Mathematics',
+        'Topic :: Scientific/Engineering :: Visualization',
     ],)