Commits

Carsten Senger committed 799b08f

[loaders][s] Replace swiss with datautil

* require datautil
* change imports from swiss to datautil
* remove unused imports

Comments (0)

Files changed (5)

     author_email='wdmmg@okfn.org',
     url='http://www.wheredoesmymoneygo.org/',
     install_requires=[
-        "datapkg>=0.7", 
+        "datapkg>=0.7",
+        "datautil>=0.3",
         "BeautifulSoup>=3.2.0",
         "lxml>=2.2.7",
         "xlrd>=0.7.1"

wdmmgext/load/cofog.py

-from datetime import date
-from zipfile import ZipFile
-from StringIO import StringIO
-import os, csv
+import csv
+import os
 
 import datapkg
-from pymongo.dbref import DBRef
-from swiss.tabular import gdocs
+from datautil.tabular import gdocs
 from pylons import config
 
-from wdmmg import model
-from wdmmg.model import Classifier, Entry
-import util
+from wdmmg.model import Classifier
 
 TAXONOMY = 'cofog'
 
     Drops Keys 'cofog1', 'cofog2' and 'cofog3' and all associated
     EnumerationValues and KeyValues.
     '''
-    
+
     Classifier.c.remove({'taxonomy': TAXONOMY})
     Explanation.c.remove({'collection': Classifier.collection_name, 'context': TAXONOMY})
-    
+
+
 def load():
     '''
     Downloads the COFOG list, and loads it into the database with key names
     dejargonise()
     promote_notes()
 
+
 def dejargonise():
     '''
     Downloads alternative names for the COFOG codes, chosen specially for WDMMG,
     and replaces the names in the database. The original names are backed up.
     '''
-    
+
     # Loop through the rows of the Google spreadsheet where the data is maintained.
     for row in gdocs.GDocsReaderTextDb(
-        'tQSJ9dxTh8AKl-ON4Qqja8Q', # Google spreadsheet key.
-        config['gdocs_username'],
-        config['gdocs_password']
+	'tQSJ9dxTh8AKl-ON4Qqja8Q', # Google spreadsheet key.
+	config['gdocs_username'],
+	config['gdocs_password']
     ).read().to_list()[2:]:
-        code, official_name, alternative_name, color, notes = row
-        concept = Classifier.find_one({'taxonomy': TAXONOMY, 'name': code})
-        assert concept, 'Spreadsheet contains an unknown COFOG code: %r' % row
-        if alternative_name:
-            concept['official_label'] = concept.label
-            concept.label = unicode(alternative_name)
-            print "Setting name of %r to %r" % (code, alternative_name)
-        if color is not None:
-            concept['color'] = color
-        Classifier.c.update({'_id': concept.id}, concept)
+	code, official_name, alternative_name, color, notes = row
+	concept = Classifier.find_one({'taxonomy': TAXONOMY, 'name': code})
+	assert concept, 'Spreadsheet contains an unknown COFOG code: %r' % row
+	if alternative_name:
+	    concept['official_label'] = concept.label
+	    concept.label = unicode(alternative_name)
+	    print "Setting name of %r to %r" % (code, alternative_name)
+	if color is not None:
+	    concept['color'] = color
+	Classifier.c.update({'_id': concept.id}, concept)
 
 def promote_notes():
     '''
     supply the missing description by copying it from the level 3 sub-code.
     '''
     for item in Classifier.find({'taxonomy': TAXONOMY, 'level': 2}):
-        if item.description:
-            continue
-        children = Classifier.find({'taxonomy': TAXONOMY, 'level': 3, 'parent':
-            item.to_ref()})
-        if children.count() != 1: 
-            continue
-        child = children.next()
-        if child.description:
-            print "Copying notes from %s to %s" % (child.name, item.name)
-            print "Old notes = ", item.description.encode('utf-8')
-            item.description = child.description
-            print "New notes = ", item.description.encode('utf-8')
-            Classifier.c.update({'_id': child.id}, child)
+	if item.description:
+	    continue
+	children = Classifier.find({'taxonomy': TAXONOMY, 'level': 3, 'parent':
+	    item.to_ref()})
+	if children.count() != 1:
+	    continue
+	child = children.next()
+	if child.description:
+	    print "Copying notes from %s to %s" % (child.name, item.name)
+	    print "Old notes = ", item.description.encode('utf-8')
+	    item.description = child.description
+	    print "New notes = ", item.description.encode('utf-8')
+	    Classifier.c.update({'_id': child.id}, child)
 
 def load_file(fileobj):
     '''
     'cofog1', 'cofog2' and 'cofog3'.
     '''
     code_refs = {}
-    
+
     reader = csv.reader(fileobj)
     header = reader.next() # Code, Title, Details, Change date
-    
+
     for row_index, cells in enumerate(reader):
-        code, title, details, change_date = [unicode(x, 'UTF-8') for x in cells]
-        q = {"name": code, "taxonomy": TAXONOMY}
-        classifier = {
-            'label': title, 
-            'description': details, 
-            'change_date': change_date
-            }
-        # Create the enumeration values.
-        #print '%r, %r' % (code, title)
-        parts = code.split('.')
-        parents = [u'.'.join(parts[:i+1]) for i, _ in enumerate(parts)]
-        #print parents
-        if len(parents)==1:
-            #print 'Creating level 1 code', parents[0]
-            classifier['level'] = 1
-        elif len(parents)==2:
-            #print 'Creating level 2 code', parents[1]
-            classifier['parent'] = code_refs[parents[0]]
-            classifier['level'] = 2
-        elif len(parents)==3:
-            #print 'Creating level 3 code', parents[2]
-            classifier['parent'] = code_refs[parents[1]]
-            classifier['level'] = 3
-        else:
-            assert False, code   
-        
-        Classifier.c.update(q, {"$set": classifier}, upsert=True)
-        classifier = Classifier.find_one(q)
-        code_refs[code] = classifier.to_ref() 
+	code, title, details, change_date = [unicode(x, 'UTF-8') for x in cells]
+	q = {"name": code, "taxonomy": TAXONOMY}
+	classifier = {
+	    'label': title,
+	    'description': details,
+	    'change_date': change_date
+	    }
+	# Create the enumeration values.
+	#print '%r, %r' % (code, title)
+	parts = code.split('.')
+	parents = [u'.'.join(parts[:i+1]) for i, _ in enumerate(parts)]
+	#print parents
+	if len(parents)==1:
+	    #print 'Creating level 1 code', parents[0]
+	    classifier['level'] = 1
+	elif len(parents)==2:
+	    #print 'Creating level 2 code', parents[1]
+	    classifier['parent'] = code_refs[parents[0]]
+	    classifier['level'] = 2
+	elif len(parents)==3:
+	    #print 'Creating level 3 code', parents[2]
+	    classifier['parent'] = code_refs[parents[1]]
+	    classifier['level'] = 3
+	else:
+	    assert False, code
+
+	Classifier.c.update(q, {"$set": classifier}, upsert=True)
+	classifier = Classifier.find_one(q)
+	code_refs[code] = classifier.to_ref()
-import os, sys, csv, re
-from datetime import date
-try: import json
-except ImportError: import simplejson as json
+import os, csv
+try:
+    import json
+except ImportError:
+    import simplejson as json
 
 import datapkg
-from swiss.tabular import gdocs
+from datautil.tabular import gdocs
 from pylons import config
 
 import wdmmg.model as model
 
 dataset_name = u'gla'
 
+
 def load_file(fileobj, period_mapper={}, commit_every=None):
     '''
     Loads a file from `fileobj` into a dataset with name 'gra'.
-from datetime import date
-from zipfile import ZipFile
-from StringIO import StringIO
-import os, csv
+import csv
+import os
 
 import datapkg
-from swiss.tabular import gdocs
+from datautil.tabular import gdocs
 from pylons import config
 
 from wdmmg import model
 from wdmmg.getdata import util
 
+
 def drop():
     '''
     Drops Keys 'cofog1', 'cofog2' and 'cofog3' and all associated
     model.Session.commit()
     model.Session.remove()
 
+
 def load():
     '''
     Downloads the COFOG list, and loads it into the database with key names
     fileobj = pkg.stream('cofog.csv')
     load_file(fileobj)
 
+
 def dejargonise():
     '''
     Downloads alternative names for the COFOG codes, chosen specially for WDMMG,
         else:
             assert False, code
     model.Session.commit()
-    model.Session.remove()
+    model.Session.remove()

wdmmgext/load/uganda.py

     import simplejson as json
 
 from pylons import config
-from swiss.tabular import gdocs
+from datautil.tabular import gdocs
 import xlrd
 
 from wdmmg.lib import loader, munge