Commits

open...@team.openplans.org  committed 86e3bef Merge

i'm not sure what i'm merging but these are definitely my changes. hg is confusing me.

  • Participants
  • Parent commits d89203c, 52385b1

Comments (0)

Files changed (16)

 *orig
 *\#
 pip-log.txt
+*.log
+Random grab-bag of todos and historical project info
+
 OFFICES TA CARES ABOUT
 ======================
 
 =============
 
 
-
-Plan
-=====
-
-* Use SRID = 4326 aka
-  WGS84(http://en.wikipedia.org/wiki/World_Geodetic_System) 
-  ... which seems to be what blockparty uses; stores things as lon/lat
-  pairs so i can easily query for which district contains a lon/lat 
-
 * This apparently works to find my home districts (LONGITUDE FIRST):
 
   select * from districts where districts.geometry is not NULL and st_contains(districts.geometry, ST_GeomFromText('POINT(-73.96805047988892 40.677578720206355 )', 4326));
 
-* Refactor the models as per below.
-
 
 
 NEW DATA MODEL
 
 Basically same as above, except Office becomes a first-class object,
 and we add a concept of Party.
-   
 
 
 Votesmart usage examples:
 nyc_id = 898
 nyc_officials = votesmart.local.getOfficials(nyc_id)
 pprint([(c.lastName, c.title, c.officeParties) for c in nyc_officials])
+
+
+
+ADMIN UI
+========
+
+Might want one of these.
+Maybe bootstrap it with FormAlchemy? and/or its pylons admin UI?
+http://docs.formalchemy.org/current/ext/pylons.html#administration-interface
+
+
+TO DO
+
 Then to populate the data:
 
 $ source bin/activate
-$ paster setup-app development.ini
+$ paster setup-app --name=pvoter development.ini
 $ python manage.py version_control
 $ python manage.py upgrade
 

File development.ini

 host = 127.0.0.1
 port = 9000
 
-[app:main]
+[pipeline:main]
+pipeline = translogger pvoter
+
+[app:pvoter]
 use = egg:purplevoter
 full_stack = true
 static_files = true
 # execute malicious code after an exception is raised.
 #set debug = false
 
-# Apache-style access log
-filter-with = translogger
 
 [filter:translogger]
 use = egg:Paste#translogger
 # qualname = purplevoter
 
 [logger_sqlalchemy]
-level = WARN 
-#level = INFO
+#level = WARN 
+level = INFO
 handlers = 
 qualname = sqlalchemy.engine
 # "level = INFO" logs SQL queries.

File migrations/versions/007_Fix_character_encodings.py

     people = meta.Session.query(People).all()
     for p in people:
         try:
-            fullname = p.fullname.decode('utf8').encode('latin1').decode('utf8')
+            fullname = p.fullname.decode('utf8').encode('latin1').decode('utf8').strip()
         except UnicodeDecodeError:
-            print "couldn't fix %s, skipping" % fullname
+            print "couldn't fix %s, skipping" % p.fullname
             continue
         if fullname != p.fullname:
             print "Fixed", fullname.encode('utf8')
             if p.fullname != fullname:
                 p.fullname = fullname
         except UnicodeDecodeError:
-            print "couldn't mangle %s, skipping" % fullname
+            print "couldn't mangle %s, skipping" % p.fullname
     Session.commit()
 

File migrations/versions/008_Add_ny_city_borough_borders.py

 
         # Generate a shape for all of NYC by taking the union of the boroughs.
         connection.execute("INSERT INTO districts (state, district_type, level_name, district_name, geometry) SELECT 'NY', 'City', 'City', 'New York City', ST_Union(geometry) from districts where state = 'NY' and level_name = 'City' and (district_type = 'Borough'  or district_type = 'City Council');")
+
+        # Set parent id's for city council districts.
+        # Note the use of ST_Centroid: this returns the center of the district.
+        # This is a workaround because some of the districts have
+        # borders that extend into water for no apparent reason,
+        # whereas the borough shapes don't do that.
+        # It could break, given sufficiently weird district
+        # gerrymandering near the edge of a borough.
+        connection.execute(
+            "UPDATE districts cc"
+            " SET parent_id = boro.id"
+            " FROM districts boro"
+            " WHERE (ST_Within(ST_Centroid(cc.geometry), boro.geometry)"
+            "     AND (cc.district_type = 'City Council')"
+            "     AND (boro.district_type = 'Borough'));"
+            )
+
         trans.commit()
     except:
         trans.rollback()
 def downgrade():
     # Operations to reverse the above upgrade go here.
     connection = migrate_engine.connect()
-    try:
-        connection.execute("DELETE FROM districts WHERE state = 'NY' and (district_type = 'Borough' or district_type = 'City');")
-    except:
-        pass
+    trans = connection.begin()
+    connection.execute("DELETE FROM districts WHERE state = 'NY' and (district_type = 'Borough' or district_type = 'City');")
+    trans.commit()

File migrations/versions/010_add_more_candidate_data.py

                               name='New York City 2009',
                               stagename='Primary')
 
-    files = [dict(path='city_council.csv', office='City Council',
+    files = [dict(path='city_council.csv', office=u'City Council',
                   district_type='City Council', district_format=r'District %s'),
-             dict(path='mayoral.csv', office='Mayor', 
+             dict(path='mayoral.csv', office=u'Mayor', 
                   district_type='City', district_format='New York City'),
-             dict(path='comptroller.csv', office='Comptroller',
+             dict(path='comptroller.csv', office=u'Comptroller',
                   district_type='City', district_format='New York City'),
-             dict(path='district_attorney.csv', office='District Attorney',
+             dict(path='district_attorney.csv', office=u'District Attorney',
                   district_type='Borough', district_format=r'%s'),
-             dict(path='borough_president.csv', office='Borough President',
+             dict(path='borough_president.csv', office=u'Borough President',
                   district_type='Borough', district_format=r'%s'),
-             dict(path='public_advocate.csv', office='Public Advocate',
+             dict(path='public_advocate.csv', office=u'Public Advocate',
                   district_type='City', district_format='New York City'),
              ]
 
         transaltid = (info.get('NID') or '').strip()
         if transaltid:
             ta_id = find_or_create(meta.Session, PeopleMeta,
-                                   meta_key='transaltid', meta_value=transaltid,
+                                   meta_key=u'transaltid', 
+                                   meta_value=unicode(transaltid),
                                    person=person)
 
         print u"Added", person.fullname
 
+
     meta.Session.commit()
 
 def downgrade():

File misc_import_data/ta_candidate_info_20090818/city_council.csv

 "Karen Koslowitz",29,"Queens",195,
 "Walter G  Nestler",18,"Bronx",196,
 "Janine Materna",51,"Staten Island",202,
-"Mel Gagarin",29,"Queens",204,
 "Josh Skaller",39,"Brooklyn",208,
 "Paul A Vallone ",19,"Queens",210,
 "Brent M O'Leary",26,"Queens",212,

File production.ini

 host = 127.0.0.1
 port = 9000
 
-[app:main]
+[pipeline:main]
+pipeline = translogger pvoter
+
+[app:pvoter]
 use = egg:purplevoter
 full_stack = true
 static_files = true
 # execute malicious code after an exception is raised.
 set debug = false
 
-# Apache-style access log
-filter-with = translogger
 
 [filter:translogger]
+# Apache-style access log.
 use = egg:Paste#translogger
 setup_console_handler = False
 

File purplevoter/controllers/people.py

                 'election_date': obj.election.date,
                 'candidates': sorted(obj.candidates, key=lambda x: x.fullname),
                 'incumbents': sorted(obj.incumbents, key=lambda x: x.fullname),
+                'parent_district_name': obj.district.parent_district_name,
                 }
         return info
     else:
         raise TypeError
 
+    
+def _json_error(status, reason, data=None):
+    # Not sure of the best way to do custom error responses.
+    # Calling abort(400) seems to trigger the default error-handling 
+    # middleware which spits out HTML.  This works okay:
+    response.status = status
+    request.environ['pylons.status_code_redirect'] = True
+    response.headers['Content-Type'] = 'application/json'
+    return json.dumps({'error': reason, 'error_data': data})    
+
 
 class PeopleController(BaseController):
 
         """
         self._search()
         response.headers['Content-Type'] = 'application/json'
+        if len(c.address_matches) > 1:
+            addresses = [address[0] for address in c.address_matches]
+            return _json_error(400, "Ambiguous address", data=addresses)
         return json.dumps(c.races, sort_keys=True, indent=1, default=_to_json)
 
     def _search(self):
         """Find districts and people, given an address."""
         lat = lon = None
-
+        c.races = []
+        c.districts = []
         c.search_term = request.params.get('address', '')
         c.election_date = datetime.date(2009, 9, 15)
         c.election_stagename = 'Primary'  # XXX parameterize this.
                 addr_str, (lat, lon) = address_matches[0]
             elif len(address_matches) > 1:
                 # Let the user figure it out
-                c.address_matches = address_matches
+                c.address_matches = sorted(address_matches)
                 return
             else:
-                # XXX signal an error
+                # XXX signal an error?
+                c.address_matches = []
                 return
         # We should have a location to work with now.
         if lat and lon:
        """ convert an address string into a list of (addr_str, (lat,lon))
        tuples """
        # move
-       google_api_key = config['google_api_key']
-       geocoder = geopy.geocoders.Google(api_key=google_api_key)
-       address_gen = geocoder.geocode(address, exactly_one=False)
-       return [(addr, (lat, lon)) for addr, (lat, lon) in address_gen]
+
+       if config.get('mock_geocoder'):
+           # Intended for use in testing.
+           address_gen = request.environ['mockgeocoder.results']
+       else:
+           google_api_key = config['google_api_key']
+           geocoder = geopy.geocoders.Google(api_key=google_api_key)
+           address_gen = geocoder.geocode(address, exactly_one=False)
+       result = [(addr, (lat, lon)) for addr, (lat, lon) in address_gen]
+       #print result
+       return result
 
 
     def _search_races(self, districts, level_names):

File purplevoter/model/__init__.py

 
 from purplevoter.model import meta
 from sqlalchemy import orm
+from sqlalchemy.orm.exc import NoResultFound
 from sqlgeotypes import Geometry
 import sqlalchemy as sa
 
     sa.Column("district_type", sa.types.String(255), nullable=False),
     sa.Column("level_name", sa.types.String(255), nullable=False),
     sa.Column("district_name", sa.types.String(255), nullable=False),
+    sa.Column("parent_id", sa.schema.ForeignKey('districts.id',
+                                                ondelete='SET DEFAULT'),
+              nullable=True),
     sa.Column("geometry", Geometry(meta.storage_SRID, 'GEOMETRY', 2), nullable=True),
     )
 
 class Districts(object):
-    pass
+
+    @property
+    def parent_district(self):
+        """occasionally it's useful to know a geographic district that
+        encompasses this one, eg. in New York, a City Council district
+        is part of a certain Borough.
+        """
+        # Tried to do this via an orm.relation but I couldn't figure out
+        # how to get it to work the right way with a self-join.
+        if self.parent_id is None:
+            return None
+        q = meta.Session.query(Districts)
+        try:
+            parent = q.filter_by(id=self.parent_id).one()
+            return parent
+        except NoResultFound:
+            return None
+
+    @property
+    def parent_district_name(self):
+        parent = self.parent_district
+        if parent:
+            return parent.district_name
+        return None
+            
 
 
 people_table = sa.Table(
     sa.Column("id", sa.types.Integer, primary_key=True, autoincrement=True),
     sa.Column("fullname", sa.types.String(255), nullable=False),
     sa.Column("incumbent_office", sa.types.String(255), nullable=True),
-    sa.Column("incumbent_district", sa.types.Integer, sa.schema.ForeignKey('districts.id'),
+    sa.Column("incumbent_district", sa.types.Integer, 
+              sa.schema.ForeignKey('districts.id', ondelete='SET DEFAULT'),
               nullable=True),
     sa.Column("incumbent_district", sa.types.Integer, sa.schema.ForeignKey('districts.id'),
               nullable=True),
                        })
        
 
-orm.mapper(Districts, districts_table,
-           properties={# Don't use cascade here, I think that we don't want
-                       # want people to be deleted even if their district is
-                       # deleted. Not sure.
-                       'incumbents': orm.relation(People),
-                       })
+orm.mapper(
+    Districts, districts_table,
+    properties={# Don't use cascade here, I think that we don't want
+                # want people to be deleted even if their district is
+                # deleted. Not sure.
+                'incumbents': orm.relation(People),
+                })
 
 orm.mapper(People, people_table,
            properties={'meta': orm.relation(PeopleMeta, 

File purplevoter/templates/search_form.mako

  % for district in c.districts:
     <dl class="race">
     % for race in sorted(district.races, key=lambda r: r.office) :
-      <dt class="office">Candidates for ${race.office} in ${district.district_name}</dt>
+      <dt class="office">Candidates for ${race.office} in
+        ${district.district_name}
+        % if district.parent_district_name:
+	  ,  ${district.parent_district_name}
+	% endif
+      </dt>
       <dd><dl class="candidate">
       % for person in race.candidates:
         <dt class="fullname"><strong>${person.fullname}</strong></dt>

File purplevoter/tests/__init__.py

 
 class TestController(TestCase):
 
+    extra_environ = None
+
     def __init__(self, *args, **kwargs):
         if pylons.test.pylonsapp:
             wsgiapp = pylons.test.pylonsapp
         else:
             wsgiapp = loadapp('config:%s' % config['__file__'])
-        self.app = TestApp(wsgiapp)
+        if self.extra_environ:
+            self.app = TestApp(wsgiapp, extra_environ=self.extra_environ)
+        else:
+            self.app = TestApp(wsgiapp)
         url._push_object(URLGenerator(config['routes.map'], environ))
         TestCase.__init__(self, *args, **kwargs)

File purplevoter/tests/functional/test_admin.py

 
 class TestAdminController(TestController):
 
-    def test_index(self):
-        response = self.app.get(url(controller='admin', action='index'))
-        # Test response...
+    pass
+
+#     def test_index(self):
+#         response = self.app.get(url(controller='admin', action='index'))
+#         # Test response...

File purplevoter/tests/functional/test_people.py

-from purplevoter.tests import *
+from purplevoter.tests import url, TestController
 
 class TestPeopleController(TestController):
 
-    def test_index(self):
-        response = self.app.get(url(controller='people', action='index'))
-        # Test response...
+    degraw = '669 degraw st., 11217'
 
-    def test_geocoder(self):
-        response = self.app.get(url(controller='people', action='search', address='669 degraw st, 11217'))
-        # senate
-        assert 'Charles Schumer' in response
-        assert 'Kirsten Gillibrand' in response
+    mockgeocoder_results = []
+    extra_environ = {'mockgeocoder.results': mockgeocoder_results}
 
-        # fed rep - D 11
-        assert 'Yvette D. Clarke' in response
+#     def test_geocoder_senate(self):
+#         response = self.app.get(url(controller='people', action='search', address=self.degraw, level_name='federal'))
+#         assert 'Charles Schumer' in response
+#         assert 'Kirsten Gillibrand' in response
 
-        # city council
+#     def test_geocoder_us_house(self):
+#         response = self.app.get(url(controller='people', action='search', address=self.degraw))
+#         # fed rep - D 11
+#         assert 'Yvette D. Clarke' in response
+
+#     def test_geocoder_state_assembly(self):
+#         response = self.app.get(url(controller='people', action='search', address=self.degraw))
+#         # state assembly - D 52
+#         assert 'Joan Millman' in response
+
+#     def test_geocoder_state_senate(self):
+#         response = self.app.get(url(controller='people', action='search', address=self.degraw))
+#         # state senate - D 18
+#         assert 'Velmanette Montgomery' in response
+
+    def test_geocoder_council(self):
+        self.mockgeocoder_results[:] = [(u'669 Degraw St, Brooklyn, NY 11217, USA',
+                                         (40.678329400000003, -73.981242499999993))]
+        response = self.app.get(url(controller='people', action='search', address=self.degraw))
         assert 'David Yassky' in response
 
-        # state assembly - D 52
-        assert 'Joan Millman' in response
+    def test_geocoder_races(self):
+        self.mockgeocoder_results[:] = [(u'669 Degraw St, Brooklyn, NY 11217, USA',
+                                         (40.678329400000003, -73.981242499999993))]
+        response = self.app.get(url(controller='people', action='search', address=self.degraw))
+        assert isinstance(response.c.races, list)
+        assert response.c.races, "got empty races list"
+        self.assertEqual(len(response.c.races), 4)
+        offices = sorted(r.office for r in response.c.races)
+        assert offices[0] == u'City Council'
+        assert offices[1] == u'Comptroller'
+        assert offices[2] == u'Mayor'
+        assert offices[3] == u'Public Advocate'
 
-        # state senate - D 18
-        assert 'Velmanette Montgomery' in response
+
+    def test_no_address(self):
+        self.mockgeocoder_results[:] = []
+        response = self.app.get(url(controller='people', action='search'))
+        assert 'No results found' in response
+
+    def test_bogus_address(self):
+        response = self.app.get(url(controller='people', action='search', address='Zbasp89ba~#$'))
+        assert 'No results found' in response
+
+    def test_address_out_of_known_world(self):
+        self.mockgeocoder_results[:] = [(u'Beverly Hills, CA 90210, USA',
+                                      (34.103003200000003, -118.4104684))]
+        response = self.app.get(url(controller='people', action='search', address='90210'))
+        assert 'No results found' in response
+
+    def test_address_ambiguous(self):
+        self.mockgeocoder_results[:] = [
+            (u'Main St, New York, NY 10044, USA', (40.761251999999999, -73.950389000000001)),
+            (u'Main St, Madawaska, ME 04756, USA', (47.299903999999998, -68.377725999999996)),
+            (u'Main St, Green Bay, WI, USA', (44.461621999999998, -87.953372000000002)),
+            (u'Main St, NV, USA', (38.926498000000002, -119.728736)),
+            (u'Main St, Lugoff, SC 29078, USA', (34.206927999999998, -80.728324000000001)),
+            (u'Main St, Bamberg, SC 29003, USA', (33.254896000000002, -81.075939000000005)),
+            (u'Main St, Columbia, SC, USA', (34.031680000000001, -81.042098999999993)),
+            (u'Main St, Springville, AL 35146, USA', (33.766407000000001, -86.479605000000006)),
+            (u'Main St, ME, USA', (44.884884, -68.671289000000002)),
+            (u'Main St, ME, USA', (47.303555000000003, -68.150024999999999))]
+        response = self.app.get(url(controller='people', action='search', address='main st.'))
+        assert 'Did you mean one of these addresses?' in response
+
+
+
+
+class TestPeopleControllerJsonOutput(TestController):
+
+    lafayette = '148 lafayette st., new york, ny'
+
+    mockgeocoder_results = []
+    extra_environ = {'mockgeocoder.results': mockgeocoder_results}
+ 
+    def _search_json(self, *args, **kw):
+        response = self.app.get(url(controller='people', action='search_json', *args, **kw))
+        return response
+        
+    def test_no_address(self):
+        response = self._search_json()
+        assert response.json == []
+
+    def test_bogus_address(self):
+        response = self._search_json(address='Zbasp89ba~#$')
+        assert response.json == []
+
+    def test_address_out_of_known_world(self):
+        response = self._search_json(address='90210')
+        assert response.json == []
+
+    def test_address_ambiguous(self):
+        expected_results = [
+            (u'Main St, Springfield, CO 81073, USA',
+             (37.404052999999998, -102.61655399999999)),
+            (u'Main St, Springfield, OR, USA',
+             (44.045764400000003, -122.96185699999999)),
+            (u'Main St, Springfield, MA, USA',
+             (42.106045000000002, -72.597044499999996)),
+            (u'Main St, Springfield, IL 62711, USA',
+             (39.709083999999997, -89.704749000000007)),
+            (u'Main St, Springfield, LA 70462, USA',
+             (30.438205, -90.568635999999998)),
+            (u'Main St, Springfield, MA 01151, USA',
+             (42.159067, -72.4967015)), 
+            (u'Main St, Springfield, NJ 07081, USA',
+             (40.712091000000001, -74.307867000000002)), 
+            (u'Main St, Springfield, VT 05156, USA',
+             (43.296849000000002, -72.481633000000002))]
+
+        self.mockgeocoder_results[:] = expected_results
+        response = self.app.get(url(controller='people', action='search_json',
+                                    address='main street, springfield'),
+                                status=400)
+        assert response.status == '400 Bad Request'
+        expected_addresses = sorted([str(a[0]) for a in expected_results])
+        self.assertEqual(response.json['error_data'], expected_addresses)
+        self.assertEqual(response.json['error'], 'Ambiguous address')
 port = 5000
 
 [app:main]
-use = config:development.ini
+use = config:development.ini#pvoter
+mock_geocoder = true
+
+# Maybe use a sqlite db with spatialite for tests?
+# http://www.gaia-gis.it/spatialite/
 
 # Add additional test specific configuration options as necessary.