hideki nara avatar hideki nara committed 7e80d99

headerless files can be loaded if colums order is same as model fields.

Comments (0)

Files changed (10)

 ^build/
 ^dist/
 app/*.log
+app/ken*

csvutils/models.py

 from django.utils.encoding import force_unicode, smart_str
 from django.template.defaultfilters import capfirst
-from django.db.models.fields import FieldDoesNotExist
+from django.db.models.fields import FieldDoesNotExist,AutoField
 
 from csvutils.http import CSVResponse
 from csvutils.utils import UnicodeWriter,UnicodeReader
         try:
             rec = self.get(**key_vals)
             try:
-                map(lambda k,v : setattr(rec,k,v), dictrow.items() )
+                for k,v in dictrow.items():
+                    setattr(rec,k,v)
             except Exception,e: 
                 return rec,e
         except self.model.DoesNotExist,e:
         except Exception,e:
             return rec, e
         
-    def csv_load(self,stream,encoding=None,keys=['id']):
+    def csv_load(self,stream,header=True,encoding=None,keys=['id']):
 
-        for index,row in UnicodeReader.csv_enumerator(stream):
+        if header==False:
+            fieldnames = [ f.name for f in self.model._meta.fields if not isinstance(f,AutoField) ]
+        else:
+            fieldnames=None
+
+        for index,row in UnicodeReader.csv_enumerator(stream,fieldnames):
             rec,e = self.from_dictrow(row,keys)
             yield index,rec,e 
 

csvutils/utils.py

         return map(lambda s: force_unicode(s, encoding=self.encoding, errors=self.errors), self.reader.next())
 
     @classmethod
-    def csv_enumerator(cls,stream, dialect=None, encoding=None, errors="strict", **kwds):
+    def csv_enumerator(cls,stream, fieldnames=None,dialect=None, encoding=None, errors="strict", **kwds):
         ''' enumerator factory for UnicodeReader
 
             - Key must be encode in unicode.
                 inits = dict( [ ( k.encode('ascii') , v ) for k,v in data_dict.iteritems() ]
                 Account(**inits).save()
         '''
-        reader = csv.DictReader(stream) 
+        reader = csv.DictReader(stream,fieldnames) 
         reader.reader = cls(stream, dialect=dialect, encoding=encoding, errors=errors , **kwds)
         return enumerate(reader)
 

test/app/sample/admin.py

 class FeedAdmin(admin.ModelAdmin):
     pass
 admin.site.register(Feed,FeedAdmin)
+
+### JpAddr 
+class JpAddrAdmin(admin.ModelAdmin):
+    list_display = [ f.name for f in JpAddr._meta.fields ]
+admin.site.register(JpAddr,JpAddrAdmin)

test/app/sample/fixtures/jp_adder.utf8.csv

+01101,"060  ","0600000","ホッカイドウ","サッポロシチュウオウク","イカニケイサイガナイバアイ","北海道","札幌市中央区","以下に掲載がない場合",0,0,0,0,0,0
+01101,"064  ","0640941","ホッカイドウ","サッポロシチュウオウク","アサヒガオカ","北海道","札幌市中央区","旭ケ丘",0,0,1,0,0,0
+01101,"060  ","0600041","ホッカイドウ","サッポロシチュウオウク","オオドオリヒガシ","北海道","札幌市中央区","大通東",0,0,1,0,0,0
+01101,"060  ","0600042","ホッカイドウ","サッポロシチュウオウク","オオドオリニシ(1-19チョウメ)","北海道","札幌市中央区","大通西(1〜19丁目)",1,0,1,0,0,0
+01101,"064  ","0640820","ホッカイドウ","サッポロシチュウオウク","オオドオリニシ(20-28チョウメ)","北海道","札幌市中央区","大通西(20〜28丁目)",1,0,1,0,0,0
+01101,"060  ","0600031","ホッカイドウ","サッポロシチュウオウク","キタ1ジョウヒガシ","北海道","札幌市中央区","北一条東",0,0,1,0,0,0
+01101,"060  ","0600001","ホッカイドウ","サッポロシチュウオウク","キタ1ジョウニシ(1-19チョウメ)","北海道","札幌市中央区","北一条西(1〜19丁目)",1,0,1,0,0,0
+01101,"064  ","0640821","ホッカイドウ","サッポロシチュウオウク","キタ1ジョウニシ(20-28チョウメ)","北海道","札幌市中央区","北一条西(20〜28丁目)",1,0,1,0,0,0
+01101,"060  ","0600032","ホッカイドウ","サッポロシチュウオウク","キタ2ジョウヒガシ","北海道","札幌市中央区","北二条東",0,0,1,0,0,0
+01101,"060  ","0600002","ホッカイドウ","サッポロシチュウオウク","キタ2ジョウニシ(1-19チョウメ)","北海道","札幌市中央区","北二条西(1〜19丁目)",1,0,1,0,0,0
Add a comment to this file

test/app/sample/management/__init__.py

Empty file added.

Add a comment to this file

test/app/sample/management/commands/__init__.py

Empty file added.

test/app/sample/management/commands/load_jpaddr.py

+# -*- coding: utf-8 -*-
+
+from django.core.management.base import BaseCommand, CommandError
+from django.contrib.auth.models import User
+from optparse import make_option
+from datetime import datetime
+import commands
+import os
+#
+from  app.sample.models import JpAddr
+
+class Command(BaseCommand):
+    args = ''
+    help = ''
+    option_list = BaseCommand.option_list + (
+        make_option('--url',
+            action='store',
+            dest='url',
+            default='http://www.post.japanpost.jp/zipcode/dl/kogaki/lzh/ken_all.lzh',
+            help='JP Address Data URL'),
+        make_option('--file',
+            action='store',
+            dest='file',
+            default='ken_all.csv',
+            help='JP Address Data File Name'),
+        )
+
+    def handle(self, *args, **options):
+        if os.path.exists( options['file'] ) == False:
+            print commands.getoutput( "curl %(url)s | lha x - " %  options )
+        if os.path.exists( options['file']+".utf8" ) == False:
+            print commands.getoutput( "nkf -w %(file)s > %(file)s.utf8" % options ) 
+
+        print datetime.now().strftime('Starting... %Y-%m-%d %H:%M:%S')
+        
+        errors =0
+        for i, addr, e  in JpAddr.objects.csv_load(
+                            open(options['file']+".utf8" ) ,
+                            header=False,
+                            keys=['zip']):
+            if e:
+                errors = errors +1
+            
+        print "total=%d error=%d" %  (i, errors)
+        print datetime.now().strftime('End... %Y-%m-%d %H:%M:%S')

test/app/sample/models.py

         return self.title
 
 
+class JpAddrManager(models.Manager,CsvModelManager):
+    pass
+
+class JpAddr(models.Model):
+    code = models.CharField(max_length=10,db_index=True,unique=False)
+    zip5 = models.CharField(max_length=5 )
+    zip  = models.CharField(max_length=7,db_index=True)
+    prefecture_kana = models.CharField(max_length =50,db_index=True )
+    city_kana = models.CharField(max_length =50 ,db_index=True)
+    town_kana = models.CharField(max_length =100,db_index=True)
+    prefecture= models.CharField(max_length =50 ,db_index=True)
+    city = models.CharField(max_length =50 ,db_index=True)
+    town = models.CharField(max_length =50 ,db_index=True)
+    is_split = models.BooleanField() 
+    is_small = models.BooleanField() 
+    is_towncode = models.BooleanField() 
+    is_multi  = models.BooleanField() 
+    changed  = models.IntegerField()  # 0 = no changed , 1 = changed , 2 = discontinued
+    reason   = models.IntegerField() 
+
+    objects = JpAddrManager() 
+
+    @classmethod
+    def is_changed(cls,val):
+        if type(val) == dict :
+            if False == val.has_key('changed'):
+                return False
+            else:
+                val = val['changed']    
+        try:
+            return int(val) == 1
+        except:
+            return False

test/app/sample/tests.py

         for i, feed, e  in Feed.objects.csv_load(open(self.fixture_path('fixtures','dump.csv'))):
             print i,feed,e
         self.assertEqual(i+1,Feed.objects.count() )
+
+    def test_headerless_csv_to_model(self):
+        """
+        python manage.py test sample.UtilsTest.test_headerless_csv_to_model
+
+        """
+        from models import JpAddr
+
+        JpAddr.objects.all().delete()
+        self.assertEqual(0,JpAddr.objects.count() )
+
+        for i, addr, e  in JpAddr.objects.csv_load(
+                            open(self.fixture_path('fixtures','jp_adder.utf8.csv')),
+                            header=False,
+                            keys=['zip']):
+            print i,",",addr.id,addr.zip,",",e
+        self.assertEqual(i+1,JpAddr.objects.count() )
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.