Commits

Richard Shea committed e5325e3

Adding script to add met data. Currently only does stations

Comments (0)

Files changed (2)

dcdemo/dataloader_met.py

+"""
+Populate data structures from the external files
+"""
+__author__ = "Richard Shea (rshea@thecubagroup.com)"
+__version__ = "$Revision: 0.1 $"
+__date__ = "$Date: 2013/08/23 19:00:00 $"
+__copyright__ = "Copyright (c) 2013 Richard Shea"
+
+from django.conf import settings
+def makeDatabasesSetting():
+    '''
+    Set up database configuration for this standalone script
+    '''
+
+    DBPATH = '''/home/rshea/dev/djchdemo/db/djchdemo.db'''
+    MYDB = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3', 
+            'NAME': DBPATH, 
+            'USER': '',
+            'PASSWORD': '',
+            'HOST': '',   
+            'PORT': '',  
+        }
+    }
+    settings.configure(DATABASES=MYDB)
+
+import pprint
+import csv
+makeDatabasesSetting()
+from dcdemo.apps.dmodrv.models import Country
+from dcdemo.apps.dmodrv.models import City   
+
+from dcdemo.apps.dmodrv.models import Station
+from dcdemo.apps.dmodrv.models import Observation
+from dcdemo.apps.dmodrv.models import WindObservation
+from dcdemo.apps.dmodrv.models import TemperatureObservation
+
+
+from django.conf import settings
+from decimal import Decimal
+
+STATIONDATAPATH = """/home/rshea/dev/djchdemo/data/weatherstationdata.csv"""
+WINDDATAPATH = """/home/rshea/dev/djchdemo/data/worldcitiespop.txt"""
+TEMPDATAPATH = """/home/rshea/dev/djchdemo/data/worldcitiespop.txt"""
+def removeLineFeed(line):
+    '''
+    Removes line feeds from the argument supplied
+    '''
+    line = line.replace("\r","")
+    line = line.replace("\n","")
+    return line
+
+def processStationData(line_from_file, notUsed):
+    '''
+    Name,Agent Number,Network Number,Latitude (dec.deg),Longitude (dec.deg),Height (m),Posn_Precision,Observing Authority
+    '''
+    lstData = line_from_file.split(",")
+    import pprint
+    pprint.pprint(lstData)
+    nam = lstData[0]
+    lat = Decimal(lstData[3])
+    lng = Decimal(lstData[4])
+    ht = Decimal(lstData[5])
+    s = Station(name=nam, longitude=lng, latitude=lat, height_in_m=ht)
+    s.save()
+
+def populateAbstract(pathToData, lineProcessingFunction, lpfArg0=None):
+    '''
+    Opens file on specified path, ignores first line and then
+    iterates over all others invoking the 'lineProcessingFunction'
+    function each time with the line of input as an argument to it.
+    
+    The returned objects are saved into a list and at the end of
+    processing the input this list is returned to the caller
+    '''
+    lstOut = []
+    firstLine = True
+    for line in open(pathToData):
+        if firstLine:
+            firstLine = False
+        else:
+            line = removeLineFeed(line)
+            if line == None or len(line.strip()) < 1:
+                pass
+            else:
+                lstOut.append(lineProcessingFunction(line, lpfArg0))
+    return lstOut
+
+def populateStations(pathToStationData):
+    '''
+    Invokes populateAbstract to process the input data
+    defined in the file at 'pathToStationData'
+    '''
+    populateAbstract(pathToStationData, processStationData)
+
+def main():
+    lstStations = populateStations(STATIONDATAPATH)
+if __name__ == '__main__':
+    main()

dcdemo/dcdemo/apps/dmodrv/models.py

                 pop, lng, \
                 lat)
 
+class Station(models.Model):
+    name = models.CharField("station name", max_length=200)
+    longitude = models.DecimalField("longitude", default=0, max_digits=9, decimal_places=5)
+    latitude = models.DecimalField("latitude", default=0, max_digits=9, decimal_places=5)
+    height_in_m = models.DecimalField("height in metres", default=0, max_digits=6, decimal_places=2) 
+
+class Observation(models.Model):
+    name = models.CharField("station name", max_length=200)
+    when = models.DateTimeField()
+
+class WindObservation(models.Model):
+    station = models.ForeignKey(Station)
+    observation = models.ForeignKey(Observation)
+    speed = models.DecimalField("speed (km/h)", default=0, max_digits=6, decimal_places=3)
+    direction = models.IntegerField("direction", default=0)
+
+class TemperatureObservation(models.Model):
+    station = models.ForeignKey(Station)
+    observation = models.ForeignKey(Observation)
+    air = models.DecimalField("air", default=0, max_digits=6, decimal_places=3)
+    wet = models.DecimalField("wet", default=0, max_digits=6, decimal_places=3)
+    relative_humidity = models.DecimalField("air", default=0, max_digits=6, decimal_places=3)
+    dew_point = models.DecimalField("wet", default=0, max_digits=6, decimal_places=3)