Commits

Frederic De Groef committed dc7de92

per-student calendar : 1st working version

  • Participants
  • Parent commits 6f5cb6d
  • Tags v0.1.5

Comments (0)

Files changed (8)

dependencies/gehol/converters/icalwriter.py

 from datetime import datetime, timedelta
 from icalendar import Calendar, Event, vText
-
+from StringIO import StringIO
 
 
 def to_ical(head,events,first_monday):
 
             cal_event.add('summary', summary)
             cal_event.add('dtstart', dtstart)
-            #cal_event.add('dtstamp', dtstart)
-
             cal_event.add('dtend', dtend)
             cal_event['location'] = vText(location)
             cal_event['organizer'] = vText(organizer)
 
     return cal.as_string()
 
+
+
+class IcalWriter(object):
+    """
+    Simple ical writer, compatible with rfc 5545 (although not extensively)
+    """
+
+    provider = 'https://bitbucket.org/odebeir/geholimport'
+    version = '2.0'
+
+    def __init__(self):
+        self.name = ""
+        self.description = ""
+        self.summary = ""
+        self.start_date = ""
+        self.events = []
+
+
+
+    def to_string(self):
+        out = StringIO()
+        out.write("BEGIN:VCALENDAR")
+        out.write("VERSION:%s" % self.version)
+        out.write("PRODID:%s" % self.provider)
+
+        out.write("X-WR-CALNAME:%s" % self.name)
+        out.write("X-WR-CALDESC:%s" % self.description)
+
+
+        out.write("BEGIN:VTIMEZONE")
+        out.write("TZID:Europe/Brussels")
+        out.write("BEGIN:STANDARD")
+        out.write("TZOFFSETFROM:+0100")
+        out.write("TZOFFSETTO:+0100")
+        out.write("DTSTART:%s" % self.start_date)
+        out.write("END:STANDARD")
+        out.write("END:VTIMEZONE")
+
+
+        for event in self.events:
+            out.write("BEGIN:VEVENT")
+            out.write("DTSTAMP:20101229T213000")
+            out.write("DTSTART;TZID=Europe/Brussels:20101229T213000")
+            out.write("DTEND;TZID=Europe/Brussels:20101229T235900")
+            out.write("SUMMARY:%s")
+            out.write("DESCRIPTION:%s")
+            out.write("LOCATION:%s" % event['location'])
+            #out.write("GEO:5.092867;51.557655")
+            out.write("END:VEVENT")
+
+        out.write("END:VCALENDAR")
+
+
+
+def convert_student_calendar_to_ical(student_calendar, first_monday):
+    date_init = datetime.strptime(first_monday,'%d/%m/%Y')
+
+    ical = Calendar()
+    ical.add('prodid', 'https://bitbucket.org/odebeir/geholimport')
+    ical.add('version', '2.0')
+    ical.add('summary', student_calendar.description)
+    ical.add('x-wr-calname', student_calendar.profile)
+    ical.add('x-wr-caldesc', student_calendar.description)
+
+    print "adding : %d events" % len(student_calendar.events)
+
+
+    for event in student_calendar.events:
+        # get some common data for all generated events
+        event_summary =  "%s (%s)" % (event['title'], event['type'])
+        event_organizer = event['organizer']
+        event_location = event['location']
+        #print 'expanding events for %s' % event_summary.encode('iso-8859-2')
+
+        #expand to ical events
+        for (i, event_week) in enumerate(event['weeks']):
+            delta = timedelta(days=(event_week-1)*7+(event['day']))
+            dtstart = date_init+delta + timedelta(hours = event['start_time'].hour,
+                                                    minutes = event['start_time'].minute)
+            dtend = date_init+delta + timedelta(hours = event['stop_time'].hour,
+                                            minutes = event['stop_time'].minute)
+            ical_event = Event()
+
+            ical_event.add('summary', event_summary)
+            ical_event.add('location', event_location)
+            ical_event.add('dtstart', dtstart)
+            ical_event.add('dtend', dtend)
+            ical_event.add('description', event_organizer)
+
+            ical.add_component(ical_event)
+
+    return ical.as_string()
+
+
+
+def convert_student_calendar_to_ics_rfc5545(student_calendar, first_monday):
+    date_init = datetime.strptime(first_monday,'%d/%m/%Y')
+
+    def write_line(out, line):
+        out.write(line+'\r\n')
+
+
+    out = StringIO()
+    write_line(out, "BEGIN:VCALENDAR")
+    write_line(out, "VERSION:%s" % "2.0")
+    write_line(out, "PRODID:%s" % 'https://bitbucket.org/odebeir/geholimport')
+
+    write_line(out, "X-WR-CALNAME:%s" % student_calendar.profile)
+    write_line(out, "X-WR-CALDESC:%s" % student_calendar.description)
+
+    write_line(out, "BEGIN:VTIMEZONE")
+    write_line(out, "TZID:Europe/Brussels")
+    write_line(out, "BEGIN:STANDARD")
+    write_line(out, "TZOFFSETFROM:+0100")
+    write_line(out, "TZOFFSETTO:+0100")
+    write_line(out, "DTSTART:%s" % date_init.strftime("%Y%m%dT%H%M%S"))
+    write_line(out, "END:STANDARD")
+    write_line(out, "END:VTIMEZONE")
+
+
+    for event in student_calendar.events:
+        event_summary =  "%s (%s)" % (event['title'], event['type'])
+        event_organizer = event['organizer']
+        event_location = event['location']
+        event_description = "%s [%s]" % (event_summary, event_organizer)
+
+        for (i, event_week) in enumerate(event['weeks']):
+            delta = timedelta(days=(event_week-1)*7+(event['day']))
+            dtstart = date_init+delta + timedelta(hours = event['start_time'].hour,
+                                                    minutes = event['start_time'].minute)
+            dtend = date_init+delta + timedelta(hours = event['stop_time'].hour,
+                                            minutes = event['stop_time'].minute)
+
+            write_line(out, "BEGIN:VEVENT")
+            write_line(out, "DTSTAMP:%s" %  dtstart.strftime("%Y%m%dT%H%M%S"))
+            write_line(out, "DTSTART;TZID=Europe/Brussels:%s" % dtstart.strftime("%Y%m%dT%H%M%S"))
+            write_line(out, "DTEND;TZID=Europe/Brussels:%s" % dtend.strftime("%Y%m%dT%H%M%S"))
+            write_line(out, "SUMMARY:%s" % event_summary)
+            write_line(out, "DESCRIPTION:%s" % event_description)
+            write_line(out, "LOCATION:%s" % event_location)
+            #write_line(out, "GEO:5.092867;51.557655")
+            write_line(out, "END:VEVENT")
+
+    write_line(out, "END:VCALENDAR")
+    ical_string = out.getvalue()
+    out.close()
+
+    return ical_string.encode('utf-8')
+
+
 def export_ical(head,events,dest_filename, first_monday):
     ical_string = to_ical(head,events,first_monday)
     fd = open(dest_filename,'w')
     fd.write(ical_string)
+
+
+def write_ical_to_file(ical_data, dest_filename):
+    fd = open(dest_filename,'w')
+    fd.write(ical_data)

dependencies/gehol/coursecalendar.py

         except AttributeError:
             self._guess_query_error(self.html_content)
 
-            conn.request("GET", self.url, headers = headers)
-            response = conn.getresponse()
-
-            print response.status, response.reason
-            html_content = response.read()
-
-            print html_content
-            conn.close()
 
 
     def _extract_header(self, html):

dependencies/gehol/studentcalendar.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+__author__ = 'Frederic'
+
+from datetime import datetime, timedelta
+from BeautifulSoup import BeautifulSoup
+from utils import split_weeks, convert_time
+import chardet
+
+class StudentCalendar(object):
+    def __init__(self, markup):
+        if self._is_file_type_object(markup):
+            markup = markup.read()
+        self.html_content = markup
+        soup = BeautifulSoup(self.html_content, fromEncoding='iso-8859-1')
+        self.header_data = {'student_profile':None, 'faculty':None}
+        self.events = []
+        self._load_content_from_soup(soup)
+
+    @property
+    def description(self):
+        descr = "[%s] %s" % (self.header_data['faculty'], self.header_data['student_profile'])
+        return descr.replace(':', '-') #.encode("iso-8859-2")
+
+
+    @property
+    def profile(self):
+        return  self.header_data['student_profile'].replace(':', '-')
+
+
+    def _load_content_from_soup(self, soup):
+        top_level_tables = soup.html.body.findAll(name="table", recursive=False)
+        # Take only the first 3 top-level tables. Sometimes the html is broken and we don't get the 4th.
+        # We also don't get the closing tags. This piece of software is pretty brilliant
+        header, event_grid, footer = top_level_tables[:3]
+
+        self._load_header_data(header)
+        self._load_events(event_grid)
+
+
+    def _load_header_data(self, header):
+        all_entries = header.findAll(name='table')
+        faculty_table = all_entries[4]
+        profile_table = all_entries[6]
+        self.header_data['faculty'] = self._extract_data_from_header_table(faculty_table)
+        self.header_data['student_profile'] = self._extract_data_from_header_table(profile_table)
+
+
+    @staticmethod
+    def _extract_data_from_header_table(table):
+        t = table.td.getText()
+        return t
+
+
+    def _load_events(self, event_table):
+        all_rows = event_table.findChildren('tr', recursive=False)
+
+        # get the column labels, save as actual hours objects
+        hours_row = all_rows[0].findChildren('td', recursive=False)
+        hours = [convert_time(hour_col.text) for hour_col in hours_row[1:]]
+
+        # get the events for each day
+        event_rows = all_rows[1:]
+        self.events = []
+
+        rows_per_day = self._get_num_row_per_day(event_rows)
+        #print rows_per_day
+        current_row_index = 0
+
+        for (num_day, day_string, num_rows) in rows_per_day:
+            day_events = []
+            for day_subrow in range(num_rows):
+                events_in_row = self._load_weekday_events(event_rows[current_row_index + day_subrow],
+                                                          num_day,
+                                                          hours)
+                day_events.extend(events_in_row)
+            #print "found %d events for day: %s" % (len(day_events), day_string)
+            self.events.extend(day_events)
+            current_row_index += num_rows
+
+
+    def _get_num_row_per_day(self, event_rows):
+        day_string = ['lun.', 'mar.', 'mer.' , 'jeu.', 'ven.', 'sam.']
+        num_rows = []
+        for row in event_rows:
+            num_rows += [int(col['rowspan']) for col in row.findAll('td', recursive=False) if col.text in day_string]
+        return zip(range(6), day_string, num_rows)
+
+
+    def _load_weekday_events(self, weekday_row, num_day, hours):
+        """
+        """
+        # At this point we should have a bunch of <td> elements. Some cells are empty, some cells have an event in them.
+        # First <td> is the weekday string, so we skip it.
+        row = weekday_row.findChildren('td', recursive=False)
+        all_day_slots = row
+
+        events = []
+        current_time_idx = 0
+        for time_slot in all_day_slots:
+            if self._slot_has_event(time_slot):
+                new_event = self._process_event(time_slot, hours[current_time_idx], num_day)
+                #print "[%d] %s (ts:%d)" % (num_day, new_event['title'], current_time_idx)
+                events.append(new_event)
+                current_time_idx += new_event['num_timeslots']
+            else:
+                if time_slot.text not in ['lun.', 'mar.', 'mer.' , 'jeu.', 'ven.', 'sam.']:
+                    current_time_idx += 1
+                    #print "[%d] ." % num_day
+                #else:
+                #    print "[%d] #" % num_day
+                
+        return events
+
+
+    def _process_event(self, object_cell, starting_hour, num_day):
+        num_timeslots = int(object_cell['colspan'])
+        cell_tables = object_cell.findChildren('table', recursive=False)
+        # event box : 3 tables, one per line :
+        #   - location/course type 
+        #   - title
+        #   - tutor/weeks
+        location_type_table, title_table, tutor_weeks_table = cell_tables
+
+        location = location_type_table.tr.findChildren('td')[0].text
+        course_type = location_type_table.tr.findChildren('td')[1].text
+
+        course_title = title_table.tr.td.text
+
+        children = tutor_weeks_table.findChildren('td')
+        course_tutor = children[0].text
+        course_weeks = children[2].text
+
+        return {
+            'type':course_type,
+            'location':location,
+            'organizer':course_tutor,
+            'title':course_title,
+            'weeks':split_weeks(course_weeks),
+            'num_timeslots':num_timeslots,
+            'start_time':starting_hour,
+            'stop_time':starting_hour + timedelta(hours = self._convert_num_timeslots_to_hours(num_timeslots)),
+            'day':num_day
+        }
+
+    @staticmethod
+    def _convert_num_timeslots_to_hours(num_timeslots):
+        # 1 timeslot = 30 minutes
+        return float(num_timeslots) / 2
+
+    @staticmethod
+    def _slot_has_event(slot):
+        return slot.table is not None
+
+
+    @staticmethod
+    def _is_file_type_object(f):
+        return hasattr(f, 'read')
 from gehol.coursecalendar import CourseCalendar, GeholException
 from gehol.converters.csvwriter import to_csv
 from gehol.converters.icalwriter import to_ical
+from gehol.converters.icalwriter import convert_student_calendar_to_ics_rfc5545
 from gehol import GeholProxy
-
+from gehol.studentcalendar import StudentCalendar
+import httplib, urlparse
 
 host = '164.15.72.157:8080'
 first_monday = '20/09/2010'
         csv_string = 'Problem with: "%s" [%s]'%(cal.metadata['mnemo'], e.message)
         ical_string = ''
         
-    return (error, csv_string,ical_string)
+    return error, csv_string,ical_string
 
+
+def get_student_calendar(url):
+    try:
+        headers = {"Content-type": "application/x-www-form-urlencoded",
+                   "Accept": "text/html",
+                   "Accept-Charset":"*"}
+
+        parsed_url = urlparse.urlparse(url)
+        scheme, netloc, path, params, query, frag = parsed_url
+
+        conn = httplib.HTTPConnection(netloc)
+        conn.request("GET", "%s;%s?%s" % (path, params, query), headers = headers)
+        response = conn.getresponse()
+        html = response.read()
+        cal = StudentCalendar(html)
+        return cal
+    except Exception,e:
+        raise ValueError('Could not get fetch url : %s (Reason : %s)' % (url, e.message))
+
+
+def convert_student_calendar(cal):
+    try:
+        ical_data = convert_student_calendar_to_ics_rfc5545(cal, first_monday)
+        return ical_data
+    except Exception,e:
+        return None
 from coursecalendar import CourseCalendar
 from icalrenderer import IcalRenderer
 from csvrenderer import CSVRenderer
+from studentcalendar import StudentCalendarPage, StudentURLQuery, StudentCalendarIcalRenderer
 from savedrequests import PreviousRequest
 
 
                                      ('/ical/.*', IcalRenderer),
                                      ('/csv/.*', CSVRenderer),
                                      ('/geholstatus',  UpdateGeholStatus),
+                                     ('/student_url', StudentURLQuery),
+                                     ('/student/ical.*', StudentCalendarIcalRenderer),
+                                     ('/student', StudentCalendarPage )
                                      ],
                                      debug=True)
 

studentcalendar.py

+__author__ = 'sevas'
+
+import os
+import urlparse
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import template
+from status import is_status_down, get_last_status_update
+from gehol2csv import get_student_calendar, convert_student_calendar
+
+class StudentURLQuery(webapp.RequestHandler):
+    def get(self):
+
+        template_values = {'gehol_is_down': is_status_down(),
+                         'last_status_update': get_last_status_update(),
+        }
+
+        #http://164.15.72.157:8080/Reporting/Individual;Student%20Set%20Groups;id;%23SPLUS3C1E8E?&template=Ann%E9e%20d%27%E9tude&weeks=1-14&days=1-6&periods=5-33&width=0&height=0
+        path = os.path.join(os.path.dirname(__file__), 'templates/student_url.html')
+        self.response.out.write(template.render(path, template_values))
+
+
+class StudentCalendarPage(webapp.RequestHandler):
+    def post(self):
+        gehol_url = self.request.get('gehol_url')
+        group_id = self._extract_group_id(gehol_url)
+        # TODO = sanitize url
+        cal = get_student_calendar(gehol_url)
+
+        faculty, student_profile = cal.header_data['faculty'], cal.header_data['student_profile']
+        event_titles = set(["%s (%s) [%s]" %  (e['title'], e['type'], e['organizer']) for e in cal.events])
+        ical_url = "/student/ical/%s" % group_id
+        ical_url_title = "ULB - %s" % student_profile
+
+        template_values = {'gehol_is_down': is_status_down(),
+                         'last_status_update': get_last_status_update(),
+                         'gehol_url':gehol_url,
+                         'cal_faculty':faculty,
+                         'cal_student_profile':student_profile,
+                         'cal_events':event_titles,
+                         'ical_url':ical_url,
+                         'ical_url_title':ical_url_title
+        }
+        
+        path = os.path.join(os.path.dirname(__file__), 'templates/student.html')
+        self.response.out.write(template.render(path, template_values))
+
+    @staticmethod
+    def _extract_group_id(gehol_url):
+        parsed = urlparse.urlparse(gehol_url)
+        p = parsed.params
+        # should look like : 'Student%20Set%20Groups;id;%23SPLUS35F073'
+        # we keep the last part
+        return p.split(';')[-1]
+
+
+
+class StudentCalendarIcalRenderer(webapp.RequestHandler):
+    def get(self):
+        parsed = urlparse.urlparse(self.request.uri)
+        group_id = parsed.path.split("/")[3]
+
+
+        gehol_url = self._rebuild_gehol_url(group_id)
+
+        cal = get_student_calendar(gehol_url)
+        ical_data = convert_student_calendar(cal)
+
+        student_profile = cal.header_data['student_profile']
+        ical_filename = "ULB - %s" % student_profile
+
+        self.response.headers['Content-Type'] = "text/calendar;  charset=utf-8"
+        self.response.headers['Content-disposition'] = "attachment; filename=%s.ics" % ical_filename
+        self.response.out.write(ical_data)
+
+
+    @staticmethod
+    def _rebuild_gehol_url(group_id):
+        return "http://164.15.72.157:8080/Reporting/Individual;Student%20Set%20Groups;id;"+group_id+"?&template=Ann%E9e%20d%27%E9tude&weeks=1-14&days=1-6&periods=5-33&width=0&height=0"

templates/student.html

+{% extends "main.html" %}
+
+{% block title %}
+    {{cal_student_profile}} | Gehol Importer
+{% endblock %}
+
+{% block content %}
+
+    
+     <h3>We fetched the events from the <a href="{{gehol_url}}">URL</a> you gave us.</h3>
+    We found the schedule for the following profile:
+        <br/>
+    <i>Faculty : </i>{{cal_faculty}} <br/>
+    <i>Student profile : </i>{{cal_student_profile}}<br/>
+    <br/>
+
+    <h3><a href="{{ical_url}}">{{ical_url_title}}.ics</a></h3>
+    <p>
+        Click this link to download an iCal file with the entire schedule. iCal files can be opened with
+        Apple iCal, Microsoft Outlook and Mozilla Sunbird. This link can also be imported or added as a live url
+        in Google Calendar.
+    </p>
+
+
+        
+
+     <br/>
+    <h3>This schedule contains events for the following courses:</h3>
+    <ul>
+    {% for event in cal_events %}
+        <li>{{event}}</li>
+    {% endfor %}
+    </ul>
+
+        
+
+{% endblock %}

templates/student_url.html

+{% extends "main.html" %}
+
+{% block title %}
+      Calendar for student profile  | Gehol Importer
+{% endblock %}
+
+{% block content %}
+
+
+    <h3>Insert the complete gehol query URL for the desired student profile</h3>
+
+    <form action="/student" method="post">
+      <div>
+          <label>
+              <input name="gehol_url" style="width:400px">
+          </label>
+          <input type="submit" value="Get schedule">
+      </div>
+    </form>
+
+
+{% endblock %}