Commits

Jernej Virag committed a88a41c

Implemented bugcomic and xkcd parser, database fixes

Comments (0)

Files changed (7)

 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
-from sqlalchemy import Column, String, Integer, Boolean, create_engine
+from sqlalchemy import Column, String, Integer, Boolean, Unicode, create_engine
 import settings
 
 Base = declarative_base()
 
     id = Column(Integer, primary_key=True)
     site = Column(String)
-    title = Column(String)
+    title = Column(Unicode)
     img_url = Column(String, index=True, unique=True)
-    subtext = Column(String)
+    subtext = Column(Unicode)
+    sort = Column(Integer, default=99999)
     dispatched = Column(Boolean)
 
 Base.metadata.create_all(engine)
         if article["img_url"] in existing_urls:
             continue
 
-        db_article = Article(site=article["site"], title=article.get("title", None), img_url=article["img_url"], subtext=article.get("subtext", None),
-                             dispatched=False)
+        db_article = Article(site=article["site"], title=article.get("title", None), img_url=article["img_url"], 
+                             subtext=article.get("subtext", None), sort=article.get("sort", 99999), dispatched=False)
         session.add(db_article)
     session.commit()
 
+def get_undispatched_articles(mark_dispatched=False):
+    session = Session()
+    db_articles = session.query(Article).filter_by(dispatched=False).order_by(Article.sort).all()
+    articles = [ { "site": article.site, "title" : article.title, "img_url" : article.img_url, "subtext":article.subtext } for article in db_articles ]
+    if mark_dispatched:
+        for article in db_articles:
+            article.dispatched = True
+            session.add(article)
+        session.commit()
+    
+    return articles 

parsers/__init__.py

 import parsers.icanhaz
+import parsers.bugcomic
+import parsers.xkcd
 
-parsers = [ icanhaz.ICanHazParser(), ]
+parsers = [ icanhaz.ICanHazParser(), bugcomic.BugComicParser(), xkcd.XkcdParser(), ]
 
 def get_parsers():
     return parsers

parsers/bugcomic.py

+import feedparser
+import bs4
+
+
+class BugComicParser(object):
+    RSS_URL = "http://www.bugcomic.com/feed/"
+
+    def get_data(self):
+        # Grab only the final entry
+        feed = feedparser.parse(self.RSS_URL)
+        last_comic = feed.entries[0]
+        content = bs4.BeautifulSoup(last_comic.content[0].value)
+
+        title = last_comic.title
+        img_url = content.img["src"]
+
+        return [{ "title" : title, "img_url" : img_url, "site": "BugComic", "sort": 50 }]

parsers/icanhaz.py

         for post in post_divs:
             title = post["title"]
             url = post["src"]
-            posts.append({ "title" : title, "img_url" : url, "site":"ICanHazCheezburger"})
+            posts.append({ "title" : title, "img_url" : url, "site":"ICanHazCheezburger", "sort" : 300 })
         return posts
 
 
+import feedparser
+import bs4
+
+class XkcdParser(object):
+    RSS_URL = "http://xkcd.com/rss.xml"
+
+    def get_data(self):
+        feed = feedparser.parse(self.RSS_URL)
+
+        # Grab last entry
+        title = feed.entries[0].title
+        last_entry = bs4.BeautifulSoup(feed.entries[0].summary)
+        subtext = last_entry.img["alt"]
+        img_url = last_entry.img["src"]
+
+        return [ { "title" : title, "subtext": subtext, "img_url":img_url, "site":"XKCD", "sort" : 25 }]
+# -*- coding: utf-8 -*-
+from jinja2 import Template, Environment, FileSystemLoader
+from model import get_undispatched_articles
+
+env = Environment(loader=FileSystemLoader("template"))
+
+def send_mails():
+    articles = get_undispatched_articles(mark_dispatched=True)
+    
+    template = env.get_template("mail.html")
+    rendered_mail = template.render(articles=articles)
+
+    f = open("test.html", "w")
+    f.write(rendered_mail.encode('utf-8'))
+    f.close()
+
+if __name__ == "__main__":
+    send_mails()

template/mail.html

                             <tr>
                                 <td class="w580" width="580">
                                     <div align="left" class="article-content">
-                                      [ {{ article.site }} ] {{ article.subtext }}
+                                      [ {{ article.site }} ] {{ article.subtext|default('', true) }}
                                     </div>
                                 </td>
                             </tr>