Commits

Stephen Tanner  committed 6361a44

Can now grab links, store them in the DB, and send an email of new links

  • Participants
  • Parent commits d136ec6

Comments (0)

Files changed (1)

File webmaster.py

-import ipdb
 #This file will hold the helper functions to allow the scavenger app to handle the
 #generic features of fetching links and parsing posts.
 from bs4 import BeautifulSoup
         for site in sites:
             url = sites[site]['domain'] + sites[site]['path']
             links = yield from self.fetch_links(url, sites[site]['link_class'])
-
+            site_info = sites[site]
+            site_info['name'] = site
             new_links = self.db_filter(links, sites[site]['domain'])
 
-
             for link in new_links:
-                post = yield from self.build_post(link, sites[site])
+                post = yield from self.build_post(link, site_info)
                 posts.append(post)
 
         if posts:
             self.insert_posts(posts)
             self.email_posts(posts)
 
-        print("Number of new posts: " + len(posts))
+        print("Number of new posts: " + str(len(posts)))
 
         return len(posts)
 
         short = yield from self.shorten_url(href)
         page = yield from self._GET(href, compress='True')
         content = self.parse_post(page, site['post_class'])
-        title = str(link.string)
         now = datetime.datetime.now().isoformat(' ')
-        return {"url": href, "site": site, "title": title, "content": content, "short": short, "time_stamp": now}
+        return {"url": href, "site": site['name'], "title":  str(link.string), "content": str(content), "short": short, "time_stamp": now}
 
 
 
         headers = [
     		"From: <" + self.email["from"] + ">",
             "To: <" + self.email["to"] + ">",
-            "Subject: " + len(posts) + " New Links Posted",
+            "Subject: " + str(len(posts)) + " New Links Posted",
             "Content-Type: text/html"
         ]
 
+
         session = smtplib.SMTP(self.email['host'], self.email['port'])
         session.ehlo()
         session.starttls() # Omit if SMTPS
         session.ehlo() # Omit if SMTPS
+
+
         session.login(self.email['user'], self.email['pass'])
-        session.sendmail(_from, _to, "\r\n".join(headers) + "\r\n\r\n" + body)
+
+
+        session.sendmail(self.email["from"], self.email["to"], "\r\n".join(headers) + "\r\n\r\n" + body)
         session.quit()
 
     #Need to build an html table of links
     def insert_posts(self, posts):
         inserts = [[post['url'], post['site'], post['title'], post['content'], post['short'], post['time_stamp']] for post in posts]
         self.db_cursor.executemany('INSERT INTO posts VALUES (?,?,?,?,?,?)', inserts)
+        self.db_conn.commit()
 
 
 
     def parse_post(self, page, post_class):
         bs_page = BeautifulSoup(page)
         post = bs_page.find("div", class_=post_class)
+
         return post