Commits

pbourgel committed 396f194

Fixed a few things with the REST API.

Comments (0)

Files changed (4)

         result.ipaddrlist = str(ip_struct[2])
         result.link = self.link
         result.link_uri = self.link.req_format()
+        result.title = self.link.title
         rdate = ''
         if direct_html or proxy_html:
             if direct_html:
     img_tags = models.TextField()
     object_tags = models.TextField()
     timestamp = models.DateTimeField()
+    title = models.CharField(max_length=50)
 
     header = JSONField() # { 'direct': HEADERS, 'proxy': HEADERS }
     #screenshot = JSONField() # { 'direct': PATH, 'proxy': PATH }
         if result_hash:
             result = res.filter(hash=result_hash)
             return result
-        result_uri = 'http://' + result_uri + '/'
+        #result_uri = 'http://' + result_uri + '/'
         result_uri = result_uri.decode('utf8')
         print 'attempting query with url %s' % result_uri
-        #TO-DO: Handle requests for HTTPS pages as well as HTTP
-        results = res.filter(link_uri=result_uri)
+        results = res.filter(title=result_uri)
         if results:
             return results
         else:

htmlgrab/models.pyc

Binary file modified.

htmlgrab/tasks.py

 from InetRequest import Link, ProxyServer
 #TO-DO: Change this when I move the project off my laptop
 
-#
-#
-#
-
-#def mhgj_wrapper():
-#    t = timedelta(days=1)
-#    s = Scan()
-#    s.title = 'blah blah'
-#    s.start_date=datetime.now()
-#    s.end_date=s.start_date + t
-#    l = Link.objects.all()
-#    my_html_grab_job(None,s,l)
-
-#def my_html_grab_job(proxy_inf, sc, links):
-#    for link in links:
-#        htmr = HTMLRequest(proxy_info=proxy_inf)
-#        htmr.set_uri(link=link)
-#        htmr.perform(sc)
-
-
-#@task(name='html-grab-job')
-#def html_grab_job(proxy_inf, sc, links):
-#    for link in links:
-#        htmr = HTMLRequest(proxy_info=proxy_inf)
-#        htmr.set_uri(link=link)
-#        htmr.perform(sc)
+class QException(BaseException):
+    pass
 
 @task(name='update-ips')
 def update_ips():
     else:
         some_links = Link.objects.all()[:num_links]
     pserver = ProxyServer.objects.filter(proxy_name=proxy)
-    ps = pserver[0]
+    if len(pserver) > 0:
+        ps = pserver[0]
+    else:
+        raise QException("Error: Proxy with specified name not found.")
+
     print "collected links and proxy server"
     for l in some_links:
         htmr = HTMLRequest(proxy_info=ps)