Commits

jgsogo committed e0cf51f Draft

working on shortener data requests

Comments (0)

Files changed (6)

shortener_data/managers.py

 
 class RequestDataManager(models.Manager):
     def human(self):
-        return self.filter(Q(user_agent_type__is_human=True), Q(user_agent_has_url=False))
+        return self.filter(Q(user_agent__is_human=True))

shortener_data/models.py

 
 class UserAgentType(models.Model):
     name = models.CharField(max_length=140)
-    is_human = models.BooleanField(default=False)
 
     def __unicode__(self):
-        return self.name
+        return str(self.name)
+
+class UserAgent(models.Model):
+    user_agent_string = models.TextField()
+    #_is_human = models.FloatField(default=1)
+    _hit_robots = models.PositiveIntegerField(default=0)
+
+    # User agent
+    typ = models.ForeignKey(UserAgentType)
+    name = models.CharField(max_length=140)
+    family = models.CharField(max_length=140)
+    url = models.URLField()
+    company = models.CharField(max_length=140)
+    company_url = models.URLField()
+    icon = models.URLField()
+
+    # Operating system
+    os_name = models.CharField(max_length=140)
+    os_family = models.CharField(max_length=140)
+    os_url = models.URLField()
+    os_company = models.CharField(max_length=140)
+    os_company_url = models.URLField()
+    os_icon = models.URLField()
+
+    def __unicode__(self):
+        return str(self.typ)
+
+    @classmethod
+    def create(cls, useragent_string, commit=True):
+        instance, created = cls.objects.get_or_create(user_agent_string=useragent_string)
+        if created:
+            try:
+                uasparser = UASparser()
+                ret = uasparser.parse(instance.user_agent_string)
+                instance.typ, created = UserAgentType.objects.get_or_create(name=ret['typ'])
+            except:
+                pass
+            finally:
+                if commit:
+                    instance.save()
+        return instance
+
+    @classmethod
+    def on_robots(cls, useragent_string):
+        instance = cls.create(useragent_string)
+        instance._hit_robots = F('_hit_robots')+1
+        instance.save()
+
+    @property
+    def is_human(self):
+        # End user must be able to modify confidence interval
+        return  len(self.requestdata_set.all()) > self._hit_robots
 
 
 class RequestData(models.Model):
     link = models.ForeignKey(Link)
     datetime = models.DateTimeField(auto_now_add=True, editable=False)
 
-    meta = models.TextField()
     accept = models.TextField()
     accept_encoding = models.CharField(max_length=140)
     accept_language = models.CharField(max_length=140)
 
     remote_addr = models.CharField(max_length=15)
 
-    user_agent_type = models.ForeignKey(UserAgentType)
-    user_agent_has_url = models.BooleanField(default=False)
+    user_agent = models.ForeignKey(UserAgent)
 
     objects = RequestDataManager()
 
         headers.update({'remote_addr' : request.META['REMOTE_ADDR']})
         return headers
 
+    @classmethod
+    def create(cls, request, link):
+        data = cls.parse_request(request)
+        request_data = cls(**data)
+        request_data.user_agent = UserAgent.create(request.META['HTTP_USER_AGENT'])
+        request_data.link = link
+        request_data.save()
+        return request_data
+
 
 """
     Signals
 
 def on_request(sender, request, **kwargs):
     try:
-        data = RequestData.parse_request(request)
-        request_data = RequestData(**data)
-        uasparser = UASparser()
-        ret = uasparser.parse(request.META['HTTP_USER_AGENT'])
-        request_data.user_agent_type, created = UserAgentType.objects.get_or_create(name=ret['typ'])
-        #request_data.user_agent_url = ret['ua_url']
-        request_data.user_agent_has_url = True if re.search(LINK_RE, request.META['HTTP_USER_AGENT']) else False
+        request_data = RequestData.create(request, sender)
     except:
         request_data = RequestData()
     request_data.link = sender

shortener_data/settings.py

+#!/usr/bin/env python
+# encoding: utf-8
+
+from django.conf import settings
+
+ROBOTS_FILEPATH = getattr(settings, 'SHORTENER_ROBOTS_STR', None)

shortener_data/tests.py

-"""
-This file demonstrates writing tests using the unittest module. These will pass
-when you run "manage.py test".
-
-Replace this with more appropriate tests for your application.
-"""
-
-from django.test import TestCase
-
-
-class SimpleTest(TestCase):
-    def test_basic_addition(self):
-        """
-        Tests that 1 + 1 always equals 2.
-        """
-        self.assertEqual(1 + 1, 2)

shortener_data/tests/__init__.py

+#!/usr/bin/env python
+# encoding: utf-8

shortener_data/views.py

-import re
 
-from chimp_shortener_data.models import UserAgentType, LINK_RE
-from chimp_shortener_data.utils import UASparser
+from django.http import HttpResponse
+
+from shortener_data.models import UserAgent
+from shortener_data.settings import ROBOTS_FILEPATH
 
 def robots(request):
-    uasparser = UASparser()
-    ret = uasparser.parse(request.META['HTTP_USER_AGENT'])
-    user_agent_type, created = UserAgentType.objects.get_or_create(name=ret['typ'])
-    user_agent_has_url = True if re.search(LINK_RE, request.META['HTTP_USER_AGENT']) else False
+    UserAgent.on_robots(request.META['HTTP_USER_AGENT'])
+    robots_str = "User-agent: *\nDisallow: /\n"
+    if ROBOTS_FILEPATH:
+        try:
+            robots_str = open(ROBOTS_FILEPATH, 'r').read()
+        except IOError:
+            pass
+    return HttpResponse(robots_str, mimetype="text/plain")
+
+
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.