Commits

jgsogo committed 3f16611 Draft

testing for shortener_data

Comments (0)

Files changed (4)

shortener_data/admin.py

 from django.contrib import admin
 
-from chimp_shortener_data.models import RequestData, UserAgentType
+from shortener_data.models import RequestData, UserAgent, UserAgentType
 
 
 class RequestDataAdmin(admin.ModelAdmin):
-    list_display = ('datetime', 'user_agent_type', 'referer', 'link')
-    list_filter = ('datetime', 'user_agent_type', 'user_agent_type__is_human',)
+    list_display = ('datetime', 'user_agent', 'referer', 'link')
+    list_filter = ('datetime', 'user_agent', 'user_agent__is_human',)
+
+
+class UserAgentAdmin(admin.ModelAdmin):
+    list_display = ('typ', 'name', 'company', '_hit_robots', 'is_human')
+    list_filter = ('typ', 'is_human')
 
 
 class UserAgentTypeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'is_human')
-    list_filter = ('name', 'is_human')
+    list_display = ('name')
 
 
 admin.site.register(RequestData, RequestDataAdmin)
+admin.site.register(UserAgent, UserAgentAdmin)
 admin.site.register(UserAgentType, UserAgentTypeAdmin)

shortener_data/models.py

 
     # User agent
     typ = models.ForeignKey(UserAgentType)
-    name = models.CharField(max_length=140)
-    family = models.CharField(max_length=140)
+    name = models.CharField(max_length=255)
+    family = models.CharField(max_length=255)
     url = models.URLField()
-    company = models.CharField(max_length=140)
+    company = models.CharField(max_length=255)
     company_url = models.URLField()
     icon = models.URLField()
 
     # Operating system
-    os_name = models.CharField(max_length=140)
-    os_family = models.CharField(max_length=140)
+    os_name = models.CharField(max_length=255)
+    os_family = models.CharField(max_length=255)
     os_url = models.URLField()
-    os_company = models.CharField(max_length=140)
+    os_company = models.CharField(max_length=255)
     os_company_url = models.URLField()
     os_icon = models.URLField()
 
 
     @classmethod
     def create(cls, useragent_string, commit=True):
-        instance, created = cls.objects.get_or_create(user_agent_string=useragent_string)
-        if created:
+        try:
+            instance = cls.objects.get(user_agent_string=useragent_string)
+            return instance
+        except cls.DoesNotExist:
+            instance = cls(user_agent_string=useragent_string)
             try:
                 uasparser = UASparser()
-                ret = uasparser.parse(instance.user_agent_string)
+                ret = uasparser.parse(useragent_string)
                 instance.typ, created = UserAgentType.objects.get_or_create(name=ret['typ'])
             except:
                 pass
             finally:
                 if commit:
                     instance.save()
-        return instance
+            return instance
 
     @classmethod
     def on_robots(cls, useragent_string):
         instance = cls.create(useragent_string)
-        instance._hit_robots = F('_hit_robots')+1
+        instance._hit_robots = models.F('_hit_robots')+1
         instance.save()
 
     @property
     datetime = models.DateTimeField(auto_now_add=True, editable=False)
 
     accept = models.TextField()
-    accept_encoding = models.CharField(max_length=140)
-    accept_language = models.CharField(max_length=140)
-    cache_control = models.CharField(max_length=140)
-    connection = models.CharField(max_length=140)
-    host = models.CharField(max_length=140)
-    referer = models.CharField(max_length=500)
-    user_agent = models.CharField(max_length=140)
-    via = models.CharField(max_length=140)
-    x_forwarded_for = models.CharField(max_length=140)
+    accept_encoding = models.CharField(max_length=255)
+    accept_language = models.CharField(max_length=255)
+    cache_control = models.CharField(max_length=255)
+    connection = models.CharField(max_length=255)
+    host = models.CharField(max_length=255)
+    referer = models.URLField(_('referer'), max_length=500)
+    user_agent = models.CharField(max_length=255)
+    via = models.CharField(max_length=255)
+    x_forwarded_for = models.CharField(max_length=255)
 
-    remote_addr = models.CharField(max_length=15)
+    remote_addr = models.IPAddressField(_('IP address'))
 
     user_agent = models.ForeignKey(UserAgent)
 
     objects = RequestDataManager()
 
     class Meta:
-        verbose_name = _('Request')
-        verbose_name_plural = _('Requests')
+        verbose_name = _('request')
+        verbose_name_plural = _('requests')
+        ordering = ('-datetime',)
+
+    def __unicode__(self):
+        return "[%s] %s" % (self.datetime, self.link)
 
     @classmethod
     def parse_request(cls, request):
         return headers
 
     @classmethod
-    def create(cls, request, link):
+    def create(cls, request, link, commit=True):
         data = cls.parse_request(request)
+        user_agent = data.pop('user_agent')
         request_data = cls(**data)
-        request_data.user_agent = UserAgent.create(request.META['HTTP_USER_AGENT'])
+        request_data.user_agent = UserAgent.create(user_agent)
         request_data.link = link
-        request_data.save()
+        if commit:
+            request_data.save()
         return request_data
 
     @property
 
 def on_request(sender, request, **kwargs):
     try:
-        request_data = RequestData.create(request, sender)
+        request_data = RequestData.create(request, sender, commit=False)
     except:
         request_data = RequestData()
-    request_data.link = sender
-    request_data.save()
+        request_data.link = sender
+    finally:
+        request_data.save()
 
 link_followed.connect(on_request)

shortener_data/tests/__init__.py

 #!/usr/bin/env python
 # encoding: utf-8
+
+from requests import *

shortener_data/tests/requests.py

+#!/usr/bin/env python
+# encoding: utf-8
+
+from django.utils.unittest import TestCase
+from django.test.client import RequestFactory, Client
+
+from shortener.models import Link
+from shortener_data.models import RequestData, UserAgent
+from shortener_data.views import robots
+
+class RequestsTest(TestCase):
+    def setUp(self):
+        self.factory = RequestFactory()
+        self.link = Link.create('http://thisis.valid.url/to/test/')
+        self.c = Client()
+
+    def test_request_creation(self):
+        # User agent strings: http://www.user-agents.org/allagents.xml
+        uas = ( ('browser','Sunrise XP/2.x'),
+                ('other', 'suchpadbot/1.0 (+http://www.suchpad.de)'),
+                ('other','TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17'),
+                ('other', 'TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8'),
+                ('browser', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 3.5.30729) (Prevx 3.0.5)'),
+                )
+        for ua in uas:
+            request = self.factory.get('/%s' % self.link._hash, HTTP_USER_AGENT=ua[1])
+            request_data = RequestData.create(request, self.link)
+            # TODO: write actual test.
+
+    def test_robots(self):
+        uas = 'Advanced Browser (http://www.avantbrowser.com)'
+        request = self.factory.get('/%s' % self.link._hash, HTTP_USER_AGENT=uas)
+
+        # Two 'human' visits ==> human
+        request_data = RequestData.create(request, self.link)
+        request_data = RequestData.create(request, self.link)
+        self.assertEqual(request_data.user_agent._hit_robots, 0)
+        self.assertTrue(request_data.user_agent.is_human)
+
+        # One 'robots' visit ==> human
+        robots(request)
+        user_agent = UserAgent.objects.get(user_agent_string=uas)
+        self.assertEqual(user_agent._hit_robots, 1)
+        self.assertTrue(user_agent.is_human)
+
+        # Another one 'robots' visit ==> robot
+        robots(request)
+        user_agent = UserAgent.objects.get(user_agent_string=uas)
+        self.assertEqual(user_agent._hit_robots, 2)
+        self.assertFalse(user_agent.is_human)
+