Commits

EGh5 committed 8432fe0

memcached and logging add

Comments (0)

Files changed (4)

doorkeeper/__init__.py

 """
-Django-doorkeeper help to deals with malicious url request
+Django-doorkeeper helps to deal with malicious url requests
 """
-version = (1, 0,1)
+version = (1,5,5)
 __version__ = '.'.join(map(str, version))

doorkeeper/middleware.py

 # -*- coding: utf-8 -*-
 # Creation date: 10.03.12
 
-import datetime
-
+import datetime, hashlib
 from django.http import Http404
 from doorkeeper.models import BotRecord
 from django.conf import settings
+from utils import MemClient
+
+import logging
+log = logging.getLogger(__name__)
 
 PATIENCE= getattr(settings,'DOORKEEPER_PATIENCE', 5)
+MEMCACHE=getattr(settings,'DOORKEEPER_MEMCACHE', False)
+TERM = getattr(settings,'DOORKEEPER_TERM', 1200)
+
 
 class DoorkeeperMiddleware(object):
     def process_request(self, request):
         """
           Each request is investigated - overheads for suspicions
+          If memcahed is used:
+            overheads = md5-hash calculation + memcache.get request
+          I think its a reasonable price
         """
         sagent = request.META.get('HTTP_USER_AGENT',"n/a")
         saddr = request.META.get('REMOTE_ADDR',"n/a")
         shash = "%s %s"% (saddr, sagent)
-        try:
-            br=BotRecord.objects.get(bot_hash=shash) # if client exists in BotRecords
+        dd=hashlib.md5()
+        dd.update(shash)
+        memhash=dd.hexdigest()
+
+        if MEMCACHE:                                 # ####  path for MEMCACHED (fast)
+            mc=MemClient(MEMCACHE)
+            br=mc.get(memhash)
+            if br is None:
+                return                               # ==>> exit for 99.9% requests
+            
             now=datetime.datetime.now()
+            
             if br.detention_upto:                    # check detention end time
                 if br.detention_upto> now:           # if it is a fresh banned client
-                    raise Http404()                  # bounce - it's active bad bot!
+                    log.error("Memcached middleware  %s penalty %s, detention %s for %s" % (shash,br.penalty_count,
+                         br.detention_upto, request.path))
+                    raise Http404()                  # ==>> bounce - it's active bad bot!
                 else:
                     if br.penalty_count > PATIENCE:  # for old banned client
                         br.penalty_count-=PATIENCE   # forgiving them partly
                         br.save()
-                    pass
-        except BotRecord.DoesNotExist:
-            pass
-
-
-
+                        mc.add(memhash,br,time=TERM*2)
+                    pass                             # ==>> it was a bad bot, maybe not now?
 
+        else:                                        # ### path for DB-driven (slow)
+            try:
+                br=BotRecord.objects.get(bot_hash=shash) # if client exists in BotRecords
+                now=datetime.datetime.now()
+                if br.detention_upto:                    # check detention end time
+                    if br.detention_upto> now:           # if it is a fresh banned client
+                        raise Http404()                  # bounce - it's active bad bot!
+                    else:
+                        if br.penalty_count > PATIENCE:  # for old banned client
+                            br.penalty_count-=PATIENCE   # forgiving them partly
+                            br.save()                    # ==>> it was a bad bot, maybe not now?
+                        pass
+            except BotRecord.DoesNotExist:               # ==>> exit for most requests
+                pass

doorkeeper/utils.py

+# -*- coding: utf-8 -*-
+# Creation date: 18.03.12
+
+import memcache
+
+class Singleton(type):
+     def __init__(cls, name, bases, dict):
+         super(Singleton, cls).__init__(name, bases, dict)
+         cls.instance = None
+     def __call__(cls,*args,**kw):
+         if cls.instance is None:
+             cls.instance = super(Singleton, cls).__call__(*args, **kw)
+         return cls.instance
+
+class MemClient(memcache.Client):
+     __metaclass__ = Singleton
+

doorkeeper/views.py

 
 from models import BotRecord
 import datetime
-from users import views
 from django.conf import settings
+import hashlib
+from utils import MemClient
+import logging
+log = logging.getLogger(__name__)
 
 TERM = getattr(settings,'DOORKEEPER_TERM', 1200)
 PATIENCE= getattr(settings,'DOORKEEPER_PATIENCE', 5)
+MEMCACHE=getattr(settings,'DOORKEEPER_MEMCACHE', False)
 
 def inspect(request):
     """
+
     View to register suspicious client and raise Http404
+    if bad request frequency is higher then PATIENCE
+    counter BotRecord.penalty_count increases until threshold.
+    Then detention_upto is set to future and middleware begins bouncing
+    this client.
 
     """
-    sagent  =request.META.get('HTTP_USER_AGENT',"n/a")
-    saddr=request.META.get('REMOTE_ADDR',"n/a")
-    shash="%s %s"% (saddr, sagent)
+    sagent = request.META.get('HTTP_USER_AGENT',"n/a")
+    saddr = request.META.get('REMOTE_ADDR',"n/a")
+    shash = "%s %s"% (saddr, sagent)
+    dd    = hashlib.md5()
+    dd.update(shash)
+    memhash = dd.hexdigest()
+
     try:
-        br=BotRecord.objects.get(bot_hash=shash)
+        if MEMCACHE:
+            mc=MemClient(MEMCACHE)
+            br=mc.get(memhash)
+            
+            if br is None:
+                br=BotRecord.objects.get(bot_hash=shash)
+                mc.add(memhash,br,time=TERM*2)
+        else:
+            br=BotRecord.objects.get(bot_hash=shash)
         now=datetime.datetime.now()
         delta=datetime.timedelta(0,TERM)
         if br.last_date+delta > now:
             if br.penalty_count > PATIENCE:
                 br.detention_upto=now+delta
                 br.jail_count+=1
-        br.save()
+        else:
+            if br.penalty_count > PATIENCE:  # for old banned client
+                br.penalty_count-=PATIENCE   # forgiving them partly
     except BotRecord.DoesNotExist:
         br=BotRecord.objects.create(bot_hash=shash,bot_name=sagent, bot_adr=saddr)
-        br.save()
+    br.save()
+    if MEMCACHE: 
+        mc.set(memhash,br)
+        log.error("Memcached view %s, penalty %s, detention %s" % (shash,br.penalty_count,br.detention_upto))
     raise Http404()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.