Commits

unax  committed 3b7f173

Version 2! It better now, see README

  • Participants
  • Parent commits fd98338

Comments (0)

Files changed (14)

File TESTS_RESULT

-Server and soft:
-OS: Debian 3.1.0-1-amd64 x86_64 GNU/Linux
-CPU: Intel(R) Core(TM)2 Duo T6600 2.20GHz
-mongodb: 2.0.0-2
-Python: 2.7.2+
-Django version: 1.4 alpha
-pymongo: 2.0.1-1
-mongoengine: 0.5.2
-redis-server: 2.4.2-1
-python-redis:  2.4.9-1
-Web-server: nginx 1.1.8-1 + uWSGI 0.9.8.3-debian(64bit)
-
-==Getting object==
--1-
-Test with used cache:
-Count operations: 15000
-Time operations: 8.84495401382
-Test without cached:
-Count operations: 15000
-Time operations: 19.808797121
--2-
-Test with used cache:
-Count operations: 15000
-Time operations: 3.99209499359
-Test without cached:
-Count operations: 15000
-Time operations: 18.7795460224
--3-
-Test with used cache:
-Count operations: 15000
-Time operations: 2.54329681396
-Test without cached:
-Count operations: 15000
-Time operations: 17.7209279537
-
-==Select list==
--1-
-Test with used cache:
-Count operations: 300
-Object count: 207195
-Time operations: 6.14473700523
-Average list length: 690
-Test without cached:
-Count operations: 300
-Object count: 199122
-Time operations: 27.7501080036
-Average list length: 663
--2-
-Test with used cache:
-Count operations: 300
-Object count: 200271
-Time operations: 5.51485180855
-Average list length: 667
-Test without cached:
-Count operations: 300
-Object count: 198798
-Time operations: 27.8188531399
-Average list length: 662
--3-
-Test with used cache:
-Count operations: 300
-Object count: 189700
-Time operations: 5.5311870575
-Average list length: 632
-Test without cached:
-Count operations: 300
-Object count: 180917
-Time operations: 25.7878158092
-Average list length: 603
-
-==Get reference object==
--1-
-Test with used cache:
-Objects count: 30000
-Time operations: 12.0339980125
-Test without cached:
-Objects count: 30000
-Time operations: 28.2414021492
--2-
-Test with used cache:
-Objects count: 30000
-Time operations: 7.19947600365
-Test without cached:
-Objects count: 30000
-Time operations: 28.7138521671
--3-
-Test with used cache:
-Objects count: 30000
-Time operations: 5.26431107521
-Test without cached:
-Objects count: 30000
-Time operations: 27.9019629955
-
-==Get reference list==
--1-
-Test with used cache:
-Count objects in reference list: 35383
-Time operations: 12.6457750797
-Average list length: 4
-Test without cached:
-Count objects in reference list: 35248
-Time operations: 28.0478198528
-Average list length: 4
--2-
-Test with used cache:
-Count objects in reference list: 35005
-Time operations: 6.16148686409
-Average list length: 4
-Test without cached:
-Count objects in reference list: 35254
-Time operations: 27.6127989292
-Average list length: 4
--3-
-Test with used cache:
-Count objects in reference list: 34964
-Time operations: 6.15631604195
-Average list length: 4
-Test without cached:
-Count objects in reference list: 34933
-Time operations: 25.8812391758
-Average list length: 4
-

File django_test_application/__init__.py

Empty file removed.

File django_test_application/manage.py

-#!/usr/bin/env python
-import os, sys
-
-if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
-
-    from django.core.management import execute_from_command_line
-
-    execute_from_command_line(sys.argv)

File django_test_application/mongoengine_rediscache/journal.py

-# -*- coding: utf-8 -*-
-'''
-Created on 12.01.2012
-
-@author: unax
-'''
-from base_cache import _internal_cache as cache
-
-def add_find_record(cache_key, collection, timeout):
-    try:     cache.append_to_list("%s:list:journal:" % collection, cache_key)
-    except:  return
-
-def add_count_record(cache_key, collection, timeout):
-    try:     cache.append_to_list("%s:count:journal:" % collection, cache_key)
-    except:  return
-
-def add_get_record(pk, cache_key, collection, timeout):
-    try:     cache.append_to_list("%s:get:journal:pk=%s" % (collection, str(pk)), cache_key)
-    except:  return
-
-def records(query_type, collection, clarify=''):
-    try:     record_list = cache.get_all_list("%s:%s:journal:%s" % (collection, query_type, clarify))
-    except:  record_list = []
-    if query_type == 'get' and isinstance(record_list, list) and clarify != '':
-        record_list.append('%s:get:%s' % (collection, clarify))
-    return record_list

File django_test_application/nocache_tests/__init__.py

Empty file removed.

File django_test_application/nocache_tests/models.py

-'''
-Created on 25.01.2012
-
-@author: unax
-'''
-
-from datetime import datetime
-from mongoengine import *
-
-class TestModelObj(Document):
-    num  =  IntField(default=0)
-    name =  StringField(max_length=255, required=False )
-    text =  StringField(max_length=255, required=False )
-    create_date = DateTimeField(default=datetime.now())
-
-class TestModelList(Document):
-    num  =  IntField(default=0)
-    name =  StringField(max_length=255, required=False )
-    models = ListField( ReferenceField(TestModelObj) )
-    
-class TestModelRef(Document):
-    num  =  IntField(default=0)
-    name =  StringField(max_length=255, required=False )
-    model = ReferenceField(TestModelObj)

File django_test_application/settings.py

-# -*- coding: utf-8 -*-
-
-DEBUG = True
-TEMPLATE_DEBUG = DEBUG
-
-ADMINS = (,)
-
-MANAGERS = ADMINS
-
-DATABASES = {
-    'default': {
-    'ENGINE': 'django.db.backends.postgresql_psycopg2',
-    'NAME'  : 'testdb',
-    'USER'  : 'admin',
-    'PASSWORD': 'password',
-    'HOST'  : 'localhost',
-    'PORT'  : '5432',
-    },
-    'mongoengine' : {
-    'NAME'  : 'testdb',
-    'USER'  : 'admin',
-    'PASSWORD': 'password',
-    'HOST'  : 'localhost',
-    'PORT'  : 27017,
-    }
-}
-
-from mongoengine import connect
-
-connect(DATABASES['mongoengine'].get('NAME'),
-        username=DATABASES['mongoengine'].get('USER'),
-        password=DATABASES['mongoengine'].get('PASSWORD'),
-        host=DATABASES['mongoengine'].get('HOST'),
-        port=DATABASES['mongoengine'].get('PORT') )
-
-TIME_ZONE = 'Asia/Novosibirsk'
-TIME_INPUT_FORMATS =('%H:%M',)
-DATE_INPUT_FORMATS  =('%H:%M',)
-LANGUAGE_CODE = 'ru'
-SITE_ID = 1
-
-USE_I18N = True
-USE_L10N = True
-USE_TZ = False
-MEDIA_ROOT = ''
-MEDIA_URL = ''
-STATIC_ROOT = ''
-STATIC_URL = '/static/'
-STATICFILES_DIRS = (
-
-)
-
-STATICFILES_FINDERS = (
-    'django.contrib.staticfiles.finders.FileSystemFinder',
-    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
-)
-
-SECRET_KEY = 'j_uptm-k^$wydy!&7inbc4b8e*bmjmnfyl76m^uwn9z)l!vcsa'
-
-TEMPLATE_LOADERS = (
-    'django.template.loaders.app_directories.Loader',
-    'django.template.loaders.filesystem.Loader',
-)
-
-TEMPLATE_CONTEXT_PROCESSORS =(
-                              "django.contrib.auth.context_processors.auth",
-                              "django.core.context_processors.debug",
-                              "django.core.context_processors.i18n",
-                              "django.core.context_processors.media",
-                              "django.core.context_processors.static",
-                              "django.contrib.messages.context_processors.messages"
-                              )
-
-MIDDLEWARE_CLASSES = (
-    'django.middleware.common.CommonMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.gzip.GZipMiddleware',
-)
-
-
-ROOT_URLCONF = 'urls'
-
-
-
-INSTALLED_APPS = (
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.sites',
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
-    'django.contrib.admin',
-    'tests',
-    'mongoengine_rediscache',
-)
-
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse'
-        }
-    },
-    'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
-        }
-    },
-    'loggers': {
-        'django.request': {
-            'handlers': ['mail_admins'],
-            'level': 'ERROR',
-            'propagate': True,
-        },
-    }
-}
-
-
-CACHE_USE=True
-
-
-MONGOENGINE_REDISCACHE = {
-    'scheme' : {
-                'tests.models.TestModelObj'  : { 'all' : 600 },
-                'tests.models.TestModelList' : { 'all' : 600 },
-                'tests.models.TestModelRef'  : { 'list' : 120, 'reference' : 600, 'get' : 120, 'list_reference' : 600 },
-                },
-    'redis' : {
-        'host': 'localhost',
-        'port': 6379,
-        'db': 1, 
-        'socket_timeout': 3,
-    },
-    'used' : True,
-    'keyhashed' : False,
-}

File django_test_application/tests/__init__.py

-# -*- coding: utf-8 -*-
-from settings import MONGOENGINE_REDISCACHE
-from django.core.exceptions import ImproperlyConfigured
-import redis
-#try:
-#    import cPickle as pickle
-#except ImportError:
-import pickle
-    
-from functools import wraps
-import os, time
-from django.utils.hashcompat import md5_constructor
-# redis connecting
-try:
-    redis_conf = MONGOENGINE_REDISCACHE.get('redis')
-except AttributeError:
-    raise ImproperlyConfigured('Check MONGOENGINE_REDISCACHE in settings. ')
-
-try:
-    redis_conn = redis.Redis(**redis_conf)
-except:
-    redis_conn = None
-
-
-
-class _queryset_list(list):
-    def __init__(self, anylist=None):
-        if anylist is None:
-            super(_queryset_list, self).__init__()
-        else:
-            super(_queryset_list, self).__init__(anylist)
-    
-    def count(self):
-        return len(self)
-
-class CacheMiss(Exception):
-    pass
-
-class BaseCache(object):
-    """
-    Simple cache with time-based invalidation
-    """
-    def cached(self, extra=None, timeout=None):
-        """
-        A decorator for caching function calls
-        """
-        def decorator(func):
-            @wraps(func)
-            def wrapper(*args, **kwargs):
-                # Calculating cache key based on func and arguments
-                md5 = md5_constructor()
-                md5.update('%s.%s' % (func.__module__, func.__name__))
-                if extra is not None:
-                    md5.update(str(extra))
-                if args:
-                    md5.update(repr(args))
-                if kwargs:
-                    md5.update(repr(sorted(kwargs.items())))
-
-                cache_key = 'c:%s' % md5.hexdigest()
-
-                try:
-                    result = self.get(cache_key)
-                except CacheMiss:
-                    result = func(*args, **kwargs)
-                    self.set(cache_key, result, timeout)
-
-                return result
-            return wrapper
-        return decorator
-
-class RedisCache(BaseCache):
-    def __init__(self, conn):
-        self.conn = conn
-
-    def get(self, cache_key):
-        if self.conn is None:
-            return None
-        data = self.conn.get(cache_key)
-        if data is None:
-            return None
-        return pickle.loads(data)
-    
-    def pipeline_get(self, cache_key_list ):
-        if isinstance(cache_key_list,list) and len(cache_key_list)>0:
-            pipe = self.conn.pipeline()
-            for key in cache_key_list:
-                pipe.get(key)
-            data=pipe.execute()
-            if data is not None and len(data)>0:
-                res=[]
-                for d in data:
-                    try: obj=pickle.loads(d)
-                    except: obj=None
-                    if obj is not None:
-                        res.append(obj)
-                return res
-        return None
-    
-    def pipeline_delete(self, cache_key_list ):
-        if isinstance(cache_key_list,list) and len(cache_key_list)>0:
-            pipe = self.conn.pipeline()
-            for key in cache_key_list:
-                pipe.delete(key)
-            data=pipe.execute()
-            if data is not None and len(data)>0:
-                return data
-        return None
-    
-    def delete(self, cache_key ):
-        return self.conn.delete(cache_key)
-
-    def set(self, cache_key, data, timeout=None):
-        if self.conn is None:
-            return
-        pickled_data = pickle.dumps(data)
-        if timeout is not None:
-            self.conn.setex(cache_key, pickled_data, timeout)
-        else:
-            self.conn.set(cache_key, pickled_data)
-
-cache = RedisCache(redis_conn)
-_internal_cache = RedisCache(redis_conn)
-
-def install_signals(app):
-    from mongoengine import signals
-    for model_name in MONGOENGINE_REDISCACHE.get('scheme'):
-        try:
-            exec('from %s.models import %s' % (app,model_name))             
-            exec("signals.post_save.connect({0}.post_save, sender={0})".format(model_name) )
-            exec("signals.post_delete.connect({0}.post_delete, sender={0})".format(model_name) )
-        except:
-            pass

File django_test_application/tests/main.py

-'''
-Created on 25.01.2012
-
-@author: unax
-'''
-
-from random import Random
-import time
-
-def run_test_get( n=1000, use_cache=True ):
-    if use_cache:
-        from models import TestModelObj
-    else:
-        from nocache_tests.models import TestModelObj
-    
-    all_pk_mobj= [ str(m.pk) for m in TestModelObj.objects() ]
-    rand=Random()
-    sum=0
-    start_time=time.time()
-    for i in range(n):
-        new_obj=TestModelObj.objects.get(pk= rand.choice(all_pk_mobj) )
-        sum+=new_obj.num
-    end_time=time.time()-start_time
-    
-    return "<p>Count operations: %d</p><p>Time operations: %s</p>" % (n, str(end_time))
-    
-
-def run_test_list( n=100, use_cache=True ):
-    if use_cache:
-        from models import TestModelObj
-    else:
-        from nocache_tests.models import TestModelObj
-    obj_count=0
-    from datetime import datetime
-    dt=datetime(1986,11,22,7,15)
-    rand=Random()
-    rand_hexname=lambda rand: "".join([ rand.choice('abcdef') for i in range(2) ])
-    start_time=time.time()
-    m=0
-    for i in range(n):
-        r=rand.randint(1, 15)
-        if rand.randint(1, 10)>5:
-            new_list=TestModelObj.objects.filter(create_date__gt=dt)
-        else:
-            new_list=TestModelObj.objects()
-        if r>12:
-            new_list.filter( num__lt = 500000 ).filter( text__contains = rand_hexname(rand) ).order_by("num")
-        elif r>9:
-            new_list.filter( text__contains = rand_hexname(rand) ).order_by("num")
-        elif r>6:
-            new_list.filter( name__contains = rand_hexname(rand) ).order_by("-name")
-        elif r>3:
-            new_list.filter( num__lt = 500000 ).order_by("-name")
-        else:
-            new_list.filter( num__gt = 500000 ).order_by("-name")
-        
-        if use_cache:
-            new_list=new_list.cache
-        
-        if new_list.count()>0:
-            m+=1
-            sum=0
-            for obj in new_list:
-                sum+=obj.num
-                obj_count+=1
-            
-    end_time=time.time()-start_time
-    
-    return "<p>Count operations: %d</p><p>Object count: %d</p><p>Time operations: %s</p><p>Average list length: %d</p>" % (n, obj_count, str(end_time), int(obj_count/m))
-
-def run_test_get_reference( n=1000, use_cache=True ):
-    if use_cache:
-        from models import TestModelRef
-    else:
-        from nocache_tests.models import TestModelRef
-    all_pk_mobj= [ str(m.pk) for m in TestModelRef.objects() ]
-    rand=Random()
-    sum=0
-    start_time=time.time()
-    for i in range(n):
-        new_obj=TestModelRef.objects.get(pk=rand.choice(all_pk_mobj) )
-        ref_obj=new_obj.model
-        sum+=ref_obj.num
-    end_time=time.time()-start_time
-    
-    return "<p>Objects count: %d</p><p>Time operations: %s</p>" % (n*2, str(end_time))
-
-def run_test_list_reference( n=1000, use_cache=True ):
-    if use_cache:
-        from models import TestModelList
-    else:
-        from nocache_tests.models import TestModelList
-    all_pk_mobj= [ str(m.pk) for m in TestModelList.objects() ]
-    rand=Random()
-    sum=0
-    obj_count=0
-    m=0
-    start_time=time.time()
-    for i in range(n):
-        new_obj=TestModelList.objects.get(pk=rand.choice(all_pk_mobj) )
-        sum=0
-        for obj in new_obj.models:
-            sum+=obj.num
-            m+=1
-        if sum>0:
-            obj_count+=1
-    end_time=time.time()-start_time
-    
-    return "<p>Count objects in reference list: %d</p><p>Time operations: %s</p><p>Average list length: %d</p>" % (m, str(end_time), int(m/obj_count))

File django_test_application/tests/models.py

-'''
-Created on 24.01.2012
-
-@author: unax
-'''
-
-from datetime import datetime
-from mongoengine import *
-from mongoengine_rediscache.fields import ReferenceFieldCached, ListFieldCached
-from mongoengine_rediscache.queryset import CachedQuerySet
-
-class TestModelObj(Document):
-    num  =  IntField(default=0)
-    name =  StringField(max_length=255, required=False )
-    text =  StringField(max_length=255, required=False )
-    create_date = DateTimeField(default=datetime.now())
-    
-    meta = { 'queryset_class': CachedQuerySet, 'cascade' : False }
-
-class TestModelList(Document):
-    num  =  IntField(default=0)
-    name =  StringField(max_length=255, required=False )
-    models = ListFieldCached( ReferenceField(TestModelObj) )
-    
-    meta = { 'queryset_class': CachedQuerySet, 'cascade' : False }
-    
-class TestModelRef(Document):
-    num  =  IntField(default=0)
-    name =  StringField(max_length=255, required=False )
-    model = ReferenceFieldCached(TestModelObj)
-    
-    meta = { 'queryset_class': CachedQuerySet, 'cascade' : False }
-    

File django_test_application/tests/views.py

-'''
-Created on 24.01.2012
-
-@author: unax
-'''
-from django.http import HttpResponse
-from models import TestModelList, TestModelObj, TestModelRef
-from random import Random
-from main import run_test_get, run_test_list, run_test_get_reference, run_test_list_reference
-
-def main_page(request):
-    return HttpResponse("""<h2>Tests for mongoengine_rediscache</h2>
-    <ul>
-    <li><a href="/create/2000">Create test collections (4000 objects of)</a></li>
-    <p><b>Run tests. For pure experimentation, run each test more than once.</b></p>
-    <li><a href="/test_get">Getting objects (TestModelObj.objects.get)</a></li>
-    <li><a href="/test_list">Getting objects list (test for CachedQuerySet with any variants of filter(..) and order_by()</a></li>
-    <li><a href="/test_get_reference">Getting objects from reference field (ReferenceFieldCached(Document))</a></li>
-    <li><a href="/test_list_reference">Getting list of reference objects (ListFieldCached( ReferenceField(Document) ))</a></li>
-    </ul>""")
-
-def test_get(request):
-    html='<p>Getting object</p>'
-    
-    html+='<ul><b>Test with used cache:</b> %s</ul>' % run_test_get(15000,True)
-    html+='<ul><b>Test without cached:</b> %s</ul>'  % run_test_get(15000,False)
-    
-    return HttpResponse(html)
-
-def test_list(request):
-    html='<p>Select list</p>'
-
-    html+='<ul><b>Test with used cache:</b> %s</ul>' % run_test_list(300,True)
-    html+='<ul><b>Test without cached:</b> %s</ul>'  % run_test_list(300,False)
-    
-    return HttpResponse(html)
-
-
-def test_get_reference(request):
-    html='<p>Get reference object</p>'
-
-    html+='<ul><b>Test with used cache:</b> %s</ul>' % run_test_get_reference(15000,True)
-    html+='<ul><b>Test without cached:</b> %s</ul>'  % run_test_get_reference(15000,False)
-    
-    return HttpResponse(html)
-
-def test_list_reference(request):
-    html='<p>Get reference list</p>'
-
-    html+='<ul><b>Test with used cache:</b> %s</ul>' % run_test_list_reference(10000,True)
-    html+='<ul><b>Test without cached:</b> %s</ul>'  % run_test_list_reference(10000,False)
-    
-    return HttpResponse(html)
-
-def create_models(request, count_models):
-    count_models=int(count_models)
-    
-    rand=Random()
-    
-    rand_hextext=lambda rand: "".join([ rand.choice('0123456789abcdef') for i in range(rand.randint(8,16)) ])
-    rand_hexname=lambda rand: "".join([ rand.choice('_abcdef') for i in range(rand.randint(2,4)) ])
-    
-    nums=[]
-    all=0
-    for i in range(count_models):
-        all+=1
-        n=rand.randint(0,1000000)
-        nums.append(n)
-        TestModelObj(num=n,
-                    name=rand_hexname(rand),
-                    text=rand_hextext(rand)).save()
-
-    for i in range(count_models/2):
-        all+=1
-        TestModelRef(num=rand.randint(0,1000000),
-                     model=TestModelObj.objects(num=rand.choice(nums))[0],
-                     name=rand_hexname(rand)).save()
-    
-    for i in range(count_models/2):
-        all+=1
-        TestModelList(num=rand.randint(0,1000000),
-                     models=TestModelObj.objects.filter(num__in= [ rand.choice(nums) for i in range( rand.randint(1,8) ) ]),
-                     name=rand_hexname(rand)).save()
-    return HttpResponse('Created %d models' % all)

File django_test_application/urls.py

-from django.conf.urls import patterns, include, url
-from django.contrib import admin
-admin.autodiscover()
-
-urlpatterns = patterns('',
-    url( r'^create/(?P<count_models>[0-9]{1,9})[/]{0,1}','tests.views.create_models', ),
-    url( r'^test_get[/]{0,1}$', 'tests.views.test_get' ),
-    url( r'^test_list[/]{0,1}$', 'tests.views.test_list' ),
-    url( r'^test_get_reference[/]{0,1}$', 'tests.views.test_get_reference' ),
-    url( r'^test_list_reference[/]{0,1}$', 'tests.views.test_list_reference' ),
-    url( r'^admin[/]{0,1}', include(admin.site.urls)),
-    url( r'^[/]{0,1}', 'tests.views.main_page' ),
-)

File django_test_application/webapp_uwsgi.py

-# -*- coding: utf-8 -*-
-import  os,sys
-sys.path.append('/usr/local/lib/python2.7/dist-packages/django/')
-sys.path.append('/data/web/cache_test/cachetest')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
-import django.core.handlers.wsgi
-application = django.core.handlers.wsgi.WSGIHandler()

File mongoengine_rediscache/journal.py

-# -*- coding: utf-8 -*-
-'''
-Created on 12.01.2012
-
-@author: unax
-'''
-from base_cache import _internal_cache as cache
-
-def add_find_record(cache_key, collection, timeout):
-    try:     cache.append_to_list("%s:list:journal:" % collection, cache_key)
-    except:  return
-
-def add_count_record(cache_key, collection, timeout):
-    try:     cache.append_to_list("%s:count:journal:" % collection, cache_key)
-    except:  return
-
-def add_get_record(pk, cache_key, collection, timeout):
-    try:     cache.append_to_list("%s:get:journal:pk=%s" % (collection, str(pk)), cache_key)
-    except:  return
-
-def records(query_type, collection, clarify=''):
-    try:     record_list = cache.get_all_list("%s:%s:journal:%s" % (collection, query_type, clarify))
-    except:  record_list = []
-    if query_type == 'get' and isinstance(record_list, list) and clarify != '':
-        record_list.append('%s:get:%s' % (collection, clarify))
-    return record_list