Commits

Peter Sanchez  committed 274d2f5

Added queryset_iterator helper function.

  • Participants
  • Parent commits e52c7fb
  • Branches trunk

Comments (0)

Files changed (1)

File webutils/djtools/helpers.py

+import gc
 from django.conf import settings
 from django.template.defaultfilters import slugify
 from webutils.helpers import grab_from_import
             new_slug = "%s%d" % (new_slug_base, i)
             i += 1
     return new_slug
+
+
+# Taken from http://djangosnippets.org/snippets/1949/
+def queryset_iterator(queryset, chunksize=1000):
+    ''' Iterate over a Django Queryset ordered by the primary key
+
+        This method loads a maximum of chunksize (default: 1000) rows in it's
+        memory at the same time while django normally would load all rows in it's
+        memory. Using the iterator() method only causes it to not preload all the
+        classes.
+
+        Note that the implementation of the iterator does not support ordered query sets.
+        
+        Example Usage:
+
+        my_queryset = queryset_iterator(MyModel.objects.all())
+        for row in my_queryset:
+            print row.pk
+    '''
+    pk = 0
+    last_pk = queryset.order_by('-pk')[0].pk
+    queryset = queryset.order_by('pk')
+    while pk < last_pk:
+        for row in queryset.filter(pk__gt=pk)[:chunksize]:
+            pk = row.pk
+            yield row
+        gc.collect()