Commits

Gregory Petukhov  committed 173c6b1

Add grab.tools.memory.memory_usage function

  • Participants
  • Parent commits 1348d75

Comments (0)

Files changed (3)

File grab/tools/memory.py

+import os
+
+SCALE = {'kB': 1024.0, 'mB': 1024.0 * 1024.0,
+         'KB': 1024.0, 'MB': 1024.0 * 1024.0}
+
+def memory_usage(since=0, render=True):
+    """
+    Return resident memory usage in bytes.
+    """
+
+    proc_status = '/proc/%d/status' % os.getpid()
+    try:
+        status = open(proc_status).read()
+    except:
+        return 0
+    else:
+        line = [x for x in status.splitlines() if 'VmRSS:' in x][0]
+        items = line.split('VmRSS:')[1].strip().split(' ')
+        mem = float(items[0]) * SCALE[items[1]] - since
+        if render:
+            metrics = ['b', 'Kb', 'Mb', 'Gb']
+            metric = metrics.pop(0)
+            for x in xrange(3):
+                if mem > 1024:
+                    mem = mem / 1024.0
+                    metric = metrics.pop(0)
+            return '%s %s' % (str(round(mem, 2)), metric)
+        else:
+            return mem
+
+
 import random
 import os
 import time
+from grab.tools.memory import memory_usage
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger('[test memory leaks]')
 
-SCALE = {'kB': 1024.0, 'mB': 1024.0 * 1024.0,
-         'KB': 1024.0, 'MB': 1024.0 * 1024.0}
-
-def memory_usage(since=0, render=True):
-    """
-    Return resident memory usage in bytes.
-    """
-
-    proc_status = '/proc/%d/status' % os.getpid()
-    try:
-        status = open(proc_status).read()
-    except:
-        return 0
-    else:
-        line = [x for x in status.splitlines() if 'VmRSS:' in x][0]
-        items = line.split('VmRSS:')[1].strip().split(' ')
-        mem = float(items[0]) * SCALE[items[1]] - since
-        if render:
-            metrics = ['b', 'Kb', 'Mb', 'Gb']
-            metric = metrics.pop(0)
-            for x in xrange(3):
-                if mem > 1024:
-                    mem = mem / 1024.0
-                    metric = metrics.pop(0)
-            return '%s %s' % (str(round(mem, 2)), metric)
-        else:
-            return mem
-
-
+CACHE = []
 
 class TestSpider(Spider):
     initial_urls = ['http://load.local/31m.html']
         logger.debug('Memory usage before %s' % memory_usage())
         #grab.tree.make_links_absolute(grab.response.url)
         #resolved = lxml.html.tostring(grab.tree)
+        CACHE.append(grab.response.body)
         logger.debug('Memory usage after %s' % memory_usage())
 
+    def shutdown(self):
+        logger.debug('Memory usage in spider shutdown: %s' % memory_usage())
 
 logger.debug('Memory usage before parser start: %s' % memory_usage())
 bot = TestSpider()
 bot.setup_grab(connect_timeout=30, timeout=180)
 bot.run()
 logging.debug('Sleeping 3 seconds')
+time.sleep(1)
+logger.debug('Memory usage after spider: %s' % memory_usage())
+
+tree = lxml.html.fromstring(CACHE[0])
+logger.debug('Memory usage after fromstring: %s' % memory_usage())
+del tree
 time.sleep(3)
-logger.debug('Memory usage before end of script: %s' % memory_usage())
+logger.debug('Memory usage after delete tree: %s' % memory_usage())
+
 logger.debug('done!')
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+import logging
+from lxml.html import fromstring
+import os
+from grab.tools.memory import memory_usage
+import time
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger('[test memory leaks]')
+
+def func():
+    tree = fromstring(open('/web/load/31m.html').read())
+
+logger.debug('Memory usage before build tree: %s' % memory_usage())
+func()
+logger.debug('Memory usage after building the tree: %s' % memory_usage())
+time.sleep(2)
+logger.debug('Memory usage after deleting the tree: %s' % memory_usage())