Commits

Joao Bueno committed b934582

Added support for off-process generators

  • Participants
  • Parent commits 9f94808

Comments (0)

Files changed (3)

File lelo/_lelo.py

 # Author: João S. O. Bueno
 
 from multiprocessing import Process, Queue
+from types import GeneratorType
 
-def _xecuter(queue, func, args, kwargs):
-    queue.put(func(*args, **kwargs))
-
+class Stopped(object): pass
 
+def _xecuter(queue, func, args, kwargs):
+    result = func(*args, **kwargs)
+    if not isinstance(result, GeneratorType):
+        queue.put(result)
+        return
+    for value in result:
+        queue.put(value)
+    queue.put(Stopped())
 
 class CallWrapper(object):
     def __init__(self, func, args, kw):
 def __subclasscheck__(cls, other):
     return True
 
+def __iter__(self):
+    queue = object.__getattribute__(self, "_queue")
+    # While probing for "__iter__", Python usually
+    # triggers the lazy return value retrieval
+    # used in the other cases. So the first value
+    # is already popped out of the queue:
+    if object.__getattribute__(self, "_computed"):
+        yield self._value
+    while True:
+        value = queue.get()
+        if isinstance(value, Stopped):
+            raise StopIteration
+        yield value
 
 class MetaParallel(type):
     def __new__(metacls, name, bases, dct):
         __delattr__ __div__ __divmod__ __doc__ __float__
         __floordiv__ __format__  __getnewargs__
         __hash__ __hex__ __index__ __int__ __invert__
-        __long__ __lshift__ __mod__ __mul__ __neg__ 
+        __long__ __lshift__ __mod__ __mul__ __neg__
         __nonzero__ __oct__ __or__ __pos__ __pow__ __radd__
         __rand__ __rdiv__ __rdivmod__ __reduce__ __reduce_ex__
         __repr__ __rfloordiv__ __rlshift__ __rmod__ __rmul__
         __str__ __sub__ __subclasshook__ __truediv_
         __trunc__ __xor__ _contains__ __delitem__ __delslice__
         __eq__ __ge__ __getitem__ __getslice__ __gt__ __iadd__
-        __imul__ __iter__ __le__ __len__ __lt__ __ne__ __reversed__
+        __imul__ __le__ __len__ __lt__ __ne__ __reversed__
         __setitem__ __setslice__  __enter__ __exit__
 
         """
         for func_name in special_methods.split():
             dct[func_name] = value_retriever(func_name)
         dct["__getattr__"] = attr_getter
+        dct["__iter__"] = __iter__
         return type.__new__(metacls, name, bases, dct)
 
 doc = """
 from setuptools import setup, find_packages
 import sys, os
 
-version = '1.0rc1'
+version = '1.0rc2'
 
 setup(name='lelo',
       version=version,

File tests/test.py

         def sum_(a, b):
             return a + b
 
-        proc  = Process(target=_xecuter, 
+        proc  = Process(target=_xecuter,
             args=(queue, sum_, (2, 3), {}))
         proc.start()
 
         result = wrapped(2,3)
         self.assertEqual(type(result), LazyCallWrapper)
         self.assertEqual(result._value, 5)
-        
+
     def test_lazy_class_factory(self):
         from lelo._lelo import MetaParallel
         X = MetaParallel("X", (object,), {})
         x = X()
+        # check for operator access and work
         object.__setattr__(x, "_value", 10)
         self.assertEqual(x + 0, 10)
+        # check for attribute access
         object.__setattr__(x, "_value", {"y": 10})
         self.assertEqual(list(x.keys()), ["y"])
 
 
         url = "http://google.com"
 
-        
+
         def retr_html(url):
             return urllib.urlopen(url).read()
 
         #print "sync: %f, async: %f" % (t1, t0)
 
         self.assert_(t0 < t1)
-        self.assertEqual(len(res_0) // 200, len(res_1) // 200)
-        
+        self.assertEqual(len(res_0) >> 8, len(res_1) >> 8)
+
+    def test_async_iterator(self):
+        from types import GeneratorType
+        @parallel
+        def fibonacci(limit):
+            n = 1
+            n1 = 0
+            counter = 0
+            while counter < limit:
+                n, n1 = n1, n + n1
+                counter += 1
+                yield n1
+
+        self.assertEqual(list(fibonacci(10)), [1, 1, 2, 3, 5, 8, 13, 21, 34, 55])
+
 
 if __name__ == "__main__":