Commits

Ronny Pfannschmidt  committed 7daf5f5

finish initial requirement dependency processing

  • Participants
  • Parent commits 6e16fa1

Comments (0)

Files changed (7)

File pu/task_queue.py

-"""
+from pu.tasks.util import task_failed
+from pu.tasks.util import task_succeeded
 
-    :license: MIT/PSF
-    :copyright: 2010 by Ronny Pfannschmidt <Ronny.Pfannschmidt@gmx.de>
-"""
 
 
 class Queue(object):
     def __init__(self):
+        self._key_to_identity = {}
         self.depends = {}
         self.completed = set()
         self.runnable = set()
     def find_and_add_new_runnable(self):
         found = set()
 
-        for k, v in self.depends.items():
+        for k, v in  self.depends.items():
             if k in self.completed:
                 continue
             if k in self.running:
             if hasattr(k, '__iter__'):
                 new_item = next(k, None)
                 if new_item is not None:
-                    self.add(new_item, parent=k)
+                    new_item = self.add(new_item, parent=k)
                     found.add(new_item)
             if not v - self.completed:
                 self.runnable.add(k)
         return bool(found)
 
-    def report_failure(self, t):
+    def report_failure(self, task):
         #XXX: evil
-        self.completed.add(t)
-        self.running.remove(t)
+        task_failed.send(task)
+        self.completed.add(task)
+        self.running.remove(task)
 
-    def report_sucess(self, t):
-        self.completed.add(t)
-        self.running.remove(t)
+    def report_sucess(self, task):
+        task_succeeded.send(task)
+        self.completed.add(task)
+        self.running.remove(task)
 
     def next(self):
         if not self.runnable:
         """add a `task`
         if `parent` is given this task is a new dependency for parent
         """
+        if parent:
+            assert parent in self.depends
 
         if task not in self.depends:
             self.depends[task] = set()
+            self._key_to_identity[task] = task
+        else:
+            task = self._key_to_identity[task]
         if parent:
             self.depends[parent].add(task)
+            task_succeeded.connect(parent._requirement_succeeded, sender=task)
+            task_failed.connect(parent._requirement_failed, sender=task)
+        return task
 
     def run_all_possible(self):
         """runs all tasks it can complete
         all completable tasks are completed
         """
         for item in self:
+            print 'trying to run', item
             try:
                 item()
                 self.report_sucess(item)
         """
         self.run_all_possible()
         if len(self.completed) < len(self.depends):
+            print 'completed: \n ', '\n  '.join(str(x)
+                                                for x in sorted(self.completed))
+            print 'depends', '\n  '.join(str(x)
+                                         for x in sorted(self.depends))
             raise RuntimeError('not all tasks are executable')

File pu/tasks/build.py

 from pu.tasks.util import TaskBase
+from pu.tasks.metadata import FindPackages
 
+import logging
+log = logging.getLogger('kij.build')
 
-class CopyModulesToBuild(TaskBase):
-    keys = 'source', 'build_lib'
+class CopySinglePackageToBuild(TaskBase):
+    keys = 'source', 'build_lib', 'package'
 
     def __call__(self):
-        self.build_lib.ensure(dir=1)
-        for module in self.source.visit('*.py'):
-            #XXX: grab from packages finder
-            target = self.build_lib.join(module.relto(self.source))
-            target.dirpath().ensure(dir=1)
-            target.write(module.read())
+        elem = self.package.split('.')
+        source_pkg = self.source.join(*elem)
+        build_pkg = self.build_lib.join(*elem).ensure(dir=1)
+        for src in source_pkg.listdir('*.py'):
+            src.copy(build_pkg.join(src.basename))
+
+
+class CopyPackagesToBuild(TaskBase):
+    keys = 'source', 'build_lib'
+    requirements = FindPackages,
+
+    def on_findpackages_success(self, task):
+        print 'found packages', task.result
+        for pkg in task.result:
+            task = CopySinglePackageToBuild(
+                    source=self.source,
+                    build_lib=self.build_lib,
+                    package=pkg,
+                    )
+            self.queue.append(task)
+
+    def __call__(self):
+        pass
 
 
 class CompileByteCode(TaskBase):
         ignores `sys.dont_write_bytecode`
     """
     keys = 'build_lib',
+    requirements = CopyPackagesToBuild,
 
     def __call__(self):
         from py_compile import compile

File pu/tasks/metadata.py

-from .util import TaskBase
+from .util import TaskBase, task_succeeded
 
 
 def find_packages(base, start):
                 yield result
 
 
-class FindPackages(TaskBase):
+class FindSubPackages(TaskBase):
     keys = 'source', 'match',
     result = None
 
 
         self.result = list(find_packages(self.source,
                                          self.source.join(*elements)))
+        print 'found', self.result
         if not self.result:
             raise IOError('package %s not found below %s' % (
                            self.match, self.source))
 class ReadYamlMetadata(TaskBase):
     keys = 'source',
     result = None
+    requirements = ()
+    def __call__(self):
+        from pu.files.dist import DistFile
+        self.result = DistFile(self.source.join('kij.yml'))
+
+
+class FindPackages(TaskBase):
+    keys = 'source',
+    requirements = ReadYamlMetadata,
+
+    def on_readyamlmetadata_success(self, item):
+        self.result = []
+        for package in item.result.packages:
+            task = FindSubPackages(
+                match=package,
+                source=self.source)
+            self.queue.append(task)
+
+    def on_findsubpackages_success(self, item):
+        print item
+        self.result.extend(item.result)
 
     def __call__(self):
-        from pu.files.dist import DistFile
-        self.result = DistFile(self.source.join('kij.yaml'))
+        assert self.result

File pu/tasks/util.py

         return hash(self._key())
 
     def __eq__(self, other):
-        return (isinstance(other, TaskBase)
+        return (type(self) == type(other)
                 and self._key() == other._key())
 
     def __repr__(self):
 
     # reporting dispatch
     def _dispatch(self, base, item):
-        task_succeeded.disconnect(
-                self._requirement_succeeded,
-                sender=item)
-        task_succeeded.disconnect(
-                self._requirement_failed,
-                sender=item)
-
         name = getattr(item, 'category', item.__class__.__name__.lower())
         method = 'on_%s_%s' % (name, base)
         default_method = 'on_' + name
 
     def _enqueue_next_requirement(self):
         if self.requirements:
-            task = self.requirements.popleft()(**self._kw)
-            task_succeeded.connect(
-                    self._requirement_succeeded,
-                    sender=task)
-            task_failed.connect(
-                    self._requirement_failed,
-                    sender=task)
+            print 'equeue next for', self
+            req = self.requirements.popleft()
+            task = req(**self._kw)
+            print 'made', task
             self.queue.append(task)
 
+    def __iter__(self):
+        return self
+
     def next(self):
         if self.queue is None:
             self.queue = deque()
             #     the returned requirement might already be in the queue
             #     but with different identity
             if self.requirements:
+                print self, 'needs', self.requirements
                 self.requirements = deque(self.requirements)
                 self._enqueue_next_requirement()
         if self.queue:

File tests/tasks/test_build.py

-from pu.tasks.build import CopyModulesToBuild, CompileByteCode
+from pu.tasks.build import CopyPackagesToBuild, CompileByteCode
 from pu.task_queue import Queue
 
 
 def test_copy_build(source, tmpdir):
     source.ensure('testpkg/__init__.py')
-    source.join('kij.yml').write('name: test\npackages: testpkg\n')
-    task = CopyModulesToBuild(
+    source.join('kij.yml').write('name: test\npackages: [testpkg]\n')
+    task = CopyPackagesToBuild(
                     source=source,
                     build_lib=tmpdir.join('build/lib'),
                     )
-    task()
+    
+    queue = Queue()
+    queue.add(task)
+    queue.run_all()
     assert tmpdir.join('build/lib/testpkg/__init__.py').check()
 
 
     queue = Queue()
 
     source.ensure('testpkg/__init__.py')
+    source.join('kij.yml').write('name: test\npackages: [testpkg]\n')
     build_lib = tmpdir.join('build/lib')
 
-    copy = CopyModulesToBuild(
-        source=source,
-        build_lib=build_lib,
-        )
-    queue.add(copy)
-
-    compile = CompileByteCode(build_lib=build_lib)
+    compile = CompileByteCode(
+            source=source,
+            build_lib=build_lib,
+            )
     queue.add(compile)
-    queue.add(copy, parent=compile)
 
     queue.run_all()
     target = build_lib.join('testpkg/__init__.pyc')

File tests/tasks/test_metadata.py

 import py
-from pu.tasks.metadata import FindPackages, find_packages as gen_find_packages
-from pu.tasks.metadata import ReadYamlMetadata
+from pu.tasks.metadata import FindSubPackages, find_packages as gen_find_packages
+from pu.tasks.metadata import ReadYamlMetadata, FindPackages
+from pu.task_queue import Queue
 
 def find_packages(source, pkg):
     return list(gen_find_packages(source, pkg))
     pkg = source.join('pkg')
     source.ensure('pkg/__init__.py')
 
-    task = FindPackages(source=source, match='pkg')
+    task = FindSubPackages(source=source, match='pkg')
     task()
     assert task.result == ['pkg']
 
 
 
 def test_read_yaml_task(source):
-    source.join('kij.yaml').write('name: test\npackages: [foo, bar]')
+    source.join('kij.yml').write('name: test\npackages: [foo, bar]')
     task = ReadYamlMetadata(source=source)
     task()
     assert task.result.name == 'test'
     assert task.result.packages == ['foo', 'bar']
+
+
+def test_find_packages_task(source):
+    source.join('kij.yml').write('name: test\npackages: [foo, bar]')
+    source.join('foo/__init__.py').ensure()
+    source.join('bar/__init__.py').ensure()
+
+    queue = Queue()
+    task = FindPackages(source=source)
+    queue.add(task)
+    queue.run_all_possible()
+    for k, v in queue.depends.items():
+        print k, 'depends on', v
+    assert task.result == ['foo', 'bar']
+

File tests/test_task_queue.py

 import py
 from pu.task_queue import Queue
 from pu.tasks.install import LinkPTH
+from pu.tasks.util import TaskBase
 
-
-class SimpleTask(object):
-
-    def __init__(self, number):
-        self.number = number
-        self.ndeps = number
-        self.called = False
-
-    def __hash__(self):
-        return hash(self.number)
+class SimpleTask(TaskBase):
+    ndeps = None
+    keys = 'number',
+    called = False
 
     def __repr__(self):
         return '<ST %d>' % self.number
 
-    def __iter__(self):
-        return self
-
     def next(self):
+        if self.ndeps is None:
+            self.ndeps = self.number
         if self.ndeps:
             self.ndeps -= 1
-            return SimpleTask(self.ndeps)
+            return SimpleTask(number=self.ndeps)
         raise StopIteration
 
     def __eq__(self, other):
         print self.number
 
 
+class WeirdTask(TaskBase):
+    keys = 'number',
+    called = False
+    requirements = SimpleTask,
+    def __call__(self):
+        self.called = True
+
 def test_create():
     Queue()
 
 def test_simple():
     queue = Queue()
     # will create SimpleTask(0) as dependency
-    queue.add(SimpleTask(1))
+    queue.add(SimpleTask(number=1))
     first_task = next(queue)
     assert first_task.number == 0
 
 
 def test_adding_the_same_twice_has_no_effect():
     queue = Queue()
-    queue.add(SimpleTask(0))
-    queue.add(SimpleTask(0))
+    queue.add(SimpleTask(number=0))
+    queue.add(SimpleTask(number=0))
     assert len(queue) == 1
 
 
 
 def test_dependencies():
     queue = Queue()
-    queue.add(SimpleTask(3))
+    queue.add(SimpleTask(number=3))
     queue.run_all()
     assert len(queue.completed) == 4
 
 
 def test_dependency_cycle_wont_complete_all():
     queue = Queue()
-    base = SimpleTask(1)
+    base = SimpleTask(number=1)
 
     queue.add(base)
     a_runnable = next(queue)
     assert set(queue.depends) > queue.completed
 
     py.test.raises(RuntimeError, queue.run_all)
+
+
+def test_dependencies():
+    queue = Queue()
+    task = queue.add(WeirdTask(number=1))
+    queue.run_all()
+    print queue.completed
+    assert task.called