Maciej Fijalkowski avatar Maciej Fijalkowski committed 2ac43e1

A first crude version of invoking multiple versions of a C compiler while
writing down source files

Comments (0)

Files changed (5)

rpython/tool/runsubprocess2.py

+
+import os, sys, time, select
+from subprocess import Popen, PIPE
+import twisted
+
+_child = None
+_source = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_source = os.path.join(os.path.join(_source, 'translator'), 'invoker.py')
+
+def spawn_subprocess():
+    global _child
+    _child = Popen([sys.executable, '-u', _source], bufsize=0,
+                   stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+spawn_subprocess()
+
+def cleanup_subprocess():
+    global _child
+    _child = None
+import atexit; atexit.register(cleanup_subprocess)
+
+def run(exe, *args):
+    _child.stdin.write(repr((exe,) + args) + "\n")
+
+class SubprocessExploded(Exception):
+    pass
+
+def results():
+    results = []
+    while True:
+        rl, _, _ = select.select([_child.stdout, _child.stderr], [], [], 0)
+        if _child.stderr in rl:
+            raise SubprocessExploded(_child.stderr)
+        elif _child.stdout in rl:
+            results.append(eval(_child.stdout.readline()))
+        else:
+            return results

rpython/translator/c/genc.py

 import contextlib
+from rpython.tool import runsubprocess2
 import py
-import sys, os
+import sys, os, time
 from rpython.rlib import exports
 from rpython.rtyper.typesystem import getfunctionptr
 from rpython.tool import runsubprocess
             str(exename), args))
         profdrv.probe(exename, args)
         return profdrv.after()
-    
+
 class CBuilder(object):
     c_source_filename = None
     _compiled = False
         self.gcpolicy = gcpolicy    # for tests only, e.g. rpython/memory/
         self.eci = self.get_eci()
         self.secondary_entrypoints = secondary_entrypoints
+        self._files_sent = 0
 
     def get_eci(self):
         pypy_include_dir = py.path.local(__file__).join('..')
                               thread_enabled=self.config.translation.thread,
                               sandbox=self.config.translation.sandbox)
         self.db = db
-        
+
         # give the gc a chance to register interest in the start-up functions it
         # need (we call this for its side-effects of db.get())
         list(db.gcpolicy.gc_startup_code())
                     export_symbols=["pypy_main_startup"]))
         self.eci, cfile, extra = gen_source(db, modulename, targetdir,
                                             self.eci, defines=defines,
-                                            split=self.split)
+                                            split=self.split,
+                                            invoke_c_callback=self._invoke_c_compiler)
         self.c_source_filename = py.path.local(cfile)
         self.extrafiles = self.eventually_copy(extra)
         self.gen_makefile(targetdir, exe_name=exe_name)
         return cfile
 
+    def _invoke_c_compiler(self, name, eci, directory):
+        if name is None or name.endswith('.h'):
+            return
+        shared = self.config.translation.shared
+        command = self.translator.platform.gen_cc_command(name, directory, eci,
+                                                          shared=shared)
+        for name, rescode, out, err in runsubprocess2.results():
+            self._files_sent -= 1
+            if rescode == 0:
+                print "Compiled", name
+            else:
+                raise Exception("CC failed with", err)
+        self._files_sent += 1
+        runsubprocess2.run(name, *command)
+
+    def _wait_for_c_compiler(self):
+        while True:
+            for name, rescode, out, err in runsubprocess2.results():
+                if rescode == 0:
+                    print "Compiled", name
+                else:
+                    raise Exception("CC failed with", err)
+                self._files_sent -= 1
+                if self._files_sent == 0:
+                    return
+            time.sleep(.1)
+
     def eventually_copy(self, cfiles):
         extrafiles = []
         for fn in cfiles:
             outputfilename=exe_name)
 
     def compile(self, exe_name=None):
+        self._wait_for_c_compiler()
         assert self.c_source_filename
         assert not self._compiled
 
 class SourceGenerator:
     one_source_file = True
 
-    def __init__(self, database):
+    def __init__(self, database, eci, invoke_c_callback=None):
         self.database = database
+        self.eci = eci
         self.extrafiles = []
         self.path = None
         self.namespace = NameManager()
+        self._last_file_name = None
+        self.invoke_c_callback = invoke_c_callback
 
     def set_strategy(self, path, split=True):
         all_nodes = list(self.database.globalcontainers())
         return self.namespace.uniquename(name[:-2]) + '.c'
 
     def makefile(self, name):
+        if self.invoke_c_callback is not None:
+            self.invoke_c_callback(self._last_file_name, self.path,
+                                   self.eci)
+        self._last_file_name = name
         log.writing(name)
         filepath = self.path.join(name)
         if name.endswith('.c'):
             if self.database.gcpolicy.need_no_typeptr():
                 pass    # XXX gcc uses toooooons of memory???
             else:
-                split_criteria_big = SPLIT_CRITERIA * 4 
+                split_criteria_big = SPLIT_CRITERIA * 4
 
         #
         # All declarations
 
 
 def gen_source(database, modulename, targetdir,
-               eci, defines={}, split=False):
+               eci, defines={}, split=False, invoke_c_callback=False):
     if isinstance(targetdir, str):
         targetdir = py.path.local(targetdir)
 
     # 1) All declarations
     # 2) Implementation of functions and global structures and arrays
     #
-    sg = SourceGenerator(database)
+    sg = SourceGenerator(database, eci, invoke_c_callback)
     sg.set_strategy(targetdir, split)
     database.prepare_inline_helpers()
     sg.gen_readable_parts_of_source(f)

rpython/translator/invoker.py

+
+import os
+from twisted.internet import reactor
+from twisted.protocols import basic
+from twisted.internet import protocol, stdio
+from twisted.internet.error import ProcessDone
+
+class SubProcessProtocol(protocol.ProcessProtocol):
+    def __init__(self, commander, name):
+        self._commander = commander
+        self._output = []
+        self._name = name
+        self._error = []
+
+    def outReceived(self, out):
+        self._output.append(out)
+
+    def errReceived(self, err):
+        self._error.append(err)
+
+    def processExited(self, status):
+        self._commander.process_exited()
+        if status.value == ProcessDone:
+            print repr((self._name, 0, "".join(self._output),
+                        "".join(self._error)))
+        else:
+            print repr((self._name, status.value.exitCode,
+                        "".join(self._output), "".join(self._error)))
+
+MAX = 8
+
+class Commander(basic.LineReceiver):
+    delimiter = '\n'
+
+    def __init__(self, max=MAX):
+        self._counter = 0
+        self._max = max
+        self._queue = []
+
+    def connectionMade(self):
+        pass
+
+    def process_exited(self):
+        self._counter -= 1
+        if self._queue:
+            self._run(self._queue.pop())
+
+    def _run(self, line):
+        args = eval(line)
+        reactor.spawnProcess(SubProcessProtocol(self, args[0]), args[1],
+                             args[1:], env=os.environ.copy())
+        self._counter += 1
+
+    def lineReceived(self, line):
+        if not line:
+            reactor.stop()
+            return
+        assert line.startswith('(')
+        if self._counter < self._max:
+            self._run(line)
+        else:
+            self._queue.append(line)
+
+stdio.StandardIO(Commander())
+reactor.run()

rpython/translator/platform/__init__.py

     def _library_dirs_for_libffi(self):
         raise NotImplementedError("Needs to be overwritten")
 
+    def gen_cc_command(self, fname, eci, shared=False):
+        raise NotImplementedError("abstract base class")
+
     def check___thread(self):
         return True
 

rpython/translator/platform/posix.py

         # strip compiler flags
         return [entry[2:] for entry in out.split()]
 
+    def gen_cc_command(self, fname, eci, path, shared=False):
+        fullname = str(path.join(fname))
+        objfullname = fullname[:-2] + '.o'
+        return ([self.cc] + self._compile_args_from_eci(eci, not shared) +
+                ['-c', '-o', objfullname, fullname])
+
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False):
         cfiles = self._all_cfiles(cfiles, eci)
         self.defs = {}
         self.lines = []
         self.makefile_dir = py.path.local(path)
-        
+
     def pathrel(self, fpath):
         if fpath.dirpath() == self.makefile_dir:
             return fpath.basename
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.