Commits

mattip committed 506a567 Merge

merge precompiled-headers which uses Makefile with precompiled headers on MSVC

Comments (0)

Files changed (11)

pypy/module/cpyext/api.py

     # implement function callbacks and generate function decls
     functions = []
     pypy_decls = []
+    pypy_decls.append("#ifndef _PYPY_PYPY_DECL_H\n")
+    pypy_decls.append("#define _PYPY_PYPY_DECL_H\n")
     pypy_decls.append("#ifndef PYPY_STANDALONE\n")
     pypy_decls.append("#ifdef __cplusplus")
     pypy_decls.append("extern \"C\" {")
     pypy_decls.append("}")
     pypy_decls.append("#endif")
     pypy_decls.append("#endif /*PYPY_STANDALONE*/\n")
+    pypy_decls.append("#endif /*_PYPY_PYPY_DECL_H*/\n")
 
     pypy_decl_h = udir.join('pypy_decl.h')
     pypy_decl_h.write('\n'.join(pypy_decls))

rpython/translator/c/dlltool.py

             entrypoints.append(getfunctionptr(graph))
         return entrypoints
 
-    def gen_makefile(self, targetdir, exe_name=None):
+    def gen_makefile(self, targetdir, exe_name=None,
+                    headers_to_precompile=[]):
         pass # XXX finish
 
     def compile(self):
         extsymeci = ExternalCompilationInfo(export_symbols=export_symbols)
         self.eci = self.eci.merge(extsymeci)
         files = [self.c_source_filename] + self.extrafiles
+        files += self.eventually_copy(self.eci.separate_module_files)
+        self.eci.separate_module_files = ()
         oname = self.name
         self.so_name = self.translator.platform.compile(files, self.eci,
                                                         standalone=False,

rpython/translator/c/genc.py

                 defines['PYPY_MAIN_FUNCTION'] = "pypy_main_startup"
                 self.eci = self.eci.merge(ExternalCompilationInfo(
                     export_symbols=["pypy_main_startup", "pypy_debug_file"]))
-        self.eci, cfile, extra = gen_source(db, modulename, targetdir,
-                                            self.eci, defines=defines,
-                                            split=self.split)
+        self.eci, cfile, extra, headers_to_precompile = \
+                gen_source(db, modulename, targetdir,
+                           self.eci, defines=defines, split=self.split)
         self.c_source_filename = py.path.local(cfile)
         self.extrafiles = self.eventually_copy(extra)
-        self.gen_makefile(targetdir, exe_name=exe_name)
+        self.gen_makefile(targetdir, exe_name=exe_name,
+                          headers_to_precompile=headers_to_precompile)
         return cfile
 
     def eventually_copy(self, cfiles):
         self._compiled = True
         return self.executable_name
 
-    def gen_makefile(self, targetdir, exe_name=None):
-        cfiles = [self.c_source_filename] + self.extrafiles
+    def gen_makefile(self, targetdir, exe_name=None, headers_to_precompile=[]):
+        module_files = self.eventually_copy(self.eci.separate_module_files)
+        self.eci.separate_module_files = []
+        cfiles = [self.c_source_filename] + self.extrafiles + list(module_files)
         if exe_name is not None:
             exe_name = targetdir.join(exe_name)
         mk = self.translator.platform.gen_makefile(
             cfiles, self.eci,
             path=targetdir, exe_name=exe_name,
+            headers_to_precompile=headers_to_precompile,
+            no_precompile_cfiles = module_files,
             shared=self.config.translation.shared)
 
         if self.has_profopt():
             profopt = self.config.translation.profopt
-            mk.definition('ABS_TARGET', '$(shell python -c "import sys,os; print os.path.abspath(sys.argv[1])" $(TARGET))')
+            mk.definition('ABS_TARGET', str(targetdir.join('$(TARGET)')))
             mk.definition('DEFAULT_TARGET', 'profopt')
             mk.definition('PROFOPT', profopt)
 
     def __init__(self, database):
         self.database = database
         self.extrafiles = []
+        self.headers_to_precompile = []
         self.path = None
         self.namespace = NameManager()
 
         filepath = self.path.join(name)
         if name.endswith('.c'):
             self.extrafiles.append(filepath)
+        if name.endswith('.h'):
+            self.headers_to_precompile.append(filepath)
         return filepath.open('w')
 
     def getextrafiles(self):
                     print >> fc, '/***********************************************************/'
                     print >> fc, '/***  Implementations                                    ***/'
                     print >> fc
-                    print >> fc, '#define PYPY_FILE_NAME "%s"' % name
                     print >> fc, '#include "common_header.h"'
                     print >> fc, '#include "structdef.h"'
                     print >> fc, '#include "forwarddecl.h"'
                     print >> fc, '#include "preimpl.h"'
+                    print >> fc, '#define PYPY_FILE_NAME "%s"' % name
                     print >> fc, '#include "src/g_include.h"'
                     print >> fc
                 print >> fc, MARKER
     print >> f, "#endif"
 
 def gen_preimpl(f, database):
+    f.write('#ifndef _PY_PREIMPLE_H\n#define _PY_PREIMPL_H\n')
     if database.translator is None or database.translator.rtyper is None:
         return
     preimplementationlines = pre_include_code_lines(
         database, database.translator.rtyper)
     for line in preimplementationlines:
         print >> f, line
+    f.write('#endif /* _PY_PREIMPL_H */\n')    
 
 def gen_startupcode(f, database):
     # generate the start-up code and put it into a function
     f = filename.open('w')
     incfilename = targetdir.join('common_header.h')
     fi = incfilename.open('w')
+    fi.write('#ifndef _PY_COMMON_HEADER_H\n#define _PY_COMMON_HEADER_H\n')
 
     #
     # Header
 
     eci.write_c_header(fi)
     print >> fi, '#include "src/g_prerequisite.h"'
+    fi.write('#endif /* _PY_COMMON_HEADER_H*/\n')
 
     fi.close()
 
     sg.set_strategy(targetdir, split)
     database.prepare_inline_helpers()
     sg.gen_readable_parts_of_source(f)
+    headers_to_precompile = sg.headers_to_precompile[:]
+    headers_to_precompile.insert(0, incfilename)
 
     gen_startupcode(f, database)
     f.close()
 
     eci = add_extra_files(eci)
     eci = eci.convert_sources_to_files()
-    files, eci = eci.get_module_files()
-    return eci, filename, sg.getextrafiles() + list(files)
+    return eci, filename, sg.getextrafiles(), headers_to_precompile

rpython/translator/c/src/profiling.c

File contents unchanged.

rpython/translator/platform/__init__.py

         return ExecutionResult(returncode, stdout, stderr)
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
-                     shared=False):
+                     shared=False, headers_to_precompile=[],
+                     no_precompile_cfiles = []):
         raise NotImplementedError("Pure abstract baseclass")
 
     def __repr__(self):

rpython/translator/platform/darwin.py

         return ["-Wl,-exported_symbols_list,%s" % (response_file,)]
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
-                     shared=False):
+                     shared=False, headers_to_precompile=[],
+                     no_precompile_cfiles = []):
         # ensure frameworks are passed in the Makefile
         fs = self._frameworks(eci.frameworks)
         if len(fs) > 0:
             # concat (-framework, FrameworkName) pairs
             self.extra_libs += tuple(map(" ".join, zip(fs[::2], fs[1::2])))
         mk = super(Darwin, self).gen_makefile(cfiles, eci, exe_name, path,
-                                              shared)
+                                shared=shared,
+                                headers_to_precompile=headers_to_precompile,
+                                no_precompile_cfiles = no_precompile_cfiles)
         return mk
 
 

rpython/translator/platform/posix.py

         return [entry[2:] for entry in out.split()]
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
-                     shared=False):
+                     shared=False, headers_to_precompile=[],
+                     no_precompile_cfiles = []):
         cfiles = self._all_cfiles(cfiles, eci)
 
         if path is None:

rpython/translator/platform/test/test_distutils.py

 
     def test_900_files(self):
         py.test.skip('Makefiles not suppoerted')
+
+    def test_precompiled_headers(self):
+        py.test.skip('Makefiles not suppoerted')
+

rpython/translator/platform/test/test_makefile.py

 
 from rpython.translator.platform.posix import GnuMakefile as Makefile
+from rpython.translator.platform import host
+from rpython.tool.udir import udir
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from StringIO import StringIO
-import re
+import re, sys, py
 
 def test_simple_makefile():
     m = Makefile()
     val = s.getvalue()
     assert not re.search('CC += +xxx', val, re.M)
     assert re.search('CC += +yyy', val, re.M)    
+
+class TestMakefile(object):
+    platform = host
+    strict_on_stderr = True
+
+    def check_res(self, res, expected='42\n'):
+        assert res.out == expected
+        if self.strict_on_stderr:
+            assert res.err == ''
+        assert res.returncode == 0        
+    
+    def test_900_files(self):
+        txt = '#include <stdio.h>\n'
+        for i in range(900):
+            txt += 'int func%03d();\n' % i
+        txt += 'int main() {\n    int j=0;'    
+        for i in range(900):
+            txt += '    j += func%03d();\n' % i
+        txt += '    printf("%d\\n", j);\n'
+        txt += '    return 0;};\n'
+        cfile = udir.join('test_900_files.c')
+        cfile.write(txt)
+        cfiles = [cfile]
+        for i in range(900):
+            cfile2 = udir.join('implement%03d.c' %i)
+            cfile2.write('''
+                int func%03d()
+            {
+                return %d;
+            }
+            ''' % (i, i))
+            cfiles.append(cfile2)
+        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
+        mk.write()
+        self.platform.execute_makefile(mk)
+        res = self.platform.execute(udir.join('test_900_files'))
+        self.check_res(res, '%d\n' %sum(range(900)))
+
+    def test_precompiled_headers(self):
+        if self.platform.cc != 'cl.exe':
+            py.test.skip("Only MSVC profits from precompiled headers")
+        import time
+        tmpdir = udir.join('precompiled_headers').ensure(dir=1)
+        # Create an eci that should not use precompiled headers
+        eci = ExternalCompilationInfo(include_dirs=[tmpdir])
+        main_c = tmpdir.join('main_no_pch.c')
+        eci.separate_module_files = [main_c]
+        ncfiles = 10
+        nprecompiled_headers = 20
+        txt = ''
+        for i in range(ncfiles):
+            txt += "int func%03d();\n" % i
+        txt += "\nint main(int argc, char * argv[])\n"
+        txt += "{\n   int i=0;\n"
+        for i in range(ncfiles):
+            txt += "   i += func%03d();\n" % i
+        txt += '    printf("%d\\n", i);\n'
+        txt += "   return 0;\n};\n"
+        main_c.write(txt)
+        # Create some large headers with dummy functions to be precompiled
+        cfiles_precompiled_headers = []
+        for i in range(nprecompiled_headers):
+            pch_name =tmpdir.join('pcheader%03d.h' % i)
+            txt = '#ifndef PCHEADER%03d_H\n#define PCHEADER%03d_H\n' %(i, i)
+            for j in range(3000):
+                txt += "int pcfunc%03d_%03d();\n" %(i, j)
+            txt += '#endif'
+            pch_name.write(txt)    
+            cfiles_precompiled_headers.append(pch_name)        
+        # Create some cfiles with headers we want precompiled
+        cfiles = []
+        for i in range(ncfiles):
+            c_name =tmpdir.join('implement%03d.c' % i)
+            txt = ''
+            for pch_name in cfiles_precompiled_headers:
+                txt += '#include "%s"\n' % pch_name
+            txt += "int func%03d(){ return %d;};\n" % (i, i)
+            c_name.write(txt)
+            cfiles.append(c_name)        
+        if sys.platform == 'win32':
+            clean = ('clean', '', 'for %f in ( $(OBJECTS) $(TARGET) ) do @if exist %f del /f %f')
+            get_time = time.clock
+        else:    
+            clean = ('clean', '', 'rm -f $(OBJECTS) $(TARGET) ')
+            get_time = time.time
+        #write a non-precompiled header makefile
+        mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir)
+        mk.rule(*clean)
+        mk.write()
+        t0 = get_time()
+        self.platform.execute_makefile(mk)
+        t1 = get_time()
+        t_normal = t1 - t0
+        self.platform.execute_makefile(mk, extra_opts=['clean'])
+        # Write a super-duper makefile with precompiled headers
+        mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir,
+                           headers_to_precompile=cfiles_precompiled_headers,)
+        mk.rule(*clean)
+        mk.write()
+        t0 = get_time()
+        self.platform.execute_makefile(mk)
+        t1 = get_time()
+        t_precompiled = t1 - t0
+        res = self.platform.execute(mk.exe_name)
+        self.check_res(res, '%d\n' %sum(range(ncfiles)))
+        print "precompiled haeder 'make' time %.2f, non-precompiled header time %.2f" %(t_precompiled, t_normal)
+        assert t_precompiled < t_normal * 0.5
+
+   

rpython/translator/platform/test/test_platform.py

         res = self.platform.execute(executable)
         self.check_res(res)
 
-    def test_900_files(self):
-        txt = '#include <stdio.h>\n'
-        for i in range(900):
-            txt += 'int func%03d();\n' % i
-        txt += 'int main() {\n    int j=0;'    
-        for i in range(900):
-            txt += '    j += func%03d();\n' % i
-        txt += '    printf("%d\\n", j);\n'
-        txt += '    return 0;};\n'
-        cfile = udir.join('test_900_files.c')
-        cfile.write(txt)
-        cfiles = [cfile]
-        for i in range(900):
-            cfile2 = udir.join('implement%03d.c' %i)
-            cfile2.write('''
-                int func%03d()
-            {
-                return %d;
-            }
-            ''' % (i, i))
-            cfiles.append(cfile2)
-        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
-        mk.write()
-        self.platform.execute_makefile(mk)
-        res = self.platform.execute(udir.join('test_900_files'))
-        self.check_res(res, '%d\n' %sum(range(900)))
-
-
     def test_nice_errors(self):
         cfile = udir.join('test_nice_errors.c')
         cfile.write('')

rpython/translator/platform/windows.py

 
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
-                     shared=False):
+                     shared=False, headers_to_precompile=[],
+                     no_precompile_cfiles = []):
         cfiles = self._all_cfiles(cfiles, eci)
 
         if path is None:
         if self.x64:
             definitions.append(('_WIN64', '1'))
 
+        rules = [
+            ('all', '$(DEFAULT_TARGET)', []),
+            ('.asm.obj', '', '$(MASM) /nologo /Fo$@ /c $< $(INCLUDEDIRS)'),
+            ]
+
+        if len(headers_to_precompile)>0:
+            stdafx_h = path.join('stdafx.h')
+            txt  = '#ifndef PYPY_STDAFX_H\n'
+            txt += '#define PYPY_STDAFX_H\n'
+            txt += '\n'.join(['#include "' + m.pathrel(c) + '"' for c in headers_to_precompile])
+            txt += '\n#endif\n'
+            stdafx_h.write(txt)
+            stdafx_c = path.join('stdafx.c')
+            stdafx_c.write('#include "stdafx.h"\n')
+            definitions.append(('CREATE_PCH', '/Ycstdafx.h /Fpstdafx.pch /FIstdafx.h'))
+            definitions.append(('USE_PCH', '/Yustdafx.h /Fpstdafx.pch /FIstdafx.h'))
+            rules.append(('$(OBJECTS)', 'stdafx.pch', []))
+            rules.append(('stdafx.pch', 'stdafx.h', 
+               '$(CC) stdafx.c /c /nologo $(CFLAGS) $(CFLAGSEXTRA) '
+               '$(CREATE_PCH) $(INCLUDEDIRS)'))
+            rules.append(('.c.obj', '', 
+                    '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) $(USE_PCH) '
+                    '/Fo$@ /c $< $(INCLUDEDIRS)'))
+            #Do not use precompiled headers for some files
+            #rules.append((r'{..\module_cache}.c{..\module_cache}.obj', '',
+            #        '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $< $(INCLUDEDIRS)'))
+            # nmake cannot handle wildcard target specifications, so we must
+            # create a rule for compiling each file from eci since they cannot use
+            # precompiled headers :(
+            no_precompile = []
+            for f in list(no_precompile_cfiles):
+                f = m.pathrel(py.path.local(f))
+                if f not in no_precompile and f.endswith('.c'):
+                    no_precompile.append(f)
+                    target = f[:-1] + 'obj'
+                    rules.append((target, f,
+                        '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
+                        '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f)))
+
+        else:
+            rules.append(('.c.obj', '', 
+                          '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
+                          '/Fo$@ /c $< $(INCLUDEDIRS)'))
+
+
         for args in definitions:
             m.definition(*args)
 
-        rules = [
-            ('all', '$(DEFAULT_TARGET)', []),
-            ('.c.obj', '', '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $< $(INCLUDEDIRS)'),
-            ('.asm.obj', '', '$(MASM) /nologo /Fo$@ /c $< $(INCLUDEDIRS)'),
-            ]
-
         for rule in rules:
             m.rule(*rule)
         
 
         self._handle_error(returncode, stdout, stderr, path.join('make'))
 
+class WinDefinition(posix.Definition):
+    def write(self, f):
+        def write_list(prefix, lst):
+            lst = lst or ['']
+            for i, fn in enumerate(lst):
+                print >> f, prefix, fn,
+                if i < len(lst)-1:
+                    print >> f, '\\'
+                else:
+                    print >> f
+                prefix = ' ' * len(prefix)
+        name, value = self.name, self.value
+        if isinstance(value, str):
+            f.write('%s = %s\n' % (name, value))
+        else:
+            write_list('%s =' % (name,), value)
+        f.write('\n')
+
+
 class NMakefile(posix.GnuMakefile):
     def write(self, out=None):
         # nmake expands macros when it parses rules.
         if out is None:
             f.close()
 
+    def definition(self, name, value):
+        defs = self.defs
+        defn = WinDefinition(name, value)
+        if name in defs:
+            self.lines[defs[name]] = defn
+        else:
+            defs[name] = len(self.lines)
+            self.lines.append(defn)
 
 class MingwPlatform(posix.BasePosix):
     name = 'mingw32'