Commits

ariovistus committed 9f1c4de

add pydexe command to celerid

Now celerid can build d apps which embed python with pyd.
Looks like --optimize was a bit flimsy, so shored that up.
Removed some obsolete build flags (such as with-meta)

Comments (0)

Files changed (4)

     'exception.d',
     'extra.d',
     'func_wrap.d',
-    'iteration.d',
     'make_object.d',
     'make_wrapper.d',
     'op_wrap.d',
     def __init__(self, *args, **kwargs):
         cc.CCompiler.__init__(self, *args, **kwargs)
         self.winonly = False
+        self.proj_name = None
+        self.build_exe = False
+        self.with_pyd = True
+        self.with_main = True
+        self.build_deimos = False
+        self.optimize = False
+        self.versionFlagsFromExt = []
+        self.debugFlagsFromExt = []
         # Get DMD/GDC specific info
         self._initialize()
         # _binpath
 
         return None
 
+    def versionOpts(self):
+        # Python version option allows extension writer to take advantage of
+        # Python/C API features available only in recent version of Python with
+        # a version statement like:
+        #   version(Python_2_4_Or_Later) {
+        #     Py_ConvenientCallOnlyAvailableInPython24AndLater();
+        #   } else {
+        #     // Do it the hard way...
+        #   }
+        def pvo(opt):
+            optf = 'Python_%d_%d_Or_Later'
+            def pv2(minor):
+                ret = []
+                if not self.build_exe:
+                    ret.append(opt % 'PydPythonExtension')
+                ret.extend([opt % (optf % (2,m)) for m in range(4,minor+1)])
+                return ret
+            def pv3(minor):
+                return [opt % (optf % (3,m)) for m in range(0,minor+1)]
+            major = sys.version_info[0]
+            minor = sys.version_info[1]
+            if major == 2: return pv2(minor)
+            if major == 3: return  pv2(7) + pv3(minor)
+            assert False, "what python version is this, anyways?"
+        return pvo(self._versionOpt) + [self._unicodeOpt]
+
     def compile(self, sources,
         output_dir=None, macros=None, include_dirs=None, debug=0,
         extra_preargs=None, extra_postargs=None, depends=None
         include_dirs = include_dirs or []
         extra_preargs = extra_preargs or []
         extra_postargs = extra_postargs or []
+            
+        pythonVersionOpts = self.versionOpts()
 
         if not os.path.exists(output_dir):
             os.makedirs(output_dir)
             else:
                 sources.append((winpath(source, self.winonly), 'outside'))
 
-        # flags = (with_pyd, with_st, with_meta, with_main)
-        with_pyd, with_st, with_meta, with_main, build_deimos = [f for f, category in macros if category == 'aux'][0]
-        # And Pyd!
-        if with_pyd:
-            # If we're not using StackThreads, don't use iteration.d in Pyd
-            if not with_st or not self._st_support:
-                _pydFiles.remove('iteration.d');
+        if self.with_pyd:
             for file in _pydFiles:
                 filePath = os.path.join(_infraDir, 'pyd', file)
                 if not os.path.isfile(filePath):
                         " missing." % filePath
                     )
                 sources.append((winpath(filePath,self.winonly), 'infra'))
-        if build_deimos:
+            for file in _metaFiles:
+                filePath = os.path.join(_infraDir, 'meta', file)
+                if not os.path.isfile(filePath):
+                    raise DistutilsPlatformError("Required meta source file"
+                        " '%s' is missing." % filePath
+                    )
+                sources.append((winpath(filePath,self.winonly), 'infra'))
+        if self.build_deimos:
             for file in _deimosFiles:
                 filePath = os.path.join(_infraDir, 'deimos', 'python', file)
                 if not os.path.isfile(filePath):
                     )
                 sources.append((winpath(filePath,self.winonly), 'infra'))
         # If using PydMain, parse the template file
-        if with_main:
-            name = [n for n, category in macros if category == 'name'][0]
+        if self.build_exe:
+            pass
+        elif self.with_main:
+            name = self.proj_name
             # Store the finished pydmain.d file alongside the object files
             infra_output_dir = winpath(os.path.join(output_dir, 'infra'), self.winonly)
             if not os.path.exists(infra_output_dir):
             mainFilename = os.path.join(infra_output_dir, 'pydmain.d')
             make_pydmain(mainFilename, name)
             sources.append((winpath(mainFilename,self.winonly), 'infra'))
-        # And meta
-        if with_meta:
-            for file in _metaFiles:
-                filePath = os.path.join(_infraDir, 'meta', file)
-                if not os.path.isfile(filePath):
-                    raise DistutilsPlatformError("Required meta source file"
-                        " '%s' is missing." % filePath
-                    )
-                sources.append((winpath(filePath,self.winonly), 'infra'))
-        # Add the infraDir to the include path for pyd, st, and meta.
+        # Add the infraDir to the include path for pyd, meta, and utils.
         includePathOpts += self._includeOpts
         includePathOpts[-1] = includePathOpts[-1] % winpath(os.path.join(_infraDir), self.winonly)
         
-        # Add DLL/SO boilerplate code file.
-        if _isPlatWin:
-            boilerplatePath = os.path.join(_infraDir, 'd',
-                'python_dll_windows_boilerplate.d'
-            )
+        if self.build_exe:
+            pass
         else:
-            boilerplatePath = os.path.join(_infraDir, 'd',
-                'python_so_linux_boilerplate.d'
-            )
-        if not os.path.isfile(boilerplatePath):
-            raise DistutilsFileError('Required supporting code file "%s"'
-                ' is missing.' % boilerplatePath
-            )
-        sources.append((winpath(boilerplatePath,self.winonly), 'infra'))
+            # Add DLL/SO boilerplate code file.
+            if _isPlatWin:
+                boilerplatePath = os.path.join(_infraDir, 'd',
+                    'python_dll_windows_boilerplate.d'
+                )
+            else:
+                boilerplatePath = os.path.join(_infraDir, 'd',
+                    'python_so_linux_boilerplate.d'
+                )
+            if not os.path.isfile(boilerplatePath):
+                raise DistutilsFileError('Required supporting code file "%s"'
+                    ' is missing.' % boilerplatePath
+                )
+            sources.append((winpath(boilerplatePath,self.winonly), 'infra'))
 
-        # Extension subclass DExtension will have packed any user-supplied
-        # version and debug flags into macros; we extract them and convert them
-        # into the appropriate command-line args.
-        versionFlags = [name for (name, category) in macros if category == 'version']
-        debugFlags = [name for (name, category) in macros if category == 'debug']
         userVersionAndDebugOpts = (
-              [self._versionOpt % v for v in versionFlags] +
-              [self._debugOpt   % v for v in debugFlags]
+              [self._versionOpt % v for v in self.versionFlagsFromExt] +
+              [self._debugOpt   % v for v in self.debugFlagsFromExt]
         )
 
-        # Python version option allows extension writer to take advantage of
-        # Python/C API features available only in recent version of Python with
-        # a version statement like:
-        #   version(Python_2_4_Or_Later) {
-        #     Py_ConvenientCallOnlyAvailableInPython24AndLater();
-        #   } else {
-        #     // Do it the hard way...
-        #   }
-        def pvo(opt):
-            optf = 'Python_%d_%d_Or_Later'
-            def pv2(minor):
-                return [opt % 'PydPythonExtension'] + [opt % (optf % (2,m)) for m in range(4,minor+1)]
-            def pv3(minor):
-                return [opt % 'PydPythonExtension'] + [opt % (optf % (3,m)) for m in range(0,minor+1)]
-            major = sys.version_info[0]
-            minor = sys.version_info[1]
-            if major == 2: return pv2(minor)
-            if major == 3: return  pv2(7) + pv3(minor)
-            assert False, "what python version is this, anyways?"
-            
-        pythonVersionOpts = pvo(self._versionOpt) 
-
         # Optimization opts
-        args = [a.lower() for a in sys.argv[1:]]
-        optimize = ('-o' in args or '--optimize' in args)
         if debug:
             optimizationOpts = self._debugOptimizeOpts
-        elif optimize:
+        elif self.optimize:
             optimizationOpts = self._releaseOptimizeOpts
         else:
             optimizationOpts = self._defaultOptimizeOpts
             outOpts[-1] = outOpts[-1] % _qp(winpath(objName,self.winonly))
             cmdElements = (
                 [binpath] + extra_preargs + compileOpts +
-                pythonVersionOpts+[ self._unicodeOpt] + optimizationOpts +
+                pythonVersionOpts + optimizationOpts +
                 includePathOpts + outOpts + userVersionAndDebugOpts +
                 [_qp(source)] + extra_postargs
             )
             print ("All binary output files are up to date.")
             return
 
-        # The .def file (on Windows) or -shared and -soname (on Linux)
-        sharedOpts = self._def_file(build_temp, output_filename)
-
+        if self.build_exe:
+            sharedOpts = []
+            pythonLibOpt = []
+            if target_desc != cc.CCompiler.EXECUTABLE:
+                raise LinkError('This CCompiler implementation should be building'
+                    ' an executable'
+                )
+        else:
+            # The .def file (on Windows) or -shared and -soname (on Linux)
+            sharedOpts = self._def_file(build_temp, output_filename)
+            if target_desc != cc.CCompiler.SHARED_OBJECT:
+                raise LinkError('This CCompiler implementation should be building '
+                    ' a shared object'
+                )
         # The python .lib file, if needed
         pythonLibOpt = self._lib_file(libraries)
         if pythonLibOpt:
             pythonLibOpt = _qp(pythonLibOpt)
 
-        if target_desc != cc.CCompiler.SHARED_OBJECT:
-            raise LinkError('This CCompiler implementation does not know'
-                ' how to link anything except an extension module (that is, a'
-                ' shared object file).'
-            )
 
         # Library linkage options
         print ("library_dirs: %s" % (library_dirs,))
         libOpts = gen_lib_options(self, library_dirs, runtime_library_dirs, libraries)
 
         # Optimization opts
-        args = [a.lower() for a in sys.argv[1:]]
-        optimize = ('-o' in args or '--optimize' in args)
         if debug:
             optimizationOpts = self._debugOptimizeOpts
-        elif optimize:
+        elif self.optimize:
             optimizationOpts = self._releaseOptimizeOpts
         else:
             optimizationOpts = self._defaultOptimizeOpts

examples/pyind/setup.py

+# usage: python setup.py pydexe
+from celerid.support import setup, Extension
+import platform
+
+
+maj = platform.python_version_tuple()[0] 
+if maj == "3":
+    projName = 'pyind3'
+    srcs = ['pyind3.d']
+    
+elif maj == "2":
+    projName = 'pyind'
+    srcs = ['pyind.d']
+else:
+    assert False, "want python 2 or python 3"
+
+setup(
+    name=projName,
+    version='1.0',
+    ext_modules=[
+    Extension(projName, srcs,
+        )
+    ],
+)

patch_distutils.py

 #   from celerid import patch_distutils
 
 from distutils import ccompiler as cc
-from distutils.command import build
+from distutils.command import build, build_ext
 
 from celerid import dcompiler
 
 
 # Force the distutils build command to recognize the '--optimize' or '-O'
 # command-line option.
-build.build.user_options.append(
-    ('optimize', 'O',
+build.build.user_options.extend(
+    [('optimize', 'O',
       'Ask the D compiler to optimize the generated code, at the expense of'
-      ' safety features such as array bounds checks.'),
-  )
+      ' safety features such as array bounds checks.'), ])
 build.build.boolean_options.append('optimize')
 
 _old_initialize_options = build.build.initialize_options
     _old_initialize_options(self)
     self.optimize = 0
 build.build.initialize_options = _new_initialize_options
+
+# Force build commands to actually send optimize option to D compilers
+
+_old_build_ext = build_ext.build_ext.build_extension
+
+def new_build_ext(self, ext):
+    if isinstance(self.compiler, dcompiler.DCompiler):
+        build = self.distribution.get_command_obj('build')
+        self.compiler.optimize = build.optimize or ext.pyd_optimize
+        self.compiler.with_pyd = ext.with_pyd
+        self.compiler.with_main = ext.with_main
+        self.compiler.build_deimos = ext.build_deimos
+        self.compiler.proj_name = ext.name
+        self.versionFlagsFromExt = ext.version_flags
+        self.debugFlagsFromExt = ext.debug_flags
+    _old_build_ext(self,ext)
+build_ext.build_ext.build_extension = new_build_ext
 __all__ = ('setup', 'Extension')
 
 from celerid import patch_distutils # Cause distutils to be hot-patched.
+import sys, os, os.path
 
-from distutils.core import setup, Extension as std_Extension
+from distutils.core import setup as base_setup, Extension as std_Extension, Command
+from distutils.util import get_platform
+from distutils.dep_util import newer_group
+from distutils import log
 from distutils.errors import DistutilsOptionError
 
 class Extension(std_Extension):
         # If the user has requested any version_flags or debug_flags, we use
         # the distutils 'define_macros' keyword argument to carry them (they're
         # later unpacked in the dcompiler module).
-        define_macros = []
-        if 'version_flags' in kwargs or 'debug_flags' in kwargs:
-            if 'version_flags' in kwargs:
-                for flag in kwargs['version_flags']:
-                    define_macros.append((flag, 'version'))
-                del kwargs['version_flags']
 
-            if 'debug_flags' in kwargs:
-                for flag in kwargs['debug_flags']:
-                    define_macros.append((flag, 'debug'))
-                del kwargs['debug_flags']
+        self.version_flags = kwargs.pop('version_flags',[])
+        self.debug_flags = kwargs.pop('debug_flags',[])
 
-        # Pass in the extension name so the compiler class can know it
-        if 'name' in kwargs:
-            define_macros.append((kwargs['name'], 'name'))
-        elif len(args) > 0:
-            define_macros.append((args[0], 'name'))
-
-        # Pass in the 'tango' flag, also
-        with_tango = kwargs.pop('tango', False)
-        if with_tango:
-            define_macros.append(('Pyd_with_Tango', 'version'))
-        kwargs['define_macros'] = define_macros
-
-        # Similarly, pass in with_pyd, &c, via define_macros.
         if 'raw_only' in kwargs:
             kwargs['with_pyd'] = False
-            kwargs['with_st'] = False
-            kwargs['with_meta'] = False
             kwargs['with_main'] = False
             del kwargs['raw_only']
-        with_pyd  = kwargs.pop('with_pyd', True)
-        build_deimos = kwargs.pop('build_deimos', False)
-        with_st   = kwargs.pop('with_st', False) # 5/23/07 st off by default.
-        # StackThreads doesn't work with Tango at the moment.
-        if with_tango:
-            with_st = False
-        with_meta = kwargs.pop('with_meta', True)
-        with_main = kwargs.pop('with_main', True)
-        if with_pyd and not with_meta:
-            raise DistutilsOptionError(
-                'Cannot specify with_meta=False while using Pyd. Specify'
-                ' raw_only=True or with_pyd=False if you want to compile a raw Python/C'
-                ' extension.'
-            )
-        if with_main and not with_pyd:
+        self.with_pyd  = kwargs.pop('with_pyd', True)
+        self.build_deimos = kwargs.pop('build_deimos', False)
+        self.with_main = kwargs.pop('with_main', True)
+        self.pyd_optimize = kwargs.pop('optimize', False)
+        if self.with_main and not self.with_pyd:
             # The special PydMain function should only be used when using Pyd
-            with_main = False
-
-        define_macros.append(((with_pyd, with_st, with_meta, with_main, build_deimos), 'aux'))
+            self.with_main = False
 
         std_Extension.__init__(self, *args, **kwargs)
+class build_pyd_embedded_exe(Command):
+    description = "Build a D application that embeds python with Pyd"
 
+    user_options = [
+            ("optimize", "O", "Ask the D compiler to optimize the generated code, at the expense of"
+                " safety features such as array bounds checks."),
+            ("print-flags", None, "Don't build, just print out version flags for pyd") ]
+
+    boolean_options = ['print-flags']
+
+    def initialize_options(self):
+        self.print_flags = False
+        self.compiler = None
+        self.build_temp = None
+        self.build_lib = None
+        self.build_platlib = None
+        self.build_base = "build"
+        self.include_dirs = None
+        self.libraries = None
+        self.library_dirs = None
+        self.link_objects = None
+        self.debug = 0
+        self.optimize = 0
+        self.dry_run = 0
+        self.verbose = 0
+        self.force = 0
+
+    def finalize_options(self):
+        self.extensions = self.distribution.ext_modules
+        plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3])
+        if self.build_platlib is None:
+            self.build_platlib = os.path.join(self.build_base, 'lib' + plat_specifier)
+        if self.build_lib is None:
+            self.build_lib = self.build_platlib
+        if self.build_temp is None:
+            self.build_temp = os.path.join(self.build_base, 'temp' + plat_specifier)
+
+    def run(self):
+        # mostly copied from distutils.command.build_ext
+        from distutils.ccompiler import new_compiler
+        if not self.extensions:
+            return
+        self.compiler = new_compiler(compiler=self.compiler,
+                verbose=self.verbose,
+                dry_run=self.dry_run,
+                force=self.force)
+        from celerid import dcompiler
+        assert isinstance(self.compiler, dcompiler.DCompiler)
+        self.compiler.build_exe = True
+        self.compiler.optimize = self.optimize
+        # irrelevant for D compilers?
+        #customize_compiler(self.compiler)
+        if self.include_dirs is not None:
+            self.compiler.set_include_dirs(self.include_dirs)
+        if self.libraries is not None:
+            self.compiler.set_libraries(self.libraries)
+        if self.library_dirs is not None:
+            self.compiler.set_library_dirs(self.library_dirs)
+        if self.link_objects is not None:
+            self.compiler.set_link_objects(self.link_objects)
+
+        if self.print_flags:
+            print( ' '.join(self.compiler.versionOpts()) )
+        else:
+            for ext in self.extensions:
+                self.per_ext(ext)
+    def per_ext(self, ext):
+            self.compiler.optimize = self.optimize or ext.pyd_optimize
+            self.compiler.with_pyd = ext.with_pyd
+            self.compiler.with_main = ext.with_main
+            self.compiler.build_deimos = ext.build_deimos
+            self.compiler.proj_name = ext.name
+            self.versionFlagsFromExt = ext.version_flags
+            self.debugFlagsFromExt = ext.debug_flags
+            # mostly copied from distutils.command.build_ext
+            sources = ext.sources
+            if sources is None or type(sources) not in (list, tuple):
+                raise DistutilsSetupError(
+                        ("in 'pydexe' option (extension '%s'), " +
+                        "'sources' must be present and must be " +
+                        "a list of source filenames") % ext.name)
+            sources = list(sources)
+            ext_path = self.get_ext_fullpath(ext.name)
+            depends = sources + ext.depends
+            if not (self.force or newer_group(depends, ext_path, 'newer')):
+                log.debug("skipping '%s' extension (up-to-date)", ext.name)
+                return
+            else:
+                log.info("building '%s' extension", ext.name)
+
+            extra_args = ext.extra_compile_args or []
+            macros = ext.define_macros[:]
+
+            objects = self.compiler.compile(sources, 
+                    output_dir=self.build_temp,
+                    macros=macros,
+                    include_dirs=ext.include_dirs,
+                    debug=self.debug,
+                    extra_postargs=extra_args,
+                    depends=ext.depends)
+            self._built_objects = objects[:]
+            if ext.extra_objects:
+                objects.extend(ext.extra_objects)
+            language = ext.language or self.compiler.detect_language(sources)
+
+            self.compiler.link_executable(objects, ext_path,
+                    libraries=self.get_libraries(ext),
+                    library_dirs=ext.library_dirs,
+                    runtime_library_dirs=ext.runtime_library_dirs,
+                    extra_postargs=extra_args,
+                    debug=self.debug,
+                    target_lang=language)
+            import shutil
+            shutil.copy(self.compiler.executable_filename(ext_path), '.')
+    def get_ext_fullpath(self, ext_name):
+        fullname = ext_name
+        modpath = fullname.split('.')
+        filename = self.get_ext_filename(ext_name)
+        filename = os.path.split(filename)[-1]
+
+        filename = os.path.join(*modpath[:-1]+[filename])
+        return os.path.join(self.build_lib, filename)
+
+    def get_ext_filename(self, ext_name):
+        from distutils.sysconfig import get_config_var
+        ext_path = ext_name.split( ".")
+        if os.name == "os2":
+            old_path = ext_path[:]
+            ext_path[len(ext_path)-1] = ext_path[len(ext_path)-1][:8]
+            assert False, ("build_ext does this, so it probably works, but check it anyways! We are on OS2, and OS2" +
+                    "does not permit file names of length > 8, so we are using %r instead of %r") % (old_path, ext_path)
+        # link_executable does this for us
+        # (override extension with compiler.exe_extension)
+        #exe_ext = get_config_var("EXE")
+        exe_ext = ''
+        if os.name == "nt" and self.debug:
+            return os.path.join(*ext_path) + "_d" + exe_ext
+        return os.path.join(*ext_path) + exe_ext
+
+    def get_libraries(self, ext):
+        return ext.libraries
+
+
+def setup(*args, **kwargs):
+    if 'cmdclass' not in kwargs:
+        kwargs['cmdclass'] = {}
+    kwargs['cmdclass']['pydexe'] = build_pyd_embedded_exe
+    base_setup(*args, **kwargs)