Commits

Jed Brown  committed 671d731 Merge

Merge branch 'balay/download-make' into jed/gnumake

* balay/download-make:
make: change formula to compute make_np [from number of cores] Also remove special code for Barry [probably needed by an a laptop older than his current]
gnumake: make it the default build mode
make: fix usage from programs.make to self.make [via packages.py]
make: clean up integration of programs.make into make.py
download-make: migrate prior 'make' detection code from programs.py to packages/make.py
download-make: add initial support. [yet to migrate stuff from programs.py]

  • Participants
  • Parent commits 8c71473, 4568d0e

Comments (0)

Files changed (8)

         if [ "$${makeflags_j}" != "$${makeflags}" ]; then parallel=true; fi;\
         if [ "$${parallel}" = "true" ]; then \
           printf ${PETSC_TEXT_HILIGHT}"********************* ERROR **********************\n" ; \
-          if [ "${PETSC_BUILD_USING_CMAKE}" != "" ]; then \
+          if [ "${PETSC_BUILD_USING_CMAKE}" != "" -o "${PETSC_BUILD_USING_GNUMAKE}" != "" ]; then \
              echo "PETSc compiles are automatically parallel, do not ";\
              echo "provide the -j option to make                     ";\
           else \
              echo "Parallel build with 'make -j' is not supported    ";\
              echo "for PETSc legacy builds. Run without -j <np>      ";\
-             echo "or ./configure PETSc with --download-cmake        ";\
+             echo "or configure PETSc with gnumake using --download-make";\
              echo "to enable parallel builds with PETSc              ";\
           fi;\
           printf "**************************************************"${PETSC_TEXT_NORMAL}"\n" ;\
 cmake: chk_makej
 	@${OMAKE} ccmake VERBOSE=1
 
+ggnumake: chk_makej
+	@echo "Building PETSc using GNU Make with ${MAKE_NP} build threads"
+	@echo "=========================================="
+	@cd ${PETSC_DIR} && ${OMAKE} -f gmakefile -j ${MAKE_NP} V=${V}
+	@if [ "${BUILDSHAREDLIB}" = "yes" -a "${DSYMUTIL}" != "true" ]; then \
+        echo "Running ${DSYMUTIL} on ${SHLIBS}";\
+        for LIBNAME in ${SHLIBS}; do ${DSYMUTIL} ${INSTALL_LIB_DIR}/$$LIBNAME.${SL_LINKER_SUFFIX}; done; fi
+	@echo "========================================="
+
+gnumake: chk_makej
+	@${OMAKE} ggnumake V=1
+
 # Does nothing; needed for some rules that require actions.
 foo:
 

File config/BuildSystem/config/package.py

     self.libraries     = framework.require('config.libraries', self)
     self.programs      = framework.require('config.programs', self)
     self.sourceControl = framework.require('config.sourceControl',self)
+    self.make          = framework.require('config.packages.make',self)
     self.mpi           = framework.require('config.packages.MPI',self)
 
     return

File config/BuildSystem/config/packages/MPI.py

     self.compilers = framework.require('config.compilers', self)
     self.compilerFlags = framework.require('config.compilerFlags', self)
     self.types = framework.require('config.types', self)
+    #self.make = framework.require('config.packages.make',self)
     return
 
   # search many obscure locations for MPI
         raise RuntimeError('Error running configure on OPENMPI/MPI: '+str(e))
       try:
         self.logPrintBox('Compiling OPENMPI/MPI; this may take several minutes')
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.programs.make+' clean', timeout=200, log = self.framework.log)
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.programs.make+' -j ' + str(self.programs.make_np)+' all', timeout=6000, log = self.framework.log)
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.programs.make+' install', timeout=6000, log = self.framework.log)
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.programs.make+' clean', timeout=200, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.make.make+' clean', timeout=200, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.make.make_jnp+' all', timeout=6000, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.make.make+' install', timeout=6000, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+openmpiDir+' && '+self.make.make+' clean', timeout=200, log = self.framework.log)
       except RuntimeError, e:
         raise RuntimeError('Error running make on OPENMPI/MPI: '+str(e))
       if not os.path.isdir(os.path.join(installDir,'lib')):
         raise RuntimeError('Error running configure on MPICH: '+str(e))
       try:
         self.logPrintBox('Running make on MPICH; this may take several minutes')
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+self.programs.make+' clean', timeout=200, log = self.framework.log)
-        makej_cmd = self.programs.make+' -j ' + str(self.programs.make_np)
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+makej_cmd+' all', timeout=6000, log = self.framework.log)
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+self.programs.make+' install', timeout=6000, log = self.framework.log)
-        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+self.programs.make+' clean', timeout=200, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+self.make.make+' clean', timeout=200, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+self.make.make_jnp+' all', timeout=6000, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+self.make.make+' install', timeout=6000, log = self.framework.log)
+        output,err,ret  = config.base.Configure.executeShellCommand('cd '+mpichDir+' && '+self.make.make+' clean', timeout=200, log = self.framework.log)
       except RuntimeError, e:
         import sys
         if sys.platform.startswith('cygwin'):

File config/BuildSystem/config/packages/__init__.py

-all = ['f2cblaslapack','BlasLapack', 'CHOLMOD', 'exodusii', 'scientificpython', 'fiat', 'MOAB', 'MPI', 'netcdf', 'PETSc', 'UMFPACK', 'boost', 'cusp', 'thrust', 'hdf5', 'netcdf-cxx']
+all = ['make','f2cblaslapack','BlasLapack', 'CHOLMOD', 'exodusii', 'scientificpython', 'fiat', 'MOAB', 'MPI', 'netcdf', 'PETSc', 'UMFPACK', 'boost', 'cusp', 'thrust', 'hdf5', 'netcdf-cxx']

File config/BuildSystem/config/packages/make.py

+from __future__ import generators
+import config.package
+
+class Configure(config.package.Package):
+  def __init__(self, framework):
+    config.package.Package.__init__(self, framework)
+    self.download          = ['http://ftp.gnu.org/gnu/make/make-3.82.tar.gz','http://ftp.mcs.anl.gov/pub/petsc/externalpackages/make-3.82.tar.gz']
+    self.complex           = 1
+    self.double            = 0
+    self.requires32bitint  = 0
+    self.worksonWindows    = 1
+    self.downloadonWindows = 1
+    self.useddirectly      = 0
+
+    self.flags             = ''
+    self.haveGNUMake       = 0
+    return
+
+  def setupHelp(self, help):
+    import nargs
+    help.addArgument('Make', '-with-make=<prog>',                            nargs.Arg(None, 'gmake', 'Specify GNU make'))
+    help.addArgument('Make', '-with-make-np=<np>',                           nargs.ArgInt(None, None, min=1, help='Default number of threads to use for parallel builds'))
+    help.addArgument('Make', '-download-make=<no,yes,filename>',             nargs.ArgDownload(None, 0, 'Download and install GNU make'))
+    help.addArgument('Make', '-download-make-cc=<prog>',                     nargs.Arg(None, None, 'C compiler for GNU make configure'))
+    help.addArgument('Make', '-download-make-configure-options=<options>',   nargs.Arg(None, None, 'additional options for GNU make configure'))
+    return
+
+  def Install(self):
+    import os
+    args = ['--prefix='+self.installDir]
+    args.append('--program-prefix=g')
+    if self.framework.argDB.has_key('download-make-cc'):
+      args.append('CC="'+self.framework.argDB['download-make-cc']+'"')
+    if self.framework.argDB.has_key('download-make-configure-options'):
+      args.append(self.framework.argDB['download-make-configure-options'])
+    args = ' '.join(args)
+    fd = file(os.path.join(self.packageDir,'make.args'), 'w')
+    fd.write(args)
+    fd.close()
+    if self.installNeeded('make.args'):
+      try:
+        self.logPrintBox('Configuring GNU Make; this may take several minutes')
+        output,err,ret  = config.package.Package.executeShellCommand('cd '+self.packageDir+' && ./configure '+args, timeout=900, log = self.framework.log)
+      except RuntimeError, e:
+        raise RuntimeError('Error running configure on GNU make (install manually): '+str(e))
+      try:
+        self.logPrintBox('Compiling GNU Make; this may take several minutes')
+        if self.getExecutable('make', getFullPath = 1,resultName='make',setMakeMacro = 0):
+          output,err,ret  = config.package.Package.executeShellCommand('cd '+self.packageDir+' && '+self.make+' &&  '+self.make+' install && '+self.make+' clean', timeout=2500, log = self.framework.log)
+        else:
+          output,err,ret  = config.package.Package.executeShellCommand('cd '+self.packageDir+' && ./build.sh && ./make install && ./make clean', timeout=2500, log = self.framework.log)
+      except RuntimeError, e:
+        raise RuntimeError('Error running make; make install on GNU Make (install manually): '+str(e))
+      self.postInstall(output+err,'make.args')
+    self.binDir = os.path.join(self.installDir, 'bin')
+    self.make = os.path.join(self.binDir, 'gmake')
+    self.addMakeMacro('MAKE',self.make)
+    return self.installDir
+
+  def configureMake(self):
+    '''Check for user specified make - or gmake, make'''
+    if self.framework.clArgDB.has_key('with-make'):
+      if not self.getExecutable(self.framework.argDB['with-make'],getFullPath = 1,resultName = 'make'):
+        raise RuntimeError('Error! User provided make not found :'+self.framework.argDB['with-make'])
+      self.found = 1
+      return
+    if not self.getExecutable('gmake', getFullPath = 1,resultName = 'make') and not self.getExecutable('make', getFullPath = 1,resultName = 'make'):
+      import os
+      if os.path.exists('/usr/bin/cygcheck.exe') and not os.path.exists('/usr/bin/make'):
+        raise RuntimeError('''\
+*** Incomplete cygwin install detected . /usr/bin/make is missing. **************
+*** Please rerun cygwin-setup and select module "make" for install.**************''')
+      else:
+        raise RuntimeError('Could not locate the make utility on your system, try --download-make')
+    self.found = 1
+    return
+
+  def configureCheckGNUMake(self):
+    '''Check for GNU make'''
+    self.getExecutable('strings', getFullPath = 1,setMakeMacro = 0)
+    if hasattr(self, 'strings'):
+      try:
+        (output, error, status) = config.base.Configure.executeShellCommand(self.strings+' '+self.make, log = self.framework.log)
+        if not status and output.find('GNU Make') >= 0:
+          self.haveGNUMake = 1
+      except RuntimeError, e:
+        self.framework.log.write('Make check failed: '+str(e)+'\n')
+      if not self.haveGNUMake:
+        try:
+          (output, error, status) = config.base.Configure.executeShellCommand(self.strings+' '+self.make+'.exe', log = self.framework.log)
+          if not status and output.find('GNU Make') >= 0:
+            self.haveGNUMake = 1
+        except RuntimeError, e:
+          self.framework.log.write('Make check failed: '+str(e)+'\n')
+    # mac has fat binaries where 'string' check fails
+    if not self.haveGNUMake:
+      try:
+        (output, error, status) = config.base.Configure.executeShellCommand(self.make+' -v dummy-foobar', log = self.framework.log)
+        if not status and output.find('GNU Make') >= 0:
+          self.haveGNUMake = 1
+      except RuntimeError, e:
+        self.framework.log.write('Make check failed: '+str(e)+'\n')
+
+    # Setup make flags
+    if self.haveGNUMake:
+      self.flags += ' --no-print-directory'
+      self.addMakeMacro('PETSC_BUILD_USING_GNUMAKE',1)
+
+    # Check to see if make allows rules which look inside archives
+    if self.haveGNUMake:
+      self.addMakeRule('libc','${LIBNAME}(${OBJSC})')
+      self.addMakeRule('libcu','${LIBNAME}(${OBJSCU})')
+    else:
+      self.addMakeRule('libc','${OBJSC}','-${AR} ${AR_FLAGS} ${LIBNAME} ${OBJSC}')
+      self.addMakeRule('libcu','${OBJSCU}','-${AR} ${AR_FLAGS} ${LIBNAME} ${OBJSCU}')
+    self.addMakeRule('libf','${OBJSF}','-${AR} ${AR_FLAGS} ${LIBNAME} ${OBJSF}')
+    return
+
+  def configureMakeNP(self):
+    '''check no of cores on the build machine [perhaps to do make '-j ncores']'''
+    make_np = self.framework.argDB.get('with-make-np')
+    if make_np is not None:
+      self.framework.logPrint('using user-provided make_np = %d' % make_np)
+    else:
+      def compute_make_np(i):
+        f16 = .80
+        f32 = .65
+        f64 = .50
+        f99 = .30
+        if (i<=2):    return 2
+        elif (i<=4):  return i
+        elif (i<=16): return int(4+(i-4)*f16)
+        elif (i<=32): return int(4+12*f16+(i-16)*f32)
+        elif (i<=64): return int(4+12*f16+16*f32+(i-32)*f64)
+        else:         return int(4+12*f16+16*f32+32*f64+(i-64)*f99)
+        return
+      try:
+        import multiprocessing # python-2.6 feature
+        cores = multiprocessing.cpu_count()
+        make_np = compute_make_np(cores)
+        self.framework.logPrint('module multiprocessing found %d cores: using make_np = %d' % (cores,make_np))
+      except (ImportError), e:
+        make_np = 2
+        self.framework.logPrint('module multiprocessing *not* found: using default make_np = %d' % make_np)
+    self.make_np = make_np
+    self.addMakeMacro('MAKE_NP',str(make_np))
+    self.make_jnp = self.make + ' -j ' + str(self.make_np)
+    return
+
+  def configure(self):
+    '''Determine whether (GNU) make exist or not'''
+
+    if self.framework.argDB['with-make'] == '0':
+      return
+    if (self.framework.argDB['download-make']):
+      config.package.Package.configure(self)
+    else:
+      self.executeTest(self.configureMake)
+    self.executeTest(self.configureCheckGNUMake)
+    self.executeTest(self.configureMakeNP)
+    self.addMakeMacro('OMAKE ', self.make+' '+self.flags)
+    return

File config/BuildSystem/config/programs.py

   def __str__(self):
     return ''
 
-  def setupHelp(self, help):
-    import nargs
-    import nargs
-    help.addArgument('PETSc', '-with-make=<prog>', nargs.Arg(None, 'make', 'Specify make'))
-    help.addArgument('PETSc', '-with-make-np=<np>', nargs.ArgInt(None, None, min=1, help='Default number of threads to use for parallel builds'))
-    return
-
-  def configureMake(self):
-    '''Check various things about make'''
-    self.getExecutable(self.framework.argDB['with-make'], getFullPath = 1,resultName = 'make')
-
-    if not hasattr(self,'make'):
-      import os
-      if os.path.exists('/usr/bin/cygcheck.exe') and not os.path.exists('/usr/bin/make'):
-        raise RuntimeError('''\
-*** Incomplete cygwin install detected . /usr/bin/make is missing. **************
-*** Please rerun cygwin-setup and select module "make" for install.**************''')
-      else:
-        raise RuntimeError('Could not locate the make utility on your system, make sure\n it is in your path or use --with-make=/fullpathnameofmake\n and run ./configure again')
-    # Check for GNU make
-    haveGNUMake = 0
-    self.getExecutable('strings', getFullPath = 1)
-    if hasattr(self, 'strings'):
-      try:
-        (output, error, status) = config.base.Configure.executeShellCommand(self.strings+' '+self.make, log = self.framework.log)
-        if not status and output.find('GNU Make') >= 0:
-          haveGNUMake = 1
-      except RuntimeError, e:
-        self.framework.log.write('Make check failed: '+str(e)+'\n')
-      if not haveGNUMake:
-        try:
-          (output, error, status) = config.base.Configure.executeShellCommand(self.strings+' '+self.make+'.exe', log = self.framework.log)
-          if not status and output.find('GNU Make') >= 0:
-            haveGNUMake = 1
-        except RuntimeError, e:
-          self.framework.log.write('Make check failed: '+str(e)+'\n')
-    # mac has fat binaries where 'string' check fails
-    if not haveGNUMake:
-      try:
-        (output, error, status) = config.base.Configure.executeShellCommand(self.make+' -v dummy-foobar', log = self.framework.log)
-        if not status and output.find('GNU Make') >= 0:
-          haveGNUMake = 1
-      except RuntimeError, e:
-        self.framework.log.write('Make check failed: '+str(e)+'\n')
-
-    # Setup make flags
-    self.flags = ''
-    if haveGNUMake:
-      self.flags += ' --no-print-directory'
-    self.addMakeMacro('OMAKE ', self.make+' '+self.flags)
-
-    # Check to see if make allows rules which look inside archives
-    if haveGNUMake:
-      self.addMakeRule('libc','${LIBNAME}(${OBJSC})')
-      self.addMakeRule('libcu','${LIBNAME}(${OBJSCU})')
-    else:
-      self.addMakeRule('libc','${OBJSC}','-${AR} ${AR_FLAGS} ${LIBNAME} ${OBJSC}')
-      self.addMakeRule('libcu','${OBJSCU}','-${AR} ${AR_FLAGS} ${LIBNAME} ${OBJSCU}')
-    self.addMakeRule('libf','${OBJSF}','-${AR} ${AR_FLAGS} ${LIBNAME} ${OBJSF}')
-
-    # check no of cores on the build machine [perhaps to do make '-j ncores']
-    make_np = self.framework.argDB.get('with-make-np')
-    if make_np is not None:
-      self.framework.logPrint('using user-provided make_np = %d' % make_np)
-    else:
-      try:
-        import multiprocessing
-        cores = multiprocessing.cpu_count()
-        make_np = max(min(cores+1,5),cores/3)
-        self.framework.logPrint('module multiprocessing found %d cores: using make_np = %d' % (cores,make_np))
-      except (ImportError), e:
-        make_np = 2
-        self.framework.logPrint('module multiprocessing *not* found: using default make_np = %d' % make_np)
-      try:
-        import os
-        import pwd
-        if 'barrysmith' == pwd.getpwuid(os.getuid()).pw_name:
-          # Barry wants to use exactly the number of physical cores (not logical cores) because it breaks otherwise.
-          # Since this works for everyone else who uses a Mac, something must be wrong with their systems. ;-)
-          try:
-            (output, error, status) = config.base.Configure.executeShellCommand('/usr/sbin/system_profiler -detailLevel full SPHardwareDataType', log = self.framework.log)
-            import re
-            match = re.search(r'.*Total Number Of Cores: (\d+)', output)
-            if match:
-              make_np = int(match.groups()[0])
-              self.framework.logPrint('Found number of cores using system_profiler: make_np = %d' % (make_np,))
-          except:
-            pass
-      except:
-        pass
-    self.make_np = make_np
-    self.addMakeMacro('MAKE_NP',str(make_np))
-    return
-
   def configureMkdir(self):
     '''Make sure we can have mkdir automatically make intermediate directories'''
     self.getExecutable('mkdir', getFullPath = 1, setMakeMacro = 0)
     return
 
   def configure(self):
-    if not self.framework.argDB['with-make'] == '0':
-      self.executeTest(self.configureMake)
-      self.executeTest(self.configureMkdir)
-      self.executeTest(self.configurePrograms)
+    self.executeTest(self.configureMkdir)
+    self.executeTest(self.configurePrograms)
     return

File config/PETSc/Configure.py

   def __str2__(self):
     desc = []
     desc.append('xxx=========================================================================xxx')
-    if self.getMakeMacro('PETSC_BUILD_USING_CMAKE'):
+    if self.make.getMakeMacro('PETSC_BUILD_USING_GNUMAKE'):
+      build_type = 'gnumake build'
+    elif self.getMakeMacro('PETSC_BUILD_USING_CMAKE'):
       build_type = 'cmake build'
     else:
       build_type = 'legacy build'
     self.functions     = framework.require('config.functions',          self)
     self.libraries     = framework.require('config.libraries',          self)
     self.atomics       = framework.require('config.atomics',            self)
+    self.make          = framework.require('config.packages.make',      self)
     self.blasLapack    = framework.require('config.packages.BlasLapack',self)
     if os.path.isdir(os.path.join('config', 'PETSc')):
       for d in ['utilities', 'packages']:
 all: chk_makej
 	@${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  PETSC_DIR=${PETSC_DIR} chk_petscdir chk_upgrade | tee ${PETSC_ARCH}/conf/make.log
 	@ln -sf ${PETSC_ARCH}/conf/make.log make.log
-	@if [ "${PETSC_BUILD_USING_CMAKE}" != "" ]; then \
+	@if [ "${PETSC_BUILD_USING_GNUMAKE}" != "" ]; then \
+	   ${OMAKE} PETSC_ARCH=${PETSC_ARCH} PETSC_DIR=${PETSC_DIR} all-gnumake-local 2>&1 | tee -a ${PETSC_ARCH}/conf/make.log; \
+	elif [ "${PETSC_BUILD_USING_CMAKE}" != "" ]; then \
 	   ${OMAKE} PETSC_ARCH=${PETSC_ARCH} PETSC_DIR=${PETSC_DIR} all-cmake-local 2>&1 | tee -a ${PETSC_ARCH}/conf/make.log \
 		| egrep -v '( --check-build-system |cmake -E | -o CMakeFiles/petsc[[:lower:]]*.dir/| -o lib/libpetsc|CMakeFiles/petsc[[:lower:]]*\.dir/(build|depend|requires)|-f CMakeFiles/Makefile2|Dependee .* is newer than depender |provides\.build. is up to date)'; \
 	 else \
         fi #solaris make likes to print the whole command that gave error. So split this up into the smallest chunk below
 	@if test -s ${PETSC_ARCH}/conf/error.log; then exit 1; fi
 
+all-gnumake:
+	@if [ "${PETSC_BUILD_USING_GNUMAKE}" != "" ]; then \
+          ${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  PETSC_DIR=${PETSC_DIR} PETSC_BUILD_USING_CMAKE="" all;\
+        else printf ${PETSC_TEXT_HILIGHT}"Build not configured for GNUMAKE. Quiting"${PETSC_TEXT_NORMAL}"\n"; exit 1; fi
+
 all-cmake:
 	@if [ "${PETSC_BUILD_USING_CMAKE}" != "" ]; then \
-          ${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  PETSC_DIR=${PETSC_DIR} all;\
+          ${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  PETSC_DIR=${PETSC_DIR} PETSC_BUILD_USING_GNUMAKE="" all;\
         else printf ${PETSC_TEXT_HILIGHT}"Build not configured for CMAKE. Quiting"${PETSC_TEXT_NORMAL}"\n"; exit 1; fi
 
 all-legacy:
-	@${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  PETSC_DIR=${PETSC_DIR} PETSC_BUILD_USING_CMAKE="" all
+	@${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  PETSC_DIR=${PETSC_DIR} PETSC_BUILD_USING_CMAKE="" PETSC_BUILD_USING_GNUMAKE="" all
+
+all-gnumake-local: chk_makej info gnumake
 
 all-cmake-local: chk_makej info cmakegen cmake