Commits

Anonymous committed 21257cb

see ChangeLog

Comments (0)

Files changed (9)

+14-03-2006 Martin Woudstra
+	cmt/requirements:
+	    - pattern declare_jobtransforms now uses standard paths
+	    - expand_files.py script now installed in InstallArea scripts area
+	cmt/expand_files.py: moved to ../share
+	python/trfutil.py: added class JobOptionsFile
+	python/trferr.py:
+	    - added class JobOptionsNotFoundError
+	    - *FileError classes now take filename as explicit additional argument
+	python/trf.py: preparing for adding user defined joboptions
+	python/basic_trf.py: fixed a few bugs in InputFileArg.preRunAction()
 7-03-2006 Martin Woudstra
 	python/trferr.py: added catching of exception KeyboardInterrupt
 	python/trf.py: 

cmt/expand_files.py

-#!/usr/bin/env python
-# usage: expand_files [options] [files] [options] [files]
-# Return a list of filenames with the package name prepended
-# Files will be collected from source dirs, and prepended with the
-# destination dir.
-# Options:
-# -r=<rootdir>: Root directory of the package. Is used to search for the
-#           files in the correct package.
-# -s=<srcdir>: Default source directory. Will be used if no directory is
-#          specified before a filename. This option can be inserted
-#          in between the filenames.
-# -d=<destdir>: destination directory. Will be prepended to each filename.
-#          Can be inserted in between the filenames. The last one before
-#          a filename will be used.
-import sys,os,glob
-srcdir = os.curdir
-destdir = ''
-rootdir=''
-files=[]
-for args in sys.argv[1:]:
-    for arg in args.split():
-        if arg.startswith('-s='):
-            # get a new default directory
-            srcdir=arg[3:]
-        elif arg.startswith('-d='):
-            destdir=arg[3:]
-        elif arg.startswith('-r='):
-            rootdir=arg[3:]
-        else:
-            argdir = os.path.dirname(arg)
-            if not argdir: argdir = srcdir
-            argname = os.path.basename(arg)
-            fullpath = os.path.normpath( os.path.join( rootdir,'cmt',argdir,argname) )
-            filelist = glob.glob( fullpath )
-            files += [ os.path.join(destdir,os.path.basename(f)) \
-                       for f in filelist ]
-
-if files:
-    print ' '.join( files )
 #      By default taken from ../share
 #       It will be installed in the Installarea/${trfs_installdir}/jobOptions
 #
-macro expand_files_cmd ${PYJOBTRANSFORMSCOREROOT}/cmt/expand_files.py
+macro expand_files_cmd expand_files.py
+
+apply_pattern declare_scripts files=expand_files.py
 
 macro trfs_installdir 'trfs'
 
 pattern declare_jobtransforms \
-	apply_pattern generic_declare_for_link kind=trfs   files='-s=../scripts <trfs>' prefix=${trfs_installdir} name=Trf ; \
-	apply_pattern generic_declare_for_link kind=trf_jo files='-s=../share <jo>' prefix="${trfs_installdir}/jobOptions" name=Trf ; \
-	macro <package>_jobtransforms "`${expand_files_cmd} -r=$(<PACKAGE>ROOT) -s=../scripts <trfs>`" ; \
-	macro_append all_jobtransforms " ${<package>_jobtransforms} " ; \
+	apply_pattern generic_declare_for_link kind=trfs_exe files='-s=../scripts <trfs>' prefix=share/bin ; \
+	apply_pattern generic_declare_for_link kind=trfs_pyt files='-s=../scripts <trfs>' prefix=python/<package> ; \
+	apply_pattern generic_declare_for_link kind=trfs_jop files='-s=../share <jo>'     prefix=jobOptions/<package> ; \
+	macro <package>_jobtransforms "`${expand_files_cmd} -r=$(<PACKAGE>ROOT) -d=<package> -s=../scripts <trfs>`" ; \
+	macro_append all_jobtransforms " ${<package>_jobtransforms} "
 
 
 # the use of CMTINSTALLAREA macro below is not documented anywhere, but it works.
 # Doing the path* and set* commands outside of a cmtpath_pattern does not work, since the
 # one in AtlasPolicy removes everything I set here, and is apparently called after the
 # 'normal' path* and set* commands.
-cmtpath_pattern \
-	path_remove  PYTHONPATH "/$(trfs_installdir)" ; \
-	path_remove  PATH  "/$(trfs_installdir)" ; \
-	set_remove   JOBOPTSEARCHPATH "/$(trfs_installdir)/jobOptions"
+#cmtpath_pattern \
+#	path_remove  PYTHONPATH "/$(trfs_installdir)" ; \
+#	path_remove  PATH  "/$(trfs_installdir)" ; \
+#	set_remove   JOBOPTSEARCHPATH "/$(trfs_installdir)/jobOptions"
 
-cmtpath_pattern \
-	path_prepend PYTHONPATH "$(CMTINSTALLAREA)/$(trfs_installdir)" ; \
-	path_prepend PATH "$(CMTINSTALLAREA)/$(trfs_installdir)" ; \
-	set_prepend  JOBOPTSEARCHPATH ",$(CMTINSTALLAREA)/$(trfs_installdir)/jobOptions"
+#cmtpath_pattern \
+#	path_prepend PYTHONPATH "$(CMTINSTALLAREA)/$(trfs_installdir)" ; \
+#	path_prepend PATH "$(CMTINSTALLAREA)/$(trfs_installdir)" ; \
+#	set_prepend  JOBOPTSEARCHPATH ",$(CMTINSTALLAREA)/$(trfs_installdir)/jobOptions"

python/basic_trfarg.py

     def preRunAction(self):
         """Check that the file has the correct suffix, exists, and is readable"""
         val = self.value()
-        if not self.checkSuffix( val ):
+        if not self.checkFileSuffix( val ):
             raise TransformArgumentError( '%s=%s does not have suffix %s' % (self.name(),repr(val),self.fileSuffix()) )
         
         if not Filename.exists(val):
                 self.setValue(att)
                 val = att
             else:
-                raise InputFileError( '%s not found' % val )
+                raise InputFileError( val, 'not found. Argument %s' % (self.name()) )
 
         if not Filename.isReadable(val):
-            raise InputFileError( '%s exists but is not readable' % val )
+            raise InputFileError( val, 'exists but not readable. Argument %s' % (self.name()) )
 
 
-        self._logger.debug( 'Inputfile %s suffix OK and file is found -> OK' % (self.name(), self.fileSuffix(), val) )
+        self._logger.debug( 'Inputfile %s suffix OK and file is found -> OK' % (val) )
     
 
     def type(self):
                     vals[vals.index(val)] = att
                     val = att
                 else:
-                    raise InputFileError( '%s not found' % val )
+                    raise InputFileError( val, 'not found. Argument %s=%s' % (self.name(),self.value()) )
 
             if not Filename.isReadable(val):
-                raise InputFileError( '%s exists but is not readable' % val )
+                raise InputFileError( val, 'exists but not readable. Argument %s=%s' % (self.name(),self.value()) )
 
 
         self._logger.debug( 'Inputfiles %s suffices OK and all files are found -> OK' % (self.name(), self.fileSuffix(), val) )
         elif Filename.exists(val):
             try: os.remove(val)
             except:
-                raise OutputFileError( 'ERROR while trying to pre-remove expected output file %s=%s' % (self.name(),repr(val)) )
+                raise OutputFileError( val, 'could not be removed before running. Argument %s' % (self.name()) )
             else:
                 mess = 'Checked suffix %s and removed output file %s -> OK' % (self.fileSuffix(), val)
         else:
         if val.upper() == 'NONE':
             mess = 'No output file expected. Nothing to be done.'
         elif not Filename.exists(val):
-            raise OutputFileError( '%s=%s not found' % (self.name(),repr(val)) )
+            raise OutputFileError( val, 'not found. Argument %s' % (self.name()) )
         elif not Filename.isRegular(val):
-            raise OutputFileError( '%s=%s is not a regular file' % (self.name(),repr(val)) )
+            raise OutputFileError( val, 'is not a regular file. Argument %s' % (self.name()) )
         else:
             mess = 'Outputfile %s found. -> OK' % (self.value())
 

python/full_trfarg.py

     def preRunAction(self):
         """Check than jobOptions file can be found"""
         val = self.value()
+        JobOptionsFile(val).preRunAction()
         full = find_joboptions( val )
-        if not full:
-            raise TransformArgumentError( '%s=%s not found' % (self.name(),repr(val)) )
+        if not full: raise JobOptionsNotFoundError( val, "Argument %s" % self.name() )
 
         self._logger.debug( 'Found %s in %s' % (val, strip_suffix(full,val)) )
 
     _runArgsName = 'runArgs'
     
     def __init__( self, version, authors, help, skeleton='default', name='default' ):
-        # set some dynamic defaults to arguments
+        # set filename of python file that defines the transformation
         filename = os.path.basename(sys._getframe(1).f_code.co_filename)
+        self._filename = filename
+        # set the name of the transformation
         if name == 'default':
             # start from filename where constructor is called
             name = filename
             name = strip_suffix( name, JobTransform._fileExtension )
             # remove suffix
             name = strip_suffix( name, JobTransform._fileSuffix )
+        self._name    = name
+        # set the skeleton name
         if skeleton == 'default':
-            skeleton = "skeleton.%s.py" % name
+            self._skeleton = self.defaultSkeletonFilename()
+        elif skeleton == '':
+            self._skeleton = None
+        else:
+            self._skeleton = skeleton
         #  set logger
         self._loggerName = name
         self._logger = logging.getLogger( self._loggerName )
         self._logger.setLevel( logging.INFO )
         trferr.errorHandler.setLoggingContext( self._loggerName )
         self._logfilename = name + '.log'
-        # arguments
-        self._filename = filename
-        self._name    = name
+        # set other init arguments
         self._version = version
         self._help    = help
         self._jobOpts = [ ]
         self._authors = [ ]
+        # initialise arguments members
         self._namedArgs = { }
         self._positionalArgs = [ ]
         self._optionalArgs = { }
         self._requiredArgs = { }
-        self._skeleton = skeleton
         self._runArgs = RunArguments()
         # prerun and postrun actions
         self._preRunActions = [ ]
         self._postRunActions = [ ]
-        # store the various jobOption templates
+        # pre-skeleton and post-skeleton joboptions
+        self._preJobOptionsFiles = [ ]
+        self._postJobOptionsFiles = [ ]
+        # store the various jobOptions templates
         self._authorsTemplate = ''
         self._runArgsHeaderTemplate = ''
         self._runArgsCodeTemplate = ''
         self._skeletonCodeTemplate = ''
         self._skeletonTemplate = ''
 
+        self.addAuthors( authors )
+
         self.addTemplates( 'runArgsHeader',
                            "# Run arguments file auto-generated on %s by:" % (time.ctime()) ,
                            "# JobTransform: %s" % (name) ,
                            "# JobTransform: %s" % (name) ,
                            "# Version: %s" % (version) )
                            
-        self.addAuthors( authors )
 
         self.addTemplates( 'skeletonCode', "import os,sys" )
         
         return self._skeleton
 
 
+    def hasSkeleton(self):
+        return self._skeleton is not None
+
+
     def logFilename(self):
         return self._logfilename
 
                 if not arg.hasValue():
                     raise TransformArgumentError, '%s has no value' % (arg.name())
 
-            #get the needed top jobOptions files
+            #get/check the needed top jobOptions files
             self.ensureSkeleton()
             self.writeRunArgs()
-            
+            for jo in self._preJobOptionsFiles + self._postJobOptionsFiles:
+                if not find_joboptions( jo ):
+                    raise TransformDefinitionError( "Top joboptions file %s not found" % (jo) )
             # run the athena job
             athena = os.path.join( trfenv.trfPath, 'athena_wrapper.py' )
-            syscommand = '%s %s %s 2>&1 | tee %s' % ( athena, self.runArgsFilename(), self.skeletonFilename(),
-                                                 self.logFilename() )
+            jobOptions = [ self.runArgsFilename() ]
+            jobOptions += self._preJobOptionsFiles
+            jobOptions += [ self.skeletonFilename() ]
+            jobOptions += self._postJobOptionsFiles
+            syscommand = '%s %s 2>&1 | tee -a %s' % ( athena, ' '.join(jobOptions), self.logFilename() )
             self._logger.debug( 'Executing %s' % (syscommand) )
             status = os.system( syscommand ) >> 8 # status code is in upper byte of 16 bit word
             if status != 0:
 from PyJobTransformsCore.trfutil import *
 
 __all__ = [ 'TransformError', 'TransformDefinitionError', 'TransformArgumentError',
-            'TransformEnvError', 'InputFileError', 'OutputFileError', 'TransformErrorHandler' ]
+            'TransformEnvError', 'InputFileError', 'OutputFileError',
+            'JobOptionsNotFoundError', 'TransformErrorHandler' ]
 
 exitStatusFile = 'exitstatus'
 
         
 
 class InputFileError( TransformArgumentError ):
-    def __init__(self,message=None,errnum=TRF_INPUTFILE_ERROR):
-        TransformArgumentError.__init__(self,message,errnum) 
+    def __init__(self,filename,message=None,errnum=TRF_INPUTFILE_ERROR):
+        mess = "Input file %s" % filename
+        if message: mess += ' ' + message
+        TransformArgumentError.__init__(self,mess,errnum) 
+        self._filename = filename
 
 
+    def filename(self):
+        return self._filename
+    
+
 class OutputFileError( TransformArgumentError ):
-    def __init__(self,message=None,errnum=TRF_OUTPUTFILE_ERROR):
-        TransformArgumentError.__init__(self,message,errnum) 
+    def __init__(self,filename,message=None,errnum=TRF_OUTPUTFILE_ERROR):
+        mess = "Output file %s" % filename
+        if message: mess += ' ' + message
+        TransformArgumentError.__init__(self,mess,errnum) 
+        self._filename = filename
+
+
+    def filename(self):
+        return self._filename
+    
+
+
+class JobOptionsNotFoundError( TransformError ):
+    """Exception raised in case a joboptions file can not be found"""
+    def __init__(self,filename,message=None,errnum=JOBOPTIONS_NOT_FOUND):
+        mess = "JobOptions file %s not found" % filename
+        if message: mess += '. ' + message
+        TransformError.__init__(self,mess,errnum)
 
 
 class TransformErrorHandler:

python/trfutil.py

         """Try if filename exists with attempt number appended. If file exists,
         return the filename including the attempt number, else return None"""
         pat = re.compile( r'%s\.[0-9]+' % filename )
-        for f in os.listdir( os.path.dirname(filename) ):
+        dir = os.path.dirname(filename) or os.curdir
+        for f in os.listdir( dir ):
             if pat.search( f ): return f
 
         return None
   
     
 
+class JobOptionsFile:
+    def __init__(self,filename):
+        self._filename = filename
+
+
+    def filename(self):
+        return self._filename
+
+
+    def preRunAction(self):
+        if not find_joboptions( self._filename ):
+            raise JobOptionsNotFoundError( self._filename )
+    

share/expand_files.py

+#!/usr/bin/env python
+# usage: expand_files [options] [files] [options] [files]
+# Return a list of filenames with the package name prepended
+# Files will be collected from source dirs, and prepended with the
+# destination dir.
+# Options:
+# -r=<rootdir>: Root directory of the package. Is used to search for the
+#           files in the correct package.
+# -s=<srcdir>: Default source directory. Will be used if no directory is
+#          specified before a filename. This option can be inserted
+#          in between the filenames.
+# -d=<destdir>: destination directory. Will be prepended to each filename.
+#          Can be inserted in between the filenames. The last one before
+#          a filename will be used.
+import sys,os,glob
+srcdir = os.curdir
+destdir = ''
+rootdir=''
+files=[]
+for args in sys.argv[1:]:
+    for arg in args.split():
+        if arg.startswith('-s='):
+            # get a new default directory
+            srcdir=arg[3:]
+        elif arg.startswith('-d='):
+            destdir=arg[3:]
+        elif arg.startswith('-r='):
+            rootdir=arg[3:]
+        else:
+            argdir = os.path.dirname(arg)
+            if not argdir: argdir = srcdir
+            argname = os.path.basename(arg)
+            fullpath = os.path.normpath( os.path.join( rootdir,'cmt',argdir,argname) )
+            filelist = glob.glob( fullpath )
+            files += [ os.path.join(destdir,os.path.basename(f)) \
+                       for f in filelist ]
+
+if files:
+    print ' '.join( files )
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.