Source

mana-core-pyjobtransformscore / python / basic_trfarg.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
################################################################################
# This file contains the basic argument types for JobTransforms.
# They can not be used directly as JobTransform arguments.
# Classes defined in module full_trfarg.py can be used in JobTransforms. 
# See the doc of the Argument class for more information.
################################################################################

import os,tarfile

from AthenaCommon.Logging import logging

from PyJobTransformsCore.trfutil import *
from PyJobTransformsCore.trferr import *

#################################################################################
# Argument base class
################################################################################
class Argument:
    """Argument base class specifying basic properties of a command line argument.
Only fully specified argument classes can be used in a JobTransform implementation.
A fully specified argument should derive from one of the basic argument classes
defined in this file, and should override the member function isFullArgument(self),
which should return True."""
    
    def __init__(self,help,name='default'):
        """help    : short help string to help user fill in the argument
       name    : name of the argument (string)."""
        if help == 'default': help = self.defaultHelp()
        if name == 'default': name = self.defaultName()
        self._logger = logging.getLogger( 'Argument:%s' % name )
        self._name = name
        self._help = help
        self._value = None
        self._position = 0 # meaning unknown


    def setLoggingContext(self,context):
        name = '%s.%s' % (context, self._name)
        self._logger = logging.getLogger( name )

        
    def help(self):
        return self._help
    

    def fullHelp(self):
        help = ''
        if self.hasPosition():
            help += '%2d ' % self.position()
        else:
            help += ' ? '
        
        help += '%s (%s,%s)' % (self._name, self.basicType(), self.argumentType())
        if self.hasDefault(): help += ' default=%s' % (repr(self.getDefault()))
        return '%s # %s' % (help,self.help())


    def defaultHelp(self):
        """Return default help string. Must be implemented in derived class"""
        raise TransfromDefinitionError, 'defaultHelp() not implemented in class %s' % self.__class__.__name__
    

    def hasDefault(self):
        return hasattr(self,'default')


    def getDefault(self):
        return getattr(self,'default',None)


    def setDefault(self,default):
        """ Setting the default value for this argument. Makes the argument optional."""
        if default is None:
            if self.hasDefault(): del self.default
        else:
            try: self.default = self.toPython(default)
            except TransformDefinitionError :
                raise TransformDefinitionError( 'Default value %s=%s is not of type %s' % \
                                                (self._name, repr(default), self.basicType()) )


    def setName(self,name):
        self._name = name


    def name(self):
        return self._name


    def defaultName(self):
        """Equal to the argumentType with first character converted to lowercase"""
        meta = self.argumentType()
        return meta[0].lower() + meta[1:]


    def setValue(self,value):
        """Set the value, which can either be:
        - a string which will be interpreted as being of type as specified by self.basicType()
        - an objectc of the python type as specified by self.basicType()"""
        
        if value is None:
            self._value = None
        else:
            self._value = self.toPython(value)

        #return value as python type
        return self._value
    

    def value(self):
        return self._value


    def hasValue(self):
        return self._value is not None
    

    def setPosition(self,pos):
        self._position = max(0,pos)
            

    def hasPosition(self):
        return self._position > 0


    def position(self):
        return self._position


    def argumentType(self):
        """Type of argument, giving information about the kind of argument. Equals classname minus the 'Arg' suffix"""
        meta = self.__class__.__name__
        # remove the "Arg" postfix from the classname
        meta = strip_suffix(meta, 'Arg')
        return meta


    def isOptional(self):
        return hasattr(self,'default')


    def namedTypeFormat(self):
        return '%%(%s)%s' % (self._name,self.typeFormat())


    def pythonVariableTemplate(self,objName=None):
        plate = ''
        if objName: plate += objName + '.'
        plate += '%s = %s' % (self._name,self.namedTypeFormat())
        return plate


    def runArgsTemplate(self,objName):
        return self.pythonVariableTemplate(objName)


    def runArgsComment(self):
        return os.linesep + CommentLine(self._help).smallComment()
        

    def jobOrTask(self):
        return 'task'


    def _notImplemented(self,what):
        """Private helper function. Raise exception with error message in case something is not implemented in a class"""
        raise TransfromDefinitionError( '%s not implemented in class %s' % (what, self.__class__.__name__) )


    #
    # Member functions that need to be implemented by some intermediate derived class 
    #
    def toPython(self,val):
        """Should convert <val> (which is either a string or a python object of the correct type) to
        a python object of the python type corresponding to the basicType() of this argument.
        If <val> is not of the correct python type and can not be converted to it, a TransformArgumentError
        exception should be raised.
        Example types: int,float,str,list"""
        self._notImplemented('toPython()')


    def basicType(self):
        """Should return a string indicating the basic python type for this argument. Mainly for debugging."""
        self._notImplemented('basicType()')


    def typeFormat(self):
        """Should return a string containing the type specifier of the format string as used in the <str> % <str> syntax.
        Example: basicType 'int' should return 'd', 'string' should return 's', etc."""
        self._notImplemented('typeFormat()')


    def type(self):
        """Should return a string specifying the type as defined by the Production System"""
        self._notImplemented('type()')


    def metaType(self):
        """Should return a string specifying the metaType as defined by the Production System"""
        self._notImplemented('metaType()')


    #
    # Member functions to be implemented by fully specified argument classes
    #
    def isFullArgument(self):
        """Should return True for a fully implemented argument that can be used in a transformation.
        This is to avoid people using non-fully specified arguments in a transformation."""
        return False

#
# end of class Argument
#

################################################################################
# Add-in argument helper base classes
################################################################################
class ArgRange:
    """Argument range add-in class."""
    def __init__(self, minimum=None, maximum=None):
        if minimum is not None: self.minimum = minimum
        if maximum is not None: self.maximum = maximum


    def hasMinimum(self):
        return hasattr(self,'minimum')


    def hasMaximum(self):
        return hasattr(self,'maximum')


    def getMinimum(self):
        return getattr(self,'minimum',None)


    def getMaximum(self):
        return getattr(self,'maximum',None)


    def checkRange(self,val):
        """ Check the range of <val> (python type). Returns integer:
         0 : in range
        -1 : below minimum
        +1 : above maximum
        """
        
        if self.hasMinimum() and val < self.minimum: return -1
        if self.hasMaximum() and val > self.maximum: return +1
        return 0

#
# end of class ArgRange
#



class ArgChoices:
    def __init__(self, choices):
        """Argument list of choices add-in class"""
        self.choices = choices


    def checkChoices(self,val):
        """ Returns boolean indicating that <val> (python type) is one of choices."""
        return val in choices

#
# end of class ArgChoices
#



################################################################################
# Basic type arguments
################################################################################
    
class IntegerArg( Argument ):
    """Basic argument type. Any integer. Only type is checked before running"""
    def __init__(self,help,name='default'):
         Argument.__init__(self,help,name)


    def basicType(self):
        return 'int'

         
    def type(self):
        return 'plain'


    def typeFormat(self):
        return 'd'


    def toPython(self,val):
        """Turn a command line argument string into an int python object"""
        try: return int(val)
        except ValueError :
            raise TransformArgumentError( '%s=%s is not of type %s' % \
                                          (self.name(), repr(val), self.basicType()) )
        

    def metaType(self):
        return 'natural'

    

class FloatArg( Argument ):
    """Basic argument type. Any float. Only type is checked before running"""
    def __init__(self,help,name='default'):
        Argument.__init__(self,help,name)


    def basicType(self):
        return 'float'


    def typeFormat(self):
        return 'g'


    def toPython(self,val):
        """Turn a command line argument string into an float python object"""
        try: return float(val)
        except ValueError :
            raise TransformArgumentError( '%s=%s is not of type %s' % \
                                          (self.name(), repr(val), self.basicType()) )


    def metaType(self):
        return 'float'

    

class StringArg( Argument ):
    """Basic argument type. Any string."""
    def __init__(self,help,name='default'):
        Argument.__init__(self,help,name)


    def basicType(self):
        return 'str'


    def type(self):
        return 'plain'


    def typeFormat(self):
        return 's'


    def toPython(self,val):
            try: return str(val+'')
            except TypeError :
                raise TransformArgumentError( '%s=%s is not of type %s' % \
                                              (self.name(), repr(val), self.basicType()) )


    def pythonVariableTemplate(self,objName=None):
        plate = ''
        if objName: plate += objName + '.'
        plate += '%s = \'%s\'' % (self.name(),self.namedTypeFormat())
        return plate


    def metaType(self):
        return 'string'


class BoolArg( Argument ):
    """Basic argument type. A boolean. Recognised input (string) values:
    '0' 'False' 'F' 'false' 'f'  : False (represented by int 0)
    '1' 'True' 'T' 'true; 't'    : True  (represented by int 1)"""
    
    _falseValues = ( '0', 'F', 'f', 'False', 'false' )
    _trueValues  = ( '1', 'T', 't', 'True' , 'true'  )

    def __init__(self,help,name='default'):
        Argument.__init__(self,help,name)


    def basicType(self):
        return 'bool'


    def type(self):
        return 'plain'


    def typeFormat(self):
        return 'd'  #python does not have a bool type. It uses int (0 or 1) instead.


    def toPython(self,val):
        if type(val) == type(True): return val != False
        if type(val) == type(''):
            if val in BoolArg._falseValues: return False
            if val in BoolArg._trueValues:  return True
            raise TransformArgumentError( '%s=%r is not one of %s' % \
                                          (self.name(), val,
                                           ','.join( BoolArg._falseValues + BoolArg._trueValues )) )
        else:
            raise TransformArgumentError( '%s=%r is not of type %s' % \
                                          (self.name(), val, self.basicType()) )


    def metaType(self):
        return 'bool'



class FileListArg( Argument ):
    """List of filenames coded with the following syntax:
    <prefix>[<list>]<postfix> where '[' and ']' are litteral characters and <list>
    gives a recipe for a list of numbers to be generated (with <n> and <m> integers): 
    <n>,<m> (enumeration) or <n>-<m> (range including m) or a combination of those.
    A list of filenames is generated where the [<list>] is replaced by the actual
    integers that the list represents.
    The [<list>] part can also be omitted, in which case a list of one filename
    is generated (as given)."""

    def __init__(self,help,name):
        Argument.__init__(self,help,name)


    def basicType():
        return 'list'


    def type(self):
        return 'plain'


    def toPython(self,valIn):
        # if already a list, nothing to be done
        if type(valIn) == type(list()): return valIn
        # make a list of python types out of the strings
        if type(valIn) == type(''):
            all = StringNumberList(valIn).getStringList()
            if all is None:
                raise TransformArgumentError( '%s=%s: Invalid string range syntax' % \
                                              (self.name(),valIn) )
            return all
        
        # if we get here, there is problem
        raise TransformDefinitionError( '%s=%s: value is not a list or a string' % (self.name(), valIn) )



class StringChoicesArg( ArgChoices, StringArg ):
    """A string from a possible list of strings. Tested before running"""
    def __init__(self,help,name,choices):
        ArgChoices.__init__(self, choices)
        StringArg.__init__(self,help,name)


    def type(self):
        return 'plain'


    def preRunAction(self):
        """Return boolean indicating if <argVal> is one of choices"""
        val = self.value()
        name = self.name()
        choices = self.choices
        if not ArgChoices.checkChoices(self,val):
            raise TransformArgumentError( '%s=%s is not in %s' %
                                          (name, repr(val), choices) )

        self._logger.debug( '%s is in list %s -> OK' % (repr(val), repr(choices)) )

            
#
# end of class StringChoicesArg
#


class InputFileArg( Filename, StringArg ):
    """Input file. Name suffix, existence and readability of file is checked before run"""
    def __init__(self,help,name,type,contents):
        Filename.__init__(self,type,contents)
        StringArg.__init__(self,help,name)
        
        
    def preRunAction(self):
        """Check that the file has the correct suffix, exists, and is readable"""
        val = self.value()
        if not self.checkFileSuffix( val ):
            raise TransformArgumentError( '%s=%s does not have suffix %s' % (self.name(),repr(val),self.fileSuffix()) )
        
        if not Filename.exists(val):
            att = Filename.existsAttempt(val);
            if att:
                self._logger.warning('replacing %s with %s' % (val,att) )
                self.setValue(att)
                val = att
            else:
                raise InputFileError( val, 'not found. Argument %s' % (self.name()) )

        if not Filename.isReadable(val):
            raise InputFileError( val, 'exists but not readable. Argument %s' % (self.name()) )


        self._logger.debug( 'Inputfile %s suffix OK and file is found -> OK' % (val) )
    

    def type(self):
        return 'inputDS'


    def metaType(self):
        return self.fileContents().upper()

    

class InputFileListArg( Filename, FileListArg ):
    """List of input files. Name suffix, existence and readability of all files is checked before run"""
    def __init__(self,help,name,type,contents):
        Filename.__init__(self,type,contents)
        FileListArg.__init__(self,help,name)
        
        
    def preRunAction(self):
        """Check that the file has the correct suffix, exists, and is readable"""
        vals = self.value()
        for val in vals:
            if not self.checkFileSuffix( val ):
                raise TransformArgumentError( '%s=%s does not have suffix %s' % (self.name(),repr(val),self.fileSuffix()) )
        
            if not Filename.exists(val):
                att = Filename.existsAttempt(val);
                if att:
                    self._logger.warning('replacing %s with %s' % (val,att) )
                    vals[vals.index(val)] = att
                    val = att
                else:
                    raise InputFileError( val, 'not found. Argument %s=%s' % (self.name(),self.value()) )

            if not Filename.isReadable(val):
                raise InputFileError( val, 'exists but not readable. Argument %s=%s' % (self.name(),self.value()) )


        self._logger.debug( 'Inputfiles %s suffices OK and all files are found -> OK' % (self.name(), self.fileSuffix(), val) )
    

    def type(self):
        return 'inputDSlist'


    def metaType(self):
        return self.fileContents().upper()
    

class InputDataFileArg( InputFileArg ):
    """Class for input POOL data files"""
    def __init__(self,help,name,contents,type=PoolDataFile.defaultType):
        InputFileArg.__init__(self,help,name,type,contents)
    


class InputTarFileArg( InputFileArg ):
    def __init__(self,help,name,destdir='.',type='tar|tar.gz|tgz'):
        InputFileArg.__init__(self,help,name,type,None)
        self._filelist = [ ]
        self._destdir = destdir


    def filelist(self):
        """Return a list of filenames in archive"""
        if not self._filelist:
            if not self.hasValue():
                raise TransformDefinitionError( "%s filelist() called before value is set" % self.name() )
            tar = tarfile.open(self.value())
            self._filelist = tar.getnames()
            tar.close()
        return self._filelist


    def filelistRE(self,pattern):
        """Return list of files in archive that match the regular expression <pattern>"""
        files = self.filelist()
        return [ s for s in files if re.search( pattern, s ) ]


    def extract(self):
        """Extract all files from the archive and check their presence"""
        # check that we have a value
        if not self.hasValue():
            raise TransformDefinitionError( "%s extract() called before value is set" % self.name() )
        # do the extraction
        tar = tarfile.open(self.value())
        for f in tar.getmembers(): tar.extract(f, self._destdir)
        # bonus: set the filelist
        self._filelist = tar.getnames()
        tar.close()
        # check that all files are extracted
        for f in self.filelist():
            if not Filename.exists( os.path.join( self._destdir, f ) ):
                raise TransformArgumentError( "%s=%s FAILED to extract file %s from archive",
                                              self.name(), self.value(), f )


    def preRunAction(self):
        """Check presence of file and untar/zip it"""
        InputFileArg.preRunAction(self)
        self.extract()

       


class OutputFileArg( Filename, StringArg ):
    """Output file. Name suffix is tested before run, and file is deleted before run (if needed).
    Existence of file is tested after run"""
    def __init__(self,help,name,type,contents):
        Filename.__init__(self,type,contents)
        StringArg.__init__(self,help,name)


    def preRunAction(self):
        """Remove output file before running"""
        val = self.value()
        mess = ''
        if val.upper() == 'NONE':
            mess = 'No output file expected. Nothing to be done.'
        elif not self.checkFileSuffix(val):
            raise TransformArgumentError( '%s=%s does not have suffix %s' % (self.name(), repr(val), self.fileSuffix()) )
        elif Filename.exists(val):
            try: os.remove(val)
            except:
                raise OutputFileError( val, 'could not be removed before running. Argument %s' % (self.name()) )
            else:
                mess = 'Checked suffix %s and removed output file %s -> OK' % (self.fileSuffix(), val)
        else:
            mess = 'Checked suffix %s. No need to remove output file %s (not present)' % (self.fileSuffix(), val)

        self._logger.debug( mess )


    def postRunAction(self):
        """Check that output file exists."""
        val = self.value()
        mess = ''
        if val.upper() == 'NONE':
            mess = 'No output file expected. Nothing to be done.'
        elif not Filename.exists(val):
            raise OutputFileError( val, 'not found. Argument %s' % (self.name()) )
        elif not Filename.isRegular(val):
            raise OutputFileError( val, 'is not a regular file. Argument %s' % (self.name()) )
        else:
            mess = 'Outputfile %s found. -> OK' % (self.value())

        self._logger.debug( mess )
    

    def type(self):
        return 'outputDS'


    def metaType(self):
        return self.fileContents().upper()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.