Commits

Anonymous committed 03d4b2e Merge

sync w/ atlasoff

Comments (0)

Files changed (7)

 cc3bd35674d2c7c3525e23711a56b06e83fe4a04 PyUtils-00-11-24
 53f4d05a94225b5978c5bdf29078296271f47b02 PyUtils-00-12-00
 d990f327d4cc3fde92c1dead70e9593ff34c38a0 PyUtils-00-12-01
+9978dc06a3870d415eb12b9ecc0bdf637804c900 PyUtils-00-12-02
+46bf026d30011082caf82c6cd9e75591176bec04 PyUtils-00-12-03
+2012-04-02  Sebastien Binet  <binet@cern.ch>
+
+	* tagging PyUtils-00-12-03
+	* fix bug #93259: dont use multiprocessing but still fork
+	* also install diff-athfile into InstallArea
+	* M python/AthFile/_init_.py
+	* M python/AthFile/impl.py
+
+2012-03-30  Sebastien Binet  <binet@cern.ch>
+
+	* tagging PyUtils-00-12-02
+	* fix bug #92267: remove LD_PRELOAD from env for AthFile-subprocesses
+	* M python/AthFile/impl.py
+
 2012-03-30  Frank Winklmeier  <fwinkl@cern>
 
 	* tagging PyUtils-00-12-01
+#!/usr/bin/env python
+
+# @file:    diff-athfile.py
+# @purpose: simple command-line utility to diff metadata in two files.
+#           Uses PyUtils.AthFile.fopen. Based on dump-athfile.py.
+# @author:  Graeme Stewart <graeme.andrew.stewart@cern.ch>
+# @date:    Jan 2012
+# @version: $Id$
+#
+# @example:
+# @code
+# diff-athfile.py aod.1.pool.root aod.2.pool.root 
+# @endcode
+#
+
+import sys
+import os
+
+try:                import cPickle as pickle
+except ImportError: import pickle
+    
+from optparse import OptionParser
+
+if __name__ == "__main__":
+
+    parser = OptionParser(usage="usage: %prog [options] my.1.file my.2.file")
+    parser.add_option( "-o",
+                       "--output",
+                       dest = "oname",
+                       default = 'athfile-infos.ascii',
+                       help = "Name of the output file which will contain the informations gathered during AthFile processing. These informations will be stored into a python-shelve file." )
+    parser.add_option( '--evtmax',
+                       dest = 'evtmax',
+                       default = 1,
+                       type = int,
+                       help = 'Maximum number of events to process in the file(s)')
+    parser.add_option( "-q",
+                       "--quiet",
+                       dest = "quiet",
+                       default = False,
+                       action = "store_true",
+                       help = 'Quiet output - only print differences')
+    
+    (options, args) = parser.parse_args()
+
+    fnames = []
+    
+    if len(args) > 0:
+        fnames = [ arg for arg in args if arg[0] != "-" ]
+        pass
+
+    if len(fnames) != 2:
+        str(parser.print_help() or "")
+        sys.exit(1)
+
+    sc = 0
+    import PyUtils.AthFile as af
+    msg = af.msg
+    if options.quiet:
+        from logging import WARNING
+        msg.setLevel(WARNING)
+    
+    fhandles = {}
+    try:
+        for fname in fnames:
+            fhandles[fname] = af.fopen(fname, evtmax=options.evtmax)
+            msg.info(':'*80)
+            msg.info('Opened file %s.' % fname)
+            
+
+        # Ignore the following keys, which are bound to be different:
+        # 'file_md5sum', 'file_name', 'file_type', 'file_guid',
+        simpleCompKeys = ('nentries','run_number', 'run_type', 'evt_number', 'evt_type', 'lumi_block',
+                        'beam_energy', 'beam_type',
+                        'stream_tags', 'stream_names', 'geometry', 'conditions_tag',)
+        bitByBitKeys = ('metadata',)
+    
+        for k in simpleCompKeys:
+            if fhandles[fnames[0]].infos[k] == fhandles[fnames[1]].infos[k]:
+                msg.info('%s equal in %s and %s: %s' % (k, fnames[0], fnames[1], fhandles[fnames[0]].infos[k]))
+            else:
+                msg.warning('%s not equal in %s and %s: %s != %s' % 
+                            (k, fnames[0], fnames[1], fhandles[fnames[0]].infos[k], fhandles[fnames[1]].infos[k]))
+                sc = 1
+
+        for k in bitByBitKeys:
+            # First check dictionary keys are the same
+            skeys = fhandles[fnames[0]].infos[k].keys()
+            skeys1 = fhandles[fnames[1]].infos[k].keys()
+            skeys.sort()
+            skeys1.sort()
+            if skeys != skeys1:
+                msg.warning('%s keys not equal for %s and %s: %s != %s' % 
+                            (k, fnames[0], fnames[1], skeys, skeys1))
+                sc = 1
+            else:
+                msg.info('%s keys are equal for %s and %s: %s' % 
+                         (k, fnames[0], fnames[1], skeys))
+                for subk in skeys:
+                    if fhandles[fnames[0]].infos[k][subk] == fhandles[fnames[1]].infos[k][subk]:
+                        # Here suppress the very long value output
+                        msg.info('%s element %s values are equal for %s and %s: (value suppressed)' % 
+                                 (k, subk, fnames[0], fnames[1]))
+                    else:
+                        msg.warning('%s element %s values are not equal for %s and %s: %s != %s' % 
+                                 (k, subk, fnames[0], fnames[1], fhandles[fnames[0]].infos[k][subk], 
+                                  fhandles[fnames[1]].infos[k][subk]))
+                        sc = 1
+                        
+                
+        msg.info(':'*80)
+    except Exception, e:
+        msg.error("Caught exception [%s] !!", str(e.__class__))
+        msg.error("What:\n%s\n%s\n%s",e,
+                  sys.exc_info()[0],
+                  sys.exc_info()[1])
+        sc = 2
+        pass
+
+    except :
+        msg.error("Caught something !! (don't know what)")
+        msg.error("\n%s\n%s",sys.exc_info()[0], sys.exc_info()[1])
+        sc = 2
+        pass
+
+    
+    if options.oname:
+        oname = options.oname
+        msg.info("saving report into [%s]..." % oname)
+        if os.path.exists(oname):
+            os.rename(oname, oname+'.bak')
+        af.server.save_cache(oname)
+
+    msg.info("Bye.")
+    sys.exit(sc)

bin/diff-athfile.py

-#!/usr/bin/env python
-
-# @file:    diff-athfile.py
-# @purpose: simple command-line utility to diff metadata in two files.
-#           Uses PyUtils.AthFile.fopen. Based on dump-athfile.py.
-# @author:  Graeme Stewart <graeme.andrew.stewart@cern.ch>
-# @date:    Jan 2012
-# @version: $Id$
-#
-# @example:
-# @code
-# diff-athfile.py aod.1.pool.root aod.2.pool.root 
-# @endcode
-#
-
-import sys
-import os
-
-try:                import cPickle as pickle
-except ImportError: import pickle
-    
-from optparse import OptionParser
-
-if __name__ == "__main__":
-
-    parser = OptionParser(usage="usage: %prog [options] my.1.file my.2.file")
-    parser.add_option( "-o",
-                       "--output",
-                       dest = "oname",
-                       default = 'athfile-infos.ascii',
-                       help = "Name of the output file which will contain the informations gathered during AthFile processing. These informations will be stored into a python-shelve file." )
-    parser.add_option( '--evtmax',
-                       dest = 'evtmax',
-                       default = 1,
-                       type = int,
-                       help = 'Maximum number of events to process in the file(s)')
-    parser.add_option( "-q",
-                       "--quiet",
-                       dest = "quiet",
-                       default = False,
-                       action = "store_true",
-                       help = 'Quiet output - only print differences')
-    
-    (options, args) = parser.parse_args()
-
-    fnames = []
-    
-    if len(args) > 0:
-        fnames = [ arg for arg in args if arg[0] != "-" ]
-        pass
-
-    if len(fnames) != 2:
-        str(parser.print_help() or "")
-        sys.exit(1)
-
-    sc = 0
-    import PyUtils.AthFile as af
-    msg = af.msg
-    if options.quiet:
-        from logging import WARNING
-        msg.setLevel(WARNING)
-    
-    fhandles = {}
-    try:
-        for fname in fnames:
-            fhandles[fname] = af.fopen(fname, evtmax=options.evtmax)
-            msg.info(':'*80)
-            msg.info('Opened file %s.' % fname)
-            
-
-        # Ignore the following keys, which are bound to be different:
-        # 'file_md5sum', 'file_name', 'file_type', 'file_guid',
-        simpleCompKeys = ('nentries','run_number', 'run_type', 'evt_number', 'evt_type', 'lumi_block',
-                        'beam_energy', 'beam_type',
-                        'stream_tags', 'stream_names', 'geometry', 'conditions_tag',)
-        bitByBitKeys = ('metadata',)
-    
-        for k in simpleCompKeys:
-            if fhandles[fnames[0]].infos[k] == fhandles[fnames[1]].infos[k]:
-                msg.info('%s equal in %s and %s: %s' % (k, fnames[0], fnames[1], fhandles[fnames[0]].infos[k]))
-            else:
-                msg.warning('%s not equal in %s and %s: %s != %s' % 
-                            (k, fnames[0], fnames[1], fhandles[fnames[0]].infos[k], fhandles[fnames[1]].infos[k]))
-                sc = 1
-
-        for k in bitByBitKeys:
-            # First check dictionary keys are the same
-            skeys = fhandles[fnames[0]].infos[k].keys()
-            skeys1 = fhandles[fnames[1]].infos[k].keys()
-            skeys.sort()
-            skeys1.sort()
-            if skeys != skeys1:
-                msg.warning('%s keys not equal for %s and %s: %s != %s' % 
-                            (k, fnames[0], fnames[1], skeys, skeys1))
-                sc = 1
-            else:
-                msg.info('%s keys are equal for %s and %s: %s' % 
-                         (k, fnames[0], fnames[1], skeys))
-                for subk in skeys:
-                    if fhandles[fnames[0]].infos[k][subk] == fhandles[fnames[1]].infos[k][subk]:
-                        # Here suppress the very long value output
-                        msg.info('%s element %s values are equal for %s and %s: (value suppressed)' % 
-                                 (k, subk, fnames[0], fnames[1]))
-                    else:
-                        msg.warning('%s element %s values are not equal for %s and %s: %s != %s' % 
-                                 (k, subk, fnames[0], fnames[1], fhandles[fnames[0]].infos[k][subk], 
-                                  fhandles[fnames[1]].infos[k][subk]))
-                        sc = 1
-                        
-                
-        msg.info(':'*80)
-    except Exception, e:
-        msg.error("Caught exception [%s] !!", str(e.__class__))
-        msg.error("What:\n%s\n%s\n%s",e,
-                  sys.exc_info()[0],
-                  sys.exc_info()[1])
-        sc = 2
-        pass
-
-    except :
-        msg.error("Caught something !! (don't know what)")
-        msg.error("\n%s\n%s",sys.exc_info()[0], sys.exc_info()[1])
-        sc = 2
-        pass
-
-    
-    if options.oname:
-        oname = options.oname
-        msg.info("saving report into [%s]..." % oname)
-        if os.path.exists(oname):
-            os.rename(oname, oname+'.bak')
-        af.server.save_cache(oname)
-
-    msg.info("Bye.")
-    sys.exit(sc)
  checkTP.py \
  checkTag.py \
  cmtClients.py \
+ diff-athfile \
  diff-jobo-cfg.py \
  diffConfigs.py \
  diffPoolFiles.py \

python/AthFile/__init__.py

     @property
     def tests(self):
         return self._tests
-    
+
+    @_decos.forking
     def fopen(self, fnames, evtmax=1):
         """
         helper function to create @c AthFile instances

python/AthFile/impl.py

 import subprocess
 import sys
 
+import PyUtils.Decorators as _decos
 import PyUtils.Helpers as H
 from PyUtils.Helpers    import ShutUp
 from .timerdecorator import timelimit, TimeoutError
         import PyUtils.Logging as _L
         #self._msg = msg
         #self._msg = self._manager.msg()
-        self._msg = _get_msg()#_L.logging.getLogger("AthFile")
+        #self._msg = _get_msg()#_L.logging.getLogger("AthFile")
         #self._msg = mp.get_logger("AthFile")
         #self._msg.setFormat("Py:%(name)-14s%(levelname)8s %(message)s")
 
         try:
             ru._pythonize_tfile()
         except Exception, err:
-            self._msg.warning('problem during TFile pythonization:\n%s', err)
+            self.msg().warning('problem during TFile pythonization:\n%s', err)
             
         #self.msg.info('importing ROOT... [done]')
 
                 self._msg.info('could not close a TFile:\n%s', err)
                 pass
         tfiles[:] = []
-        
+
     def msg(self):
-        return self._msg
+        return _get_msg()#self._msg
     
     def set_msg_lvl(self, lvl):
-        self._msg.setLevel(lvl)
+        self.msg().setLevel(lvl)
         
     def _md5_for_file(self, f, block_size=2**20, do_fast_md5=True):
         """helper function to calculate a MD5 checksum
         md5 = self._md5_for_file(fname)
         return md5
     
+    @_decos.forking
+    @timelimit(timeout=DEFAULT_AF_TIMEOUT)
     def fname(self, fname):
         """take a file name, return the pair (protocol, 'real' file name)
         """
             msg.debug('reason:\n%s', err)
             pass
         return
-    
-    @timelimit(timeout=5)
+
+    # dead-lock on self.msg (I think!)...
+    #@timelimit(timeout=DEFAULT_AF_TIMEOUT)
     def load_cache(self, fname=DEFAULT_AF_CACHE_FNAME):
         """load file informations from a cache file.
         the back-end (JSON, ASCII, pickle, ...) is inferred from the
             # illegal file...
             msg.info('load_cache: invalid file [%s]', fname)
             return
+
         ext = ext[1:] if ext[0]=='.' else ext
         try:
             loader = getattr(self, '_load_%s_cache'%ext)
             msg.info('load_cache: could not find a suitable backend for '
                      'extension [.%s] => using [ascii]', ext)
             loader = self._load_ascii_cache
+
         try:
             search_path = os.environ.get('DATAPATH',os.getcwd())
             search_path = search_path.split(os.pathsep)
         except Exception, err:
             msg.info("problem loading cache from [%s]!", fname)
             msg.info(repr(err))
+            pass
+        
         self._cache.update(cache)
         msg.debug('loading cache from [%s]... [done]', fname)
 
         self._cache = {}
         return
 
+    @_decos.forking
+    @timelimit(timeout=DEFAULT_AF_TIMEOUT)
     def ftype(self, fname):
         """
         returns the type of a file ('pool' or 'bs') together with its
         ftype = 'pool' if _is_root_file else 'bs'
         return (ftype, fname)
 
+    @_decos.forking
+    @timelimit(timeout=DEFAULT_AF_TIMEOUT)
     def exists(self, fname):
         """helper function to test if a fiven `fname` exists.
 
         if 'PYTHONINSPECT' in self._sub_env:
             del self._sub_env['PYTHONINSPECT']
 
+        # prevent from running athena with igprof
+        for k in ('LD_PRELOAD', 'IGPROF'):
+            if k in self._sub_env:
+                del self._sub_env[k]
+
     def _root_open(self, fname, raw=False):
         import PyUtils.Helpers as H
         with H.restricted_ldenviron(projects=['AtlasCore']):
                 re.compile("Error in <T.*?File::Init>:.*? not a ROOT file")]):
                 # for AttributeListLayout which uses CINT for its dict...
                 # first try the APR version
-                if 0 != root.gSystem.Load('libRootCollection'):
+                ooo = root.gSystem.Load('libRootCollection')
+                if ooo < 0:
                     # then try the POOL one
                     root.gSystem.Load('liblcg_RootCollection')
                 if raw:
                        file_name,]
                 subprocess.call(cmd, env=self._sub_env)
                 #
-                with H.restricted_ldenviron(projects=projects):
+                with H.restricted_ldenviron(projects=None):
                     is_tag, tag_ref, tag_guid, nentries, runs, evts = self._is_tag_file(f_root, evtmax)
                     if is_tag:
                         f['stream_names'] = ['TAG']
                             }
                         import os
                         stdout = open('athfile-%i.log.txt' % os.getpid(), "a")
+                        print >> stdout,"="*80
+                        print >> stdout,self._sub_env
+                        print >> stdout,"="*80
+                        stdout.flush()
                         sc = app.run(stdout=stdout, env=self._sub_env)
+                        stdout.flush()
                         stdout.close()
                         import AthenaCommon.ExitCodes as ath_codes
                         if sc == 0:
                                 raise IOError(sc, err)
                             msg.info('athena failed to initialize.')
                             msg.info('=> probably an empty input POOL file')
-                        if os.path.exists(out_pkl_fname):
-                            os.remove(out_pkl_fname)
+                        ## if os.path.exists(out_pkl_fname):
+                        ##     os.remove(out_pkl_fname)
                     # TAG-file
                     # app.exit()
             else: # bytestream