Commits

bi...@4525493e-7705-40b1-a816-d608a930855b  committed 9f95737

speed-optimizations of py-retrieve and py-record

  • Participants
  • Parent commits d452939

Comments (0)

Files changed (4)

+2008-03-21  Sebastien Binet  <binet@lblbox>
+
+	* tagging StoreGateBindings-00-01-00
+	* speed-optimizations of py-retrieve and py-record
+	* M python/Bindings.py
+	* M src/StoreGateBindingsDict.h
+	* M src/StoreGatePyExt.cxx
+
 2008-01-16  Sebastien Binet  <binet@lblbox>
 
 	* tagging StoreGateBindings-00-00-04

File python/Bindings.py

 # @author: Sebastien Binet <binet@cern.ch>
 
 ### data
-__version__ = "$Revision: 1.1 $"
+__version__ = "$Revision: 1.2 $"
 __author__  = """
 Wim Lavrijsen (WLavrijsen@lbl.gov),
 Sebastien Binet (binet@cern.ch)
 """
 
+from PyUtils.Decorators import memoize
+
 ### pythonizations for StoreGateSvc
 def _setup():
     import PyCintex
     # StoreGate bindings from dictionary
-    PyCintex.loadDictionary( "libStoreGateBindingsDict" )
+    PyCintex.loadDictionary( "libAthenaPythonDict" )      # for clidsvc
+    PyCintex.loadDictionary( "libStoreGateBindingsDict" ) # for storegatesvc
 
     # make sure the global C++ namespace has been created
     gbl = PyCintex.makeNamespace('')
+    _ath= PyCintex.makeNamespace('AthenaInternal')
     
     global py_retrieve
     py_retrieve = PyCintex.gbl.AthenaInternal.retrieveObjectFromStore
     StoreGateSvc = PyCintex.gbl.StoreGateSvc
     StoreGate    = PyCintex.gbl.StoreGate
 
+    from AthenaPython.Pythonizations import py_svc
+
+    @memoize
+    def _get_klass_infos(klass):
+        if isinstance(klass, str):
+            klass = PyCintex.makeClass(klass)
+        klassname = klass.__class__.__name__
+        if klassname.endswith('_meta'): klassname = klassname[:-5]
+        clidSvc = py_svc('ClassIDSvc', iface='IClassIDSvc')
+        clid = clidSvc.clid(klassname)
+        tname= clidSvc.typename(clid)
+        return clid, tname
+
+    # caching
+    _makeNullPtr = PyCintex.libPyROOT.MakeNullPointer
+    
     # add specialized retrieve method
     def retrieve( self, klass, key = None ):
-        ptr = PyCintex.libPyROOT.MakeNullPointer( klass )
-        py_retrieve( self, klass, key, ptr )
-        return ptr
+        return py_retrieve( self, klass, key )
     StoreGateSvc.retrieve = retrieve
 
     # add specialized record method
     def record( self, obj, key ):
-        py_record( self, obj, key )
-        return
+        return py_record( self, obj, key )
     StoreGateSvc.record = record
 
     # make dump print, rather than return string

File src/StoreGateBindingsDict.h

 
 namespace AthenaInternal {
 
-  PyObject* retrieveObjectFromStore( StoreGateSvc*, 
-				     PyObject* tp, PyObject* key, void*& res );
+  PyObject* retrieveObjectFromStore( PyObject* storeGateSvc, 
+ 				     PyObject* tp, PyObject* key );
 
-  PyObject* recordObjectToStore( StoreGateSvc*, 
+  PyObject* recordObjectToStore( PyObject* storeGateSvc, 
 				 PyObject* obj, PyObject* key );
 
   PyObject* thinContainer( IThinningSvc*,

File src/StoreGatePyExt.cxx

 #include PYTHON_INC
 
 #include "StoreGateBindingsDict.h"
-#include "SGTools/DataBucketBase.h"
+
+// Framework includes
 #include "GaudiKernel/Bootstrap.h"
 #include "GaudiKernel/DataObject.h"
 #include "AthenaKernel/IClassIDSvc.h"
 #include "AthenaKernel/IThinningSvc.h"
 
+// SGTools includes
+#include "SGTools/unordered_map.h" // move to STL once available
+#include "SGTools/DataBucketBase.h"
+
 // Reflex includes
 #include "Reflex/Type.h"
 #include "Reflex/Object.h"
 #include "TROOT.h"
 #include "Api.h"
 
+// PyROOT includes
+#include "TPython.h"
+#include "TPyException.h"
+
 #include <string>
 #include <vector>
 #include <algorithm> // for stable_partition
 // temporaryly duplicate PyROOT's API...
 // Wim will fix that up
 
+#define ObjectProxy_ASVOIDPTR(o) (((PyROOT::ObjectProxy*)(o))->GetObject())
+
 namespace PyROOT {
 
 class ObjectProxy {
     /// The class ID of the wrapped object.
     CLID m_clid;
 
-    /// Pointer to the @ IClassIDSvc to be able to cast objects based
+    /// Pointer to the @ IClassIDSvc to be able to cast objects based on
     /// the @a clid.
     static IClassIDSvc* s_clidSvc;
 
      */
     bytes_t& m_container;
   };
+
+  /** @brief a python front-end to the @c IProxyDict interface, with caching
+   *         the type of held object by the proxy (for `class` look-up issues)
+   *  A @c SG::PyProxyDict holds a dictionary of { (clid,sgkey) : pyobj }
+   */
+  struct PyProxyDict
+  {
+    StoreGateSvc* m_sgSvc;
+    PyObject* m_sgProxies;
+    PyObject* m_objProxies;
+
+    PyProxyDict( StoreGateSvc* sgSvc ) : 
+      m_sgSvc     ( sgSvc ),
+      m_sgProxies ( PyDict_New() ),
+      m_objProxies( PyDict_New() )
+    {}
+    ~PyProxyDict() 
+    { 
+      Py_DECREF(m_sgProxies ); 
+      Py_DECREF(m_objProxies);
+    }
+
+    inline
+    PyObject* proxy( PyObject* pyclid, PyObject* pykey )
+    {
+      PyObject* k = PyTuple_Pack(2, pyclid, pykey);
+      PyObject* pyproxy = PyDict_GetItem(m_sgProxies, k);
+      if ( NULL == pyproxy ) {
+	CLID id = CLID_NULL;
+	PyArg_Parse( pyclid, "l", &id );
+	const std::string skey = ( pykey == Py_None )
+	  ? ""
+	  : PyString_AS_STRING(pykey);
+	SG::DataProxy* proxy = skey.empty()
+	  ? m_sgSvc->proxy(id)
+	  : m_sgSvc->proxy(id, skey);
+	pyproxy = TPython::ObjectProxy_FromVoidPtr((void*)proxy, 
+						   "SG::DataProxy");
+	PyDict_SetItem(m_sgProxies, k, pyproxy);
+      }
+      Py_DECREF(k);
+      return pyproxy;
+    }
+
+    inline
+    PyObject* dataObject( PyObject* proxy, const char* klass, void* addr=0 )
+    {
+      PyObject* obj = PyDict_GetItem( m_objProxies, proxy );
+      if ( NULL == obj ) {
+	if ( !(obj = TPython::ObjectProxy_FromVoidPtr((void*)addr, klass)) ) {
+	  throw PyROOT::TPyException();
+	}
+	PyDict_SetItem(m_objProxies, proxy, obj);
+      }
+      if ( addr && obj ) {
+	((PyROOT::ObjectProxy*)obj)->fObject = addr;
+      }
+      return obj;
+    }
+  };
+
+  /**
+   * @brief A helper class to manage accesses to PyProxies
+   */
+  struct PyProxyMgr
+  {
+    IClassIDSvc* m_clidSvc;
+    /// a dictionary of 'typename' -> CLID
+    PyObject* m_clids;
+
+    typedef SG::unordered_map<StoreGateSvc*,SG::PyProxyDict*> PyProxyMap_t;
+    PyProxyMap_t m_proxyMap;
+
+    static
+    PyProxyMgr& instance()
+    { static PyProxyMgr mgr; return mgr; }
+
+    PyProxyMgr()
+    {
+      m_clids   = PyDict_New();
+      m_clidSvc = 0;
+      if ( !Gaudi::svcLocator()->getService("ClassIDSvc", 
+					    (IService*&)m_clidSvc)
+	   .isSuccess() ) {
+	throw std::runtime_error
+	  ("SG::PyProxyMgr: Could not retrieve ClassIDSvc");
+      }
+    }
+
+    ~PyProxyMgr() 
+    {
+      Py_DECREF(m_clids);
+      // delete the proxy dicts...
+      for ( PyProxyMap_t::iterator 
+	      i    = m_proxyMap.begin(),
+	      iEnd = m_proxyMap.end();
+	    i != iEnd;
+	    ++i ) {
+	delete i->second; i->second = 0;
+      }
+    }
+
+    inline
+    SG::PyProxyDict* pyproxy( StoreGateSvc* sg )
+    {
+      SG::PyProxyDict* p = 0;
+      PyProxyMap_t::iterator i = m_proxyMap.find(sg);
+      if ( i == m_proxyMap.end() ) {
+	m_proxyMap[sg] = p = new SG::PyProxyDict(sg);
+      } else {
+	p = i->second;
+      }
+      return p;
+    }
+
+    /// returns a borrowed reference
+    inline
+    PyObject* pyclid(PyObject* tp)
+    {
+      PyObject* clid = PyDict_GetItem(m_clids, tp);
+      if ( NULL == clid ) {
+	const CLID id = this->clid(tp);
+	if ( id == CLID_NULL ) {
+	  return NULL;
+	}
+	clid = PyLong_FromLong(id);
+	PyDict_SetItem(m_clids, tp, clid);
+	//Py_INCREF(clid);
+      }
+      return clid;
+    }
+
+    inline
+    CLID clid(PyObject* tp)
+    {
+      CLID id = CLID_NULL;
+      m_clidSvc->getIDOfTypeName(PyString_AS_STRING(tp), id).ignore();
+      return id;
+    }
+
+  };
+
 } //> end namespace SG
 
 PyObject* 
-AthenaInternal::retrieveObjectFromStore( StoreGateSvc* store, 
-					 PyObject* tp, PyObject* key, 
-					 void*& res )
+AthenaInternal::retrieveObjectFromStore( PyObject* self, 
+					 PyObject* tp, PyObject* pykey )
 {
-   static IClassIDSvc* s_clidsvc = 0;
+  void* res = 0;
+  PyObject* objProxy = NULL;
+
+  static SG::PyProxyMgr& s_mgr = SG::PyProxyMgr::instance();
+
+  StoreGateSvc* store = (StoreGateSvc*)ObjectProxy_ASVOIDPTR(self);
 
 // unlikely to happen, but checking is cheap
-   if ( ! store ) {
-      PyErr_SetString( PyExc_RuntimeError, "no store available, is Athena initialized?" );
-      return 0;
-   }
+  if ( ! store ) {
+    PyErr_SetString( PyExc_RuntimeError, 
+		     "no store available, is Athena initialized?" );
+    return 0;
+  }
 
 // expect a type or type name and an optional string key
-   PyObject* pyname = 0;
-   if ( ! PyType_Check( tp ) ) {
-      if ( ! PyString_Check( tp ) ) {
-         PyErr_SetString( PyExc_TypeError, "retrieve() argument 1 must be type or class name" );
-         return 0;
-      } else {
-         Py_INCREF( tp );
-         pyname = tp;
+  PyObject* pyname = 0;
+
+  if ( ! PyType_Check( tp ) ) {
+    if ( ! PyString_Check( tp ) ) {
+      PyErr_SetString( PyExc_TypeError, 
+		       "retrieve() argument 1 must be type or class name" );
+      return 0;
+    } else {
+      Py_INCREF( tp );
+      pyname = tp;
+    }
+  } else {
+    pyname = PyObject_GetAttrString( tp, (char*)"__name__" );
+    if ( pyname && ! PyString_Check( pyname ) ) {
+      PyObject* pystr = PyObject_Str( pyname );
+      if ( pystr ) {
+	Py_DECREF( pyname );
+	pyname = pystr;
       }
-   } else {
-      pyname = PyObject_GetAttrString( tp, (char*)"__name__" );
-      if ( pyname && ! PyString_Check( pyname ) ) {
-         PyObject* pystr = PyObject_Str( pyname );
-         if ( pystr ) {
-            Py_DECREF( pyname );
-            pyname = pystr;
-         }
-      }
+    }
 
-      if ( PyErr_Occurred() )
-         return 0;
-   }
+    if ( PyErr_Occurred() )
+      return 0;
+  }
 
-   if ( key != Py_None && ! PyString_Check( key ) ) {
-      PyErr_SetString( PyExc_TypeError, "retrieve() argument 2 must be string key" );
-      return 0;
-   }
+  if ( pykey != Py_None && ! PyString_Check( pykey ) ) {
+    PyErr_SetString( PyExc_TypeError, 
+		     "retrieve() argument 2 must be string key" );
+    return 0;
+  }
 
+  SG::PyProxyDict* proxyDict = s_mgr.pyproxy(store);
 // retrieve CLID corresponding to the request
-   if ( ! s_clidsvc ) {
-      StatusCode sc = Gaudi::svcLocator()->getService( "ClassIDSvc", (IService*&)s_clidsvc );
-      if ( sc.isFailure() ) {
-         PyErr_SetString( PyExc_RuntimeError, "ClassIDSvc does not exist" );
-         return 0;
-      }
-   }
+  PyObject* pyclid = s_mgr.pyclid(pyname);
+  if ( ! pyclid ) {
+    PyErr_Format( PyExc_NameError, 
+		  "ID of \"%s\" is unknown", PyString_AS_STRING( pyname ) );
+    return 0;
+  }
 
-   std::string name = PyString_AS_STRING( pyname );
-   CLID id = CLID_NULL;
-   StatusCode sc = s_clidsvc->getIDOfTypeName( name, id );
-   if ( sc.isFailure() ) {
-      PyErr_Format( PyExc_NameError, "ID of %s is unknown", name.c_str() );
-      return 0;
-   }
+  PyObject* pyproxy = proxyDict->proxy(pyclid, pykey);
+  if ( ! pyproxy ) {
+    PyErr_Format( PyExc_LookupError, 
+		  "no py-proxies for (clid=%lu, type=%s, key=%s)", 
+		  PyLong_AsUnsignedLong(pyclid),
+		  PyString_AS_STRING( pyname ),
+		  (char*)( (pykey == Py_None) 
+			   ? "<None>" 
+			   : PyString_AS_STRING(pykey) )
+		  );
+    return 0;
+  }
 
-   SG::DataProxy* proxy = 0;
-   if ( key != Py_None ) {
-      std::string skey = PyString_AS_STRING( key );
-      proxy = store->proxy( id, skey );
-   } else {
-      proxy = store->proxy( id );
-   }
+  SG::DataProxy* proxy = (SG::DataProxy*)ObjectProxy_ASVOIDPTR(pyproxy);
 
-   if ( ! proxy ) {
-      PyErr_Format( PyExc_LookupError, "no proxies for \"%s\"", name.c_str() );
-      return 0;
-   }
+  if ( ! proxy ) {
+    PyErr_Format( PyExc_LookupError, 
+		  "no proxies for (clid=%lu, type=%s, key=%s)", 
+		  PyLong_AsUnsignedLong(pyclid),
+		  PyString_AS_STRING( pyname ),
+		  (char*)( (pykey == Py_None) 
+			   ? "<None>" 
+			   : PyString_AS_STRING(pykey) )
+		  );
+    return 0;
+  }
 
 // cast proxy to pointer type if needed (setting on return type is evil hack)
-   DataObject* dobj = proxy->accessData();
-   if ( ! dobj ) {
-      PyErr_Format( PyExc_LookupError, "no such object \"%s\"", name.c_str() );
+  DataObject* dobj = proxy->accessData();
+  if ( ! dobj ) {
+    PyErr_Format( PyExc_LookupError, 
+		  "no such object \"%s\"", PyString_AS_STRING( pyname ) );
+    return 0;
+  }
+  
+  DataBucketBase* dbb = dynamic_cast< DataBucketBase* >( dobj );
+  if ( ! dbb ) {
+    PyErr_SetString
+      ( PyExc_TypeError,
+	"unexpected kind of DataObject: can not verify final type" );
+    return 0;
+  }
+
+  CLID id     = proxy->clID();
+  CLID realID = dbb->clID();
+  if ( id == realID ) {
+    res = dbb->object();
+    
+    if ( ! res ) {
+      PyErr_Format( PyExc_RuntimeError, "found an invalid object" );
       return 0;
-   }
+    }
+    
+    objProxy = proxyDict->dataObject(pyproxy, PyString_AS_STRING(pyname), res);
 
-   DataBucketBase* dbb = dynamic_cast< DataBucketBase* >( dobj );
-   if ( ! dbb ) {
-      PyErr_SetString( PyExc_TypeError,
-         "unexpected kind of DataObject: can not verify final type" );
+  } else {
+    // either use CLID BaseInfo<> or Reflex, try both as appropriate
+    res = dbb->cast( id );
+    
+    if ( res ) {
+
+      objProxy= proxyDict->dataObject(pyproxy, PyString_AS_STRING(pyname), res);
+    } else {
+      // -> try Reflex...
+      IClassIDSvc* clidSvc = s_mgr.m_clidSvc;
+      std::string realName = "";
+      if ( !clidSvc->getTypeNameOfID(realID, realName).isSuccess() ) {
+	PyErr_Format( PyExc_TypeError, "actual type of CLID %lu unknown",
+		      (long unsigned int)realID );
+	return 0;
+      }
+      
+      const ROOT::Reflex::Type& fromType = ROOT::Reflex::Type::ByName(realName);
+      
+      if ( (bool)fromType ) {
+	ROOT::Reflex::Object realobj( fromType, dbb->object() );
+	
+ 	const ROOT::Reflex::Type& toType = ROOT::Reflex::Type::ByName( PyString_AS_STRING( pyname ) );
+ 	const ROOT::Reflex::Object& finalobj = fromType.CastObject( toType, realobj );
+	
+ 	res = (void*)finalobj.Address();
+ 	if ( res ) {
+ 	  objProxy = proxyDict->dataObject(pyproxy, realName.c_str(), res);
+ 	}
+      }
+    }
+
+    if ( ! res ) {
+      PyErr_SetString( PyExc_TypeError, "cast to requested type failed" );
       return 0;
-   }
+    }
 
-   CLID realID = dbb->clID();
-   if ( id == realID ) {
-      res = dbb->object();
+  }
 
-      if ( ! res ) {
-         PyErr_Format( PyExc_RuntimeError, "found an invalid object" );
-         return 0;
-      }
-
-   } else {
-   // either use CLID BaseInfo<> or Reflex, try both as appropriate
-      res = dbb->cast( id );
-
-      if ( ! res ) {
-         std::string realName;
-         StatusCode sc = s_clidsvc->getTypeNameOfID( realID, name );
-         if ( sc.isFailure() ) {
-            PyErr_Format( PyExc_TypeError, "actual type of CLID %lu unknown",
-                          (long unsigned int)realID );
-            return 0;
-         }
-
-         const ROOT::Reflex::Type& fromType = ROOT::Reflex::Type::ByName( realName );
-
-         if ( (bool)fromType ) {
-            ROOT::Reflex::Object realobj( fromType, dbb->object() );
-
-            const ROOT::Reflex::Type& toType = ROOT::Reflex::Type::ByName( name );
-            const ROOT::Reflex::Object& finalobj = fromType.CastObject( toType, realobj );
-
-            res = (void*)finalobj.Address();
-         }
-
-      }
-
-      if ( ! res ) {
-         PyErr_SetString( PyExc_TypeError, "cast to requested type failed" );
-         return 0;
-      }
-
-   }
-
-   Py_INCREF( Py_None );
-   return Py_None;
+  Py_INCREF(objProxy);
+  return objProxy;
 }
 
 PyObject* 
-AthenaInternal::recordObjectToStore( StoreGateSvc* store, 
-				     PyObject* obj, PyObject* key )
+AthenaInternal::recordObjectToStore( PyObject* self,
+				     PyObject* obj, PyObject* pykey )
 {
-  static IClassIDSvc* s_clidsvc = 0;
+  static SG::PyProxyMgr& s_mgr = SG::PyProxyMgr::instance();
 
-  // unlikely to happen, but checking is cheap
-  if ( !store ) {
-    PyErr_SetString( PyExc_RuntimeError, "no store available, is Athena initialized?" );
+  StoreGateSvc* store = (StoreGateSvc*)ObjectProxy_ASVOIDPTR(self);
+
+// unlikely to happen, but checking is cheap
+  if ( ! store ) {
+    PyErr_SetString( PyExc_RuntimeError, 
+		     "no store available, is Athena initialized?" );
     return 0;
   }
-  
+
   // expect a type or type name
   PyObject* tp = PyObject_GetAttrString( obj, (char*)"__class__" );
   if ( ! PyType_Check( tp ) ) {
   if ( PyErr_Occurred() )
     return 0;
     
-  if ( ! PyString_Check( key ) ) {
+  if ( ! PyString_Check( pykey ) ) {
     PyErr_SetString( PyExc_TypeError, 
 		     "record() argument 2 must be string key" );
     return 0;
   }
   
   // retrieve CLID corresponding to the request
-  if ( ! s_clidsvc ) {
-    StatusCode sc = Gaudi::svcLocator()->getService( "ClassIDSvc", (IService*&)s_clidsvc );
-    if ( sc.isFailure() ) {
-      PyErr_SetString( PyExc_RuntimeError, "ClassIDSvc does not exist" );
-      return 0;
-    }
-  }
-  
-  std::string name = PyString_AS_STRING( pyname );
-  CLID id = CLID_NULL;
-  if ( s_clidsvc->getIDOfTypeName( name, id ).isFailure() ) {
-    PyErr_Format( PyExc_NameError, "ID of %s is unknown", name.c_str() );
+  const CLID id = s_mgr.clid(pyname);
+  if ( CLID_NULL == id ) {
+    PyErr_Format( PyExc_NameError, 
+		  "ID of \"%s\" is unknown", PyString_AS_STRING( pyname ) );
     return 0;
   }
   
   PyROOT::ObjectProxy* pyRootObj = (PyROOT::ObjectProxy*)obj;
   DataObject* dataObj = new SG::PyDataBucket( pyRootObj, id );
   static bool allowMods = true;
-  int sc = store->typeless_record( dataObj, PyString_AS_STRING(key),
+  int sc = store->typeless_record( dataObj, PyString_AS_STRING(pykey),
 				   pyRootObj->GetObject(),
 				   allowMods ).isSuccess()
     ? 1