Commits

Gaël Le Mignot committed 620fa82

Using savepoints for better handling of concurrency

  • Participants
  • Parent commits eb8e578
  • Tags VERSION_0_13_0

Comments (0)

Files changed (8)

File debian/changelog

+sesql (0.13.0-1) unstable; urgency=low
+
+  * Using savepoints for handling transactions in a smarter way.
+
+ -- Gael Le Mignot <gael@pilotsystems.net>  Thu, 01 Mar 2012 15:51:47 +0100
+
 sesql (0.12.0-1) unstable; urgency=low
 
   * Added an option to dereference proxy models in Django.

File doc/extra.txt

    --index-type=Article
 
 
+Concurrency handling
+++++++++++++++++++++
 
+Since  version  0.13  SeSQL  uses  savepoints  to  support  concurrent
+indexing. You  can use SeSQL  from a multithreaded program,  and index
+the same  object from several threads  at once, SeSQL will  detect the
+conflicts and retry them (default is 3 retries).
 
-
+If  you have  very heavy  concurrency, you  may prefer  to handle  the
+indexing  asynchronously,  in a  separate  daemon  that'll handle  the
+indexing sequentially. The  same daemon than the  one for dependencies
+tracking   can    be   used    for   that,    if   you    enable   the
+``ASYNCHRONOUS_INDEXING`` in the configuration file.

File sesql/__init__.py

 # You should have received a copy of the GNU General Public License
 # along with SeSQL.  If not, see <http://www.gnu.org/licenses/>.
 
-#
-# !!! WARNING !!! : imports of  signals and monkey patchs are moved to
-# models.py to not create conflicts with Django's app loading...
-#
-VERSION = (0, 12, 0)
+VERSION = (0, 13, 0)
 __version__ = ".".join(map(str, VERSION))

File sesql/index.py

         log.info("%s: no table found, skipping" % message)
         return
 
-    cursor.execute('LOCK %s IN EXCLUSIVE MODE' % table_name)
+    cursor.execute('SAVEPOINT sesql_index_savepoint')
 
     query = "DELETE FROM %s WHERE id=%%s AND classname=%%s" % table_name
     cursor.execute(query, (objid, classname))
     query = "INSERT INTO %s (%s) VALUES (%s)" % (table_name,
                                                  ",".join(keys),
                                                  ",".join(placeholders))
-    try:
-        cursor.execute(query, results)
-    except Exception, e:
-        log.exception('Exception %s caught while inserting (%s,%s) into %s' %
-                      (e, classname, objid, table_name))
-        raise
+    cursor.execute(query, results)
         
 
 @index_log_wrap

File sesql/orm/django/core.py

         """
         from models import SearchHit
         SearchHit(**kwargs).save()
-
-    def begin(self):
-        """
-        Get a cursor with an open transaction
-        Let Django handle transactions
-        """
-        cursor = self.cursor()
-        if transaction.is_managed():
-            transaction.set_dirty()
-        return cursor
-
-    def commit(self, cursor):
-        """
-        Commit transaction on cursor
-        Let Django handle transactions
-        """
-        transaction.commit_unless_managed()
-
-    def rollback(self, cursor):
-        """
-        Rollback transaction on cursor
-        Let Django handle transactions
-        """
-        transaction.rollback_unless_managed()
-

File sesql/orm/django/signals.py

         return sync_db(verbosity)
     signals.post_syncdb.connect(sync_db)
 
-    @transaction.commit_on_success
     def index_cb(sender, instance, *args, **kwargs):
         # Trick to defer import
         from sesql.index import index, schedule_reindex
     signals.post_save.connect(index_cb)
 
 
-    @transaction.commit_on_success
     def unindex_cb(sender, instance, *args, **kwargs):
         # Trick to defer import
         from sesql.index import unindex, schedule_reindex

File sesql/ormadapter.py

 index/delete functions as required
 """
 
+import logging
+log = logging.getLogger('sesql')
+
 class OrmAdapter(object):
     """
     Abstract class for SeSQL ORM adapaters
     not_found = Exception # Give a more specific exception for Not Found
     node_class = None # Give a Q-compatible tree implementation
 
-    #
+    NB_TRIES = 3
+
+    #    
     # Cursor/transaction API
     #
 
 
     def begin(self):
         """
-        Get a cursor with an open transaction
+        Get a cursor with an open sub-transaction
         """
         cursor = self.cursor()
-        cursor.execute('BEGIN')
+        cursor.execute('SAVEPOINT sesql_savepoint')
         return cursor
 
     def commit(self, cursor):
         """
-        Commit transaction on cursor
+        Commit sub-transaction on cursor
         """
-        cursor.execute('COMMIT')
+        cursor.execute('RELEASE SAVEPOINT sesql_savepoint')
 
     def rollback(self, cursor):
         """
-        Rollback transaction on cursor
+        Rollback sub-transaction on cursor
         """
-        cursor.execute('ROLLBACK')
+        cursor.execute('ROLLBACK TO SAVEPOINT sesql_savepoint')
 
     def transactional(self, function):
         """
-        Wrap a function to have a transaction-bound cursor
+        Wrap a function to have a sub-transaction-bound cursor
         """
         def transactional_inner(*args, **kwargs):
-            cursor = self.begin()
-            try:
-                res = function(cursor, *args, **kwargs)
-                self.commit(cursor)
-                return res
-            except:
-                self.rollback(cursor)
-                raise
+            nb_tries = 0
+            while nb_tries < self.NB_TRIES:
+                cursor = self.begin()
+                try:
+                    res = function(cursor, *args, **kwargs)
+                    self.commit(cursor)
+                    return res
+                except Exception, e:
+                    self.rollback(cursor)
+                    nb_tries += 1
+                    if nb_tries != self.NB_TRIES:
+                        log.warning('function %s(%r, %r) failed with %s, re-attempting (#%d)'
+                                    % (function.__name__, args, kwargs, e, nb_tries))
+                    else:
+                        log.error('function %s(%r, %r) failed with %s, giving up'
+                                    % (function.__name__, args, kwargs, e))
+                        raise
         transactional_inner.__name__ = function.__name__
         return transactional_inner
 
-0.12.0
+0.13.0