Commits

Max Noel committed 79b9cdc Merge

Merged in jtlebigot/dynamodb-mapper (pull request #9)

Comments (0)

Files changed (5)

dynamodb_mapper/model.py

             default = default()
         # Check default value consitency
         if not isinstance(default, schema_type):
-            raise TypeError("Default value is of type {}. Expected: {}".format(schema_type.__name__, default.__class__.__name__))
+            raise TypeError("Expected default value of type {}, got: {}".format(schema_type, type(default)))
         else:
             return default
 
         s = s[:-2] + ':' + s[-2:]
         return s
 
+    # This case prevents `'fields': False` to be added when genereating expected
+    # values dict in save as this would mean 'field does not exist' instead of
+    # 'field exists and is False'.
+    if isinstance(value, bool):
+        return int(value)
+
     if value or value == 0:
         return value
 
           Supported attribute_types are: int, long, float, str, unicode, set.
           Default values are obtained by calling the type with no args
           (so 0 for numbers, "" for strings and empty sets).
+      - __defaults__: {attribute_name: defaulter} optional mapping.
+          This dict allows to provide a default value for each attribute_name at
+          object creation time. It will *never* be used when loading from the DB.
+          It is fully optional. If no value is supplied the empty value
+          corresponding to the type will be used.
+          "defaulter" may either be a scalar value or a callable with no
+          arguments.
 
     To redefine serialization/deserialization semantics (e.g. to have more
     complex schemas, like auto-serialized JSON data structures), override the
         Default values are used for anything that's missing from the dict
         (see DynamoDBModel class docstring).
         """
+        #FIXME: rename to from_db_dict when breaking compat or disambiguing
         instance = cls()
         for (name, type_) in cls.__schema__.iteritems():
             # Set the value if we got one from DynamoDB. Otherwise, stick with the default

dynamodb_mapper/tests/test_model.py

     }
 
 
+# boolean regression test
+class DoomCampaignStatus(DynamoDBModel):
+    __table__ = "doom_campaign_status"
+    __hash_key__ = "id"
+    __schema__ = {
+        "id": int,
+        "name": unicode,
+        "completed": bool,
+    }
+
 # list attribute
 class DoomMonster(DynamoDBModel):
     __table__ = "doom_monster"
         self.assertEquals(0.0, _python_to_dynamodb(0.0))
         self.assertEquals(10, _python_to_dynamodb(10))
         self.assertEquals(10.0, _python_to_dynamodb(10.0))
+        self.assertEquals(0, _python_to_dynamodb(False))
+        self.assertEquals(1, _python_to_dynamodb(True))
 
     def test_dynamodb_to_python_number(self):
         self.assertEquals(0, _dynamodb_to_python(int, 0))
         self.assertEquals(0.0, _dynamodb_to_python(float, 0.0))
         self.assertEquals(10, _dynamodb_to_python(int, 10))
         self.assertEquals(10.0, _dynamodb_to_python(float, 10.0))
+        self.assertEquals(False, _dynamodb_to_python(bool, 0))
+        self.assertEquals(True, _dynamodb_to_python(bool, 1))
 
     def test_python_to_dynamodb_unicode(self):
         self.assertEquals(u"hello", _python_to_dynamodb(u"hello"))
         c.save(expected_values={"id": 1, "name": name, "cheats": cheats})
         m_item_instance.put.assert_called_with({"id": 1, "name": name, "cheats": cheats})
 
+
+    @mock.patch("dynamodb_mapper.model.Item")
+    @mock.patch("dynamodb_mapper.model.boto")
+    def test_save_expected_values_boolean(self, m_boto, m_item):
+        m_item_instance = m_item.return_value
+
+        name = "Knee-deep in the Dead"
+        completed = False
+
+        c = DoomCampaignStatus()
+        c.id = 0
+        c.name = name
+        c.completed = completed
+
+        c.save(expected_values={"id": 1, "name": name, "completed": completed})
+        m_item_instance.put.assert_called_with({"id": 1, "name": name, "completed": 0})
+
     @mock.patch("dynamodb_mapper.model.Item")
     @mock.patch("dynamodb_mapper.model.boto")
     def test_save_no_overwrite_composite_fails(self, m_boto, m_item):

dynamodb_mapper/tests/test_transactions.py

         "energy": int
     }
 
+class Reward(DynamoDBModel):
+    __table__ = "rewards"
+    __hash_key__ = u'user_id'
+    __range_key__ = u'name'
+    __schema__ = {
+        'user_id': int,
+        'name': unicode,
+        'collected': bool,
+    }
 
 class InsufficientEnergyError(Exception):
     """Raised when a transaction would make a User's energy negative."""
     """Raised when the universe ceases to exist."""
     pass
 
+class collectRewards(Transaction):
+    """A sample transaction using the new system to work on multiple targets
+    It also relies on the basic schema provided in the base class.
+
+    In this testsm the getter returns a brand new object each time. this is probably
+    not what you want to do in real code.
+
+    Note that any function is supported, not only lambas.
+    """
+    __table__ = "rewards"
+
+    def _get_transactors(self):
+        return [
+            (
+                lambda: Reward(user_id=1, name=u"level up", collected=False),
+                lambda target: setattr(target, "collected", True)
+            ),
+            (
+                lambda: Reward(user_id=1, name=u"5 days", collected=False),
+                lambda target: setattr(target, "collected", True)
+            )
+        ]
 
 class UserEnergyTransaction(Transaction):
     """A sample transaction that adds/removes energy to a User."""
             raise InsufficientEnergyError(target.energy, self.energy)
         target.energy = new_energy
 
+
 class TransientUserEnergyTransaction(UserEnergyTransaction):
     """Exactly like UserEnergyTransaction, but transient (never saved to the DB)."""
     transient = True
 
 
+
 class TestTransaction(unittest.TestCase):
     def _get_default_user(self):
         return User.from_dict({"id": USER_ID, "energy": ENERGY})
         self.assertEquals(m_transaction_save.call_count, 0)
 
         self.assertEquals(m_user_instance.energy, 0)
+        self.assertEquals(t.status, "done")
 
+    @mock.patch("dynamodb_mapper.transactions.Transaction.save")
     @mock.patch("dynamodb_mapper.model.DynamoDBModel.save")
     @mock.patch("dynamodb_mapper.transactions.Transaction._setup")
-    def test_setup_fails(self, m_setup, m_save):
+    def test_setup_fails(self, m_setup, m_save, m_transaction_save):
         # When the setup phase fails, nothing must be called, and nothing must be saved.
         m_setup.side_effect = UniverseDestroyedError("ONOZ!")
         t = UserEnergyTransaction.from_dict({"user_id": USER_ID, "energy": 10})
         self.assertRaises(UniverseDestroyedError, t.commit)
 
         self.assertEquals(m_save.call_count, 0)
+        m_transaction_save.assert_called()
+        self.assertEquals(t.status, "pending")
 
+    @mock.patch("dynamodb_mapper.transactions.Transaction.save")
     @mock.patch.object(User, "save")
     @mock.patch("dynamodb_mapper.model.DynamoDBModel.get")
-    def test_target_not_found(self, m_get, m_user_save):
+    def test_target_not_found(self, m_get, m_user_save, m_transaction_save):
         m_get.side_effect = DynamoDBKeyNotFoundError("ONOZ!")
         t = UserEnergyTransaction.from_dict({"user_id": USER_ID, "energy": 10})
 
         self.assertRaises(TargetNotFoundError, t.commit)
 
         self.assertEquals(m_user_save.call_count, 0)
+        m_transaction_save.assert_called()
+        self.assertEquals(t.status, "running")
 
+    @mock.patch("dynamodb_mapper.transactions.Transaction.save")
     @mock.patch.object(User, "save")
     @mock.patch.object(User, "get")
-    def test_insufficient_energy(self, m_user_get, m_user_save):
+    def test_insufficient_energy(self, m_user_get, m_user_save, m_transaction_save):
         m_user_instance = self._get_default_user()
         m_user_get.return_value = m_user_instance
         t = UserEnergyTransaction.from_dict({"user_id": USER_ID, "energy": -ENERGY * 2})
 
         self.assertEquals(m_user_instance.energy, ENERGY)
         self.assertEquals(m_user_save.call_count, 0)
+        m_transaction_save.assert_called()
+        self.assertEquals(t.status, "running")
 
     @mock.patch("dynamodb_mapper.transactions.Transaction.save")
     @mock.patch.object(User, "save")
 
         t.commit()
         m_transaction_save.assert_called()
+        self.assertEquals(t.status, "done")
         self.assertEqual(m_user_save.call_count, failed_tries)
 
+    @mock.patch("dynamodb_mapper.transactions.Transaction.save")
     @mock.patch.object(User, "save")
     @mock.patch.object(User, "get")
-    def test_max_retries_exceeded(self, m_user_get, m_user_save):
+    def test_max_retries_exceeded(self, m_user_get, m_user_save, m_transaction_save):
         # Return a clean user every time -- we will be retrying a lot.
         m_user_get.side_effect = lambda *args, **kw: self._get_default_user()
         m_user_save.side_effect = ExpectedValueError()
 
         self.assertRaises(MaxRetriesExceededError, t.commit)
         self.assertEquals(m_user_save.call_count, Transaction.MAX_RETRIES)
+        m_transaction_save.assert_called()
+        self.assertEquals(t.status, "running")
+
+    def test_get_2_transactors(self):
+        t = collectRewards()
+        transactors = t._get_transactors()
+
+        self.assertEquals(len(transactors), 2)
+
+    def test_legacy_get_transactors(self):
+        t = UserEnergyTransaction()
+        transactors = t._get_transactors()
+
+        self.assertEquals(len(transactors), 1)
+        self.assertEquals(transactors[0][0], t._get_target)
+        self.assertEquals(transactors[0][1], t._alter_target)
+
+    @mock.patch("dynamodb_mapper.transactions.Transaction.save")
+    @mock.patch.object(Reward, "save")
+    def test_commit_2_targets(self, m_reward_save, m_transaction_save):
+        t = collectRewards()
+        t.commit()
+
+        self.assertEquals(m_reward_save.call_count, 2)
+        m_transaction_save.assert_called()
+        self.assertEquals(t.status, "done")

dynamodb_mapper/transactions.py

 
     This class gracefully handles concurrent modifications and auto-retries but
     embeds no tool to rollback at the moment.
+
+    Transactions status may be persisted for tracability, further analysis...
+    for this purpose, a minimal schema is embedded in this base class. When
+    deriving, you MUST keep
+    * datetime field as rangekey
+    * status field
+    The hash key field may be changed to pick a ore relevant name or change its
+    type. In any case, you are responsible of setting its value. For example, if
+    collecting rewards for a player, you may wish to keep track of related
+    transactions by user_id hence set requester_id to user_id
+
+    Deriving class MUST set field __table__
+
     """
+
+    __hash_key__ = "requester_id"
     __range_key__ = "datetime"
+
+    __schema__ = {
+        "requester_id": int,
+        "datetime": datetime,
+        "status": unicode #IN("pending", "running", "done")
+    }
+
     # Transient transactions (with this flag set to True) are not saved in the
-    # database, and are as a result write-only.
+    # database, and are as a result write-only. This value is defined on the
+    # class level bu may be redefined on a per instance basis.
     transient = False
-
+    # Maximum attempts. Each attempt consumes write credits
     MAX_RETRIES = 100
 
     def _setup(self):
         """
         pass
 
-    def _get_target(self):
-        """Fetch the object on which this transaction is supposed to operate
-        (e.g. a User instance for UserResourceTransactions) from the DB and
-        return it.
+    def _get_transactors(self):
+        """Fetch a list of targets (getter, setter) tuples. The transaction
+        engine will walk the list. For each tuple, the getter and the setter are
+        called successively until this step of the transaction succeed or exhaust
+        the MAX_RETRIES.
 
-        It is important that this method actually connect to the database and
-        retrieve a clean, up-to-date version of the object -- because it will
-        be called repeatedly if conditional updates fail due to the target
-        object having changed.
+        * getter: Fetch the object on which this transaction is supposed to operate
+            (e.g. a User instance for UserResourceTransactions) from the DB and
+            return it.
+            It is important that this method actually connect to the database and
+            retrieve a clean, up-to-date version of the object -- because it will
+            be called repeatedly if conditional updates fail due to the target
+            object having changed.
+            The getter takes no argument and returns a DBModel instance
+
+        * setter: Applyies the transaction to the target, modifying it in-place.
+            Does *not* attempt to save the target or the transaction to the DB.
+            The setter takes a DBModel instance as argument. Its return value is
+            ignored
+
+        The list is walked from 0 to len(transactors)-1. Order may matter.
 
         :raise TargetNotFoundError: If the target doesn't exist in the DB.
         """
+        #FIXME: compat method
+        return [(self._get_target, self._alter_target)]
+
+    def _get_target(self):
+        #FIXME: legacy
         pass
 
     def _alter_target(self, target):
-        """Apply the transaction to the target, modifying it in-place.
-
-        Does *not* attempt to save the target or the transaction to the DB.
-        """
+        #FIXME: legacy
         pass
 
-    def _apply_and_save_target(self):
+    def _apply_and_save_target(self, getter, setter):
         """Apply the Transaction and attempt to save its target (but not
         the Transaction itself). May be called repeatedly until it stops
         raising :exc:`ExpectedValueError`.
 
+        Will succeed iff no attributes of the object returned by getter has been
+        modified before ou save method to prevent accidental overwrites.
+
+        :param getter: getter as defined in :py:meth:`_get_transactors`
+        :param setter: setter as defined in :py:meth:`_get_transactors`
+
         :raise ExpectedValueError: If the target is changed by an external
             source (other than the Transaction) between its retrieval from
             the DB and the save attempt.
         """
-        target = self._get_target()
-
-        # We want to redo the transaction if *anything* in the user
-        # changed, not just the target attribute (no accidental overwrites).
+        # load base object
+        target = getter()
         old_values = target.to_db_dict()
 
-        self._alter_target(target)
+        # edit and attempt to save it
+        setter(target)
         target.save(expected_values=old_values)
 
     def _assign_datetime_and_save(self):
         self.datetime = datetime.now(utc_tz)
         self.save(allow_overwrite=False)
 
-    def _retry(self, fn, exc_class):
-        """Call ``fn`` repeatedly, until it stops raising ``exc_class`` or
-        it has been called ``MAX_RETRIES`` times (in which case
+    def _retry(self, fn, exc_class, *args):
+        """Call ``fn`` repeatedly with ``*args``, until it stops raising
+        ``exc_class`` or it has been called ``MAX_RETRIES`` times (in which case
         :exc:`MaxRetriesExceededError` is raised).
 
         :param fn: The callable to retry calling.
-
         :param exc_class: An exception class (or tuple thereof) that, if raised
             by fn, means it has failed and should be called again.
             *Any other exception will propagate normally, cancelling the
             auto-retry process.*
+        :param *args: Optional arguments to pass to ``fn``
         """
         tries = 0
         while tries < self.MAX_RETRIES:
             tries += 1
             try:
-                fn()
-                # Nothing was raised: we're done!
+                fn(*args)
+                # Nothing was raised: we're done !
                 break
             except exc_class as e:
                 log.debug(
         else:
             raise MaxRetriesExceededError()
 
+    def commit(self):
+        """ Run the transaction and, if needed, store its states to the database
+
+            - set up preconditions and parameters (:meth:`_setup` -- only called
+              once no matter what).
+            - fetch all transaction steps (:meth:`_get_transactors`).
+            - for each transaction :
+                - fetch the target object from the DB.
+                - modify the target object according to the transaction's parameters.
+                - save the (modified) target to the DB
+            - save the transaction to the DB
+
+        Each transation may be retried up to ``MAX_RETRIES`` times automatically.
+        commit uses conditional writes to avoid overwriting data in the case of
+        concurrent transactions on the same target (see :meth:`_retry`).
+        """
+        try:
+            self.status = "pending"
+
+            self._setup()
+            transactors = self._get_transactors()
+
+            self.status = "running"
+
+            for transactor in transactors:
+                self._retry(self._apply_and_save_target, ExpectedValueError, *transactor)
+
+            self.status = "done"
+        finally:
+            # Always (attempt to) save transaction status
+            self._retry(self._assign_datetime_and_save, OverwriteError)
+
     def save(self, allow_overwrite=True, expected_values=None):
         """If the transaction is transient (``transient = True``),
         do nothing.
 
         If the transaction is persistent (``transient = False``), save it to
         the DB, as :meth:`DynamoDBModel.save`.
+
+        Note: this method is called automatically from ``commit``. You may but do
+        not need to call it explicitely.
         """
-        cls = type(self)
-        if cls.transient:
+        if self.transient:
             log.debug(
                 "class=%s: Transient transaction class, ignoring save attempt.",
-                cls)
+                type(self))
         else:
             super(Transaction, self).save(
                 allow_overwrite=allow_overwrite, expected_values=expected_values)
 
-    def commit(self):
-        """Commit the transaction:
-
-            - set up preconditions and parameters (:meth:`_setup` -- only called
-              once no matter what).
-            - fetch the target object in the DB (:meth:`_get_target`).
-            - modify the target object according to the transaction's parameters
-              (:meth:`_alter_target`).
-            - save the (modified) target to the DB
-            - save the transaction to the DB
-
-        commit knows how to auto-retry, and uses conditional writes to avoid
-        overwriting data in the case of concurrent transactions on the same
-        target (see :meth:`_retry`).
-        """
-        self._setup()
-
-        self._retry(self._apply_and_save_target, ExpectedValueError)
-        self._retry(self._assign_datetime_and_save, OverwriteError)
 [metadata]
 name = dynamodb-mapper
-version = 1.5.0
+version = 1.6.0
 summary = Object mapper for Amazon DynamoDB
 description_file = README.rst
 author = Max Noel