Commits

Luke Plant committed 1256219

New system for handling payments to ensure serial access to BookingAccount.total_recieved

  • Participants
  • Parent commits 8bc342f
  • Branches bookings

Comments (0)

Files changed (7)

File cciw/bookings/hooks.py

 
 from .signals import places_confirmed
 from .email import send_unrecognised_payment_email, send_places_confirmed_email
-from .models import BookingAccount
+from .models import BookingAccount, send_payment
 
 #### Handlers #####
 
 
     try:
         account = BookingAccount.objects.get(id=int(m.groups()[0]))
-        account.receive_payment(ipn_obj.mc_gross)
+        send_payment(ipn_obj.mc_gross, account, ipn_obj)
     except BookingAccount.DoesNotExist:
         unrecognised_payment(ipn_obj)
 

File cciw/bookings/management/commands/process_payments.py

+from datetime import datetime
+import os
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+import zc.lockfile
+
+from cciw.bookings.models import Payment
+
+
+# When processing payments, we need to alter the BookingAccount.total_received
+# field, and may need to deal with concurrency. One solution would be
+# serializable database transations, but this has its own complications
+# (database support especially). Even with database support, there may be cases
+# where a transaction has to be retried, and it is best not to do this in the
+# context of HTTP handling where a user is waiting.
+#
+# Instead, we arrange for updates to BookingAccount.total_received to be done in
+# a separate process, which can have its own transaction management, and/or
+# another mechanism to ensure serialized requests.
+#
+# To support this, the Payment model keeps track of payments to be created.  Any
+# function that needs to transfer funds into an account uses
+# 'cciw.bookings.models.send_payment', which creates Payment objects for later
+# processing, rather than calling BookingAccount.receive_payment directly.
+#
+# Thos process_payments management command is always run in a separate process,
+# so doesn't have the transaction management of web requests, but a manual
+# transaction around each payment processed. We use a file lock to guarantee
+# serial access, rather than rely on serializable transaction isolation level.
+#
+# The management command is run in two ways:
+# - via a non-waiting os.spawn() call, triggered from web requests or anything
+#   else that causes a Payment object to be created. This ensures
+#   the db is updated ASAP.
+# - as a cron job, every minute, to ensure that nothing slips through the cracks.
+#
+# The Payment objects also act as a log of everything that has happened
+# to the BookingAccount.total_received field.
+
+
+@transaction.commit_on_success
+def process_one_payment(payment):
+    payment.account.receive_payment(payment.amount)
+    payment.processed = datetime.now()
+    payment.save()
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+
+        # We use a lock that errors if the lock already exists, and we quit if
+        # so. This is done to ensure we don't have a pile up of processes waiting
+        # to process payments in the (unlikely) event of lots of payments received.
+        # The scheduled run of this command should clean up any Payment objects
+        # that get missed because of this strategy.
+
+        try:
+            l = zc.lockfile.LockFile(os.path.join(os.environ['HOME'], '.cciw_process_payments_lock'))
+        except zc.lockfile.LockError:
+            return
+
+        try:
+            for payment in Payment.objects.filter(processed__isnull=True).order_by('created'):
+                try:
+                    process_one_payment(payment)
+                except Exception:
+                    # Send email, but carry on with next payment
+                    from cciw.cciwmain.common import exception_notify_admins
+                    exception_notify_admins('CCIW booking - payment processing error')
+
+        finally:
+            l.close()
+

File cciw/bookings/migrations/0003_auto__add_payment.py

+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'Payment'
+        db.create_table('bookings_payment', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('amount', self.gf('django.db.models.fields.DecimalField')(max_digits=10, decimal_places=2)),
+            ('account', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bookings.BookingAccount'])),
+            ('origin_id', self.gf('django.db.models.fields.PositiveIntegerField')()),
+            ('origin_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])),
+            ('processed', self.gf('django.db.models.fields.DateTimeField')(null=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')()),
+        ))
+        db.send_create_signal('bookings', ['Payment'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'Payment'
+        db.delete_table('bookings_payment')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'bookings.booking': {
+            'Meta': {'ordering': "['-created']", 'object_name': 'Booking'},
+            'account': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bookings'", 'to': "orm['bookings.BookingAccount']"}),
+            'address': ('django.db.models.fields.TextField', [], {}),
+            'agreement': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'allergies': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'amount_due': ('django.db.models.fields.DecimalField', [], {'max_digits': '10', 'decimal_places': '2'}),
+            'booking_expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'camp': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bookings'", 'to': "orm['cciwmain.Camp']"}),
+            'church': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'contact_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'contact_phone_number': ('django.db.models.fields.CharField', [], {'max_length': '22'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'date_of_birth': ('django.db.models.fields.DateField', [], {}),
+            'dietary_requirements': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'gp_address': ('django.db.models.fields.TextField', [], {}),
+            'gp_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'gp_phone_number': ('django.db.models.fields.CharField', [], {'max_length': '22'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'illnesses': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'last_tetanus_injection': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'learning_difficulties': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'medical_card_number': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'phone_number': ('django.db.models.fields.CharField', [], {'max_length': '22', 'blank': 'True'}),
+            'post_code': ('django.db.models.fields.CharField', [], {'max_length': '10'}),
+            'price_type': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'regular_medication_required': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'serious_illness': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'sex': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'shelved': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'south_wales_transport': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'state': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'bookings.bookingaccount': {
+            'Meta': {'object_name': 'BookingAccount'},
+            'address': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'blank': 'True'}),
+            'first_login': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'phone_number': ('django.db.models.fields.CharField', [], {'max_length': '22', 'blank': 'True'}),
+            'post_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'blank': 'True'}),
+            'share_phone_number': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'total_received': ('django.db.models.fields.DecimalField', [], {'default': "'0.00'", 'max_digits': '10', 'decimal_places': '2'})
+        },
+        'bookings.payment': {
+            'Meta': {'object_name': 'Payment'},
+            'account': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['bookings.BookingAccount']"}),
+            'amount': ('django.db.models.fields.DecimalField', [], {'max_digits': '10', 'decimal_places': '2'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'origin_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'origin_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'processed': ('django.db.models.fields.DateTimeField', [], {'null': 'True'})
+        },
+        'bookings.price': {
+            'Meta': {'unique_together': "(['year', 'price_type'],)", 'object_name': 'Price'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'price': ('django.db.models.fields.DecimalField', [], {'max_digits': '10', 'decimal_places': '2'}),
+            'price_type': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'year': ('django.db.models.fields.PositiveSmallIntegerField', [], {})
+        },
+        'cciwmain.camp': {
+            'Meta': {'ordering': "['-year', 'number']", 'unique_together': "(('year', 'number'),)", 'object_name': 'Camp'},
+            'admins': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'camps_as_admin'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['auth.User']"}),
+            'chaplain': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'camps_as_chaplain'", 'null': 'True', 'to': "orm['cciwmain.Person']"}),
+            'end_date': ('django.db.models.fields.DateField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'leaders': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'camps_as_leader'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['cciwmain.Person']"}),
+            'max_campers': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '80'}),
+            'max_female_campers': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '60'}),
+            'max_male_campers': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '60'}),
+            'maximum_age': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'minimum_age': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'number': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'officers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'through': "orm['officers.Invitation']", 'symmetrical': 'False'}),
+            'online_applications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'previous_camp': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'next_camps'", 'null': 'True', 'to': "orm['cciwmain.Camp']"}),
+            'site': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cciwmain.Site']"}),
+            'start_date': ('django.db.models.fields.DateField', [], {}),
+            'year': ('django.db.models.fields.PositiveSmallIntegerField', [], {})
+        },
+        'cciwmain.person': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'Person'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'info': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'cciwmain.site': {
+            'Meta': {'object_name': 'Site'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'info': ('django.db.models.fields.TextField', [], {}),
+            'long_name': ('django.db.models.fields.CharField', [], {'max_length': "'50'"}),
+            'short_name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': "'25'"}),
+            'slug_name': ('django.db.models.fields.SlugField', [], {'db_index': 'True', 'unique': 'True', 'max_length': "'25'", 'blank': 'True'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'officers.invitation': {
+            'Meta': {'ordering': "('-camp__year', 'officer__first_name', 'officer__last_name')", 'unique_together': "(('officer', 'camp'),)", 'object_name': 'Invitation'},
+            'camp': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cciwmain.Camp']"}),
+            'date_added': ('django.db.models.fields.DateField', [], {'default': 'datetime.date.today'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'notes': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'officer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['bookings']

File cciw/bookings/models.py

 import os
 
 from dateutil.relativedelta import relativedelta
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes import generic
 from django.db import models
 from django.utils.safestring import mark_safe
 
         return total - self.total_received
 
     def receive_payment(self, amount):
+        """
+        Adds the amount to the account's total_received field.  This should only
+        ever be called by the 'process_payments' management command. Client code
+        should use the 'send_payment' function.
+        """
         # = Receiving payments =
         #
         # This system needs to be robust, and cope with all kinds of user error, and
         lock.release()
 
 
+# See process_payments management command for explanation
+
+class Payment(models.Model):
+    amount = models.DecimalField(decimal_places=2, max_digits=10)
+    account = models.ForeignKey(BookingAccount)
+    origin_id = models.PositiveIntegerField()
+    origin_type = models.ForeignKey(ContentType)
+    origin = generic.GenericForeignKey('origin_type', 'origin_id')
+    processed = models.DateTimeField(null=True)
+    created = models.DateTimeField()
+
+    def __unicode__(self):
+        return u"<Payment: %s to %s from %s>" % (self.amount, self.account, self.origin)
+
+
+def trigger_payment_processing():
+    # NB - this is always called from a web request, for which the virtualenv
+    # has been set up in os.environ, so this is passed on and the correct python
+    # runs manage.py.
+    manage_py = os.path.join(settings.BASE_DIR, 'manage.py')
+    os.spawnl(os.P_NOWAIT, manage_py, 'manage.py', 'process_payments')
+
+
+def send_payment(amount, to_account, from_obj):
+    Payment.objects.create(amount=amount,
+                           account=to_account,
+                           origin=from_obj,
+                           processed=None,
+                           created=datetime.now())
+    trigger_payment_processing()
+
+
 # Very important that the setup done in .hooks happens:
 from .hooks import *

File cciw/bookings/tests.py

         from cciw.bookings.models import paypal_payment_received
         self.login()
 
+        from paypal.standard.ipn.models import PayPalIPN
         class IpnMock(object):
             pass
 
-        ipn_1 = IpnMock()
-        ipn_1.id = 123
-        ipn_1.mc_gross = Decimal('1.00')
-        ipn_1.custom = "account:%s;" % self.get_account().id
-
+        ipn_1 = PayPalIPN.objects.create(mc_gross = Decimal('1.00'),
+                                         custom = "account:%s;" % self.get_account().id,
+                                         ipaddress='127.0.0.1',
+                                         )
         mail.outbox = []
         self.assertEqual(len(mail.outbox), 0)
         paypal_payment_received(ipn_1)
 
-        self.assertEqual(self.get_account().total_received, ipn_1.mc_gross)
+        # Since payments are processed in a separate process, we cannot
+        # test that the account was updated in this process.
 
     def test_email_for_good_payment(self):
         # This email could be triggered by whenever BookingAccount.distribute_funds

File cciw/cciwmain/common.py

 """
 import datetime
 import re
+import sys
+import traceback
 
 from django.conf import settings
+from django.core.mail import mail_admins
 from django.contrib.sites.models import Site
 from django.core.urlresolvers import reverse
 from django.http import HttpResponse
     return Site.objects.get_current().domain
 
 
+def exception_notify_admins(subject):
+    """
+    Send admins notification of an exception that occurred
+    """
+    exc_info = sys.exc_info()
+    message = '\n'.join(traceback.format_exception(*exc_info))
+    mail_admins(subject, message, fail_silently=True)
+
+
 from cciw.cciwmain import feeds

File cciw/settings.py

 
 DEFAULT_CONTENT_TYPE = "text/html"
 
+BASE_DIR = basedir