Source

eggDiscover / followerslookup / tasks.py

Full commit
"""
This set of applications are divided in two main category :

- Twitter management
- Csv generation

Twitter management are :

- fetch

Fetch is done to retreive all twiiter user following a
"TwitterUser". Each time a user is totaly fetched we reset followers
count and we re-update the followers list next hour

- cleanup

cleanup look for twitter users no more following a TwitterUser and
delete them from the database

"""

from celery.schedules import crontab
from celery.decorators import task
from celery.decorators import periodic_task
from social_auth.models import UserSocialAuth
from followerslookup.authentication import get_account
from followerslookup.fetching import get_followers_infos,get_followers_ids,cleanup
from followerslookup.models import TwitterUser, User
import csv,codecs
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO


@task()
def fetch():
    """
    Fetch all TwitterUser to get their followers.
    If we are rate limited, we return : "rate limit reached"
    If fetch is finished, we return "fetch ended".
    and we reset the last index to parse for new followers next time.
    """
    user = UserSocialAuth.objects.all()[0]
    account = get_account(user)
    for user in TwitterUser.objects.all().order_by("-lastindex"):
        followers_ids = get_followers_ids(account, user.name)
        followers_infos = get_followers_infos(account, followers_ids,user.name )
        if followers_infos == "Fetch ended":
            user.lastindex = 0
            user.save()
    return followers_infos

@task()
def followers_cleanup():
    """
    This function clean the Twitter users followers and destroy users
    that do not fowwow the Twitter User anymore
    """
    user = UserSocialAuth.objects.all()[0]
    account = get_account(user)
    for user in TwitterUser.objects.all().order_by("-lastindex"):
        followers_ids = get_followers_ids(account, user.name)
        cleanup(followers_ids)
    return "%s cleaned"%user.name

@task()
def convert_context_to_csv(context, csv_file = StringIO()):
    if type(csv_file) == unicode:
        csv_file = open("%s%s"%(csv_file,'-users.csv'),"w")
    writer = csv.writer(csv_file)
    writer.writerow(User._meta.get_all_field_names())
    for obj in context["object_list"]:
        row = []
        for elem in User._meta.get_all_field_names():
            if elem == "twitteruser":
                twitter_users = []
                for usr in obj.twitteruser_set.all():
                    twitter_users.append(usr.name)
                    row.append(','.join(twitter_users))
            else:
                row.append(unicode(getattr(obj,elem)).encode("utf-8"))
        writer.writerow(row)
    if type(csv_file) == file:
        csv_file.close()
        return
    return csv_file.getvalue()