Source

comofas / comofas2.py

Full commit
#! /usr/bin/env python
# -*- coding: utf-8 -*-

# Comofas - everyone deserves an answer from Yahoo!Answers and Twitter
# Yahoo!Answers and Twitter are trademarks from Yahoo! Inc and Twitter respectively.
#
#Copyright (C) 2008  Pedro Valente
#
#This program is free software; you can redistribute it and/or
#modify it under the terms of the GNU General Public License
#as published by the Free Software Foundation; either version 2
#of the License, or (at your option) any later version.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with this program; if not, write to the Free Software
#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

from pynswers.Answers import Answers

import urllib
import urllib2
import string
import random
import re
import simplejson
import time
import sys
import logging
import pickle
import os
import traceback

from twitter import twitter
from comofas_credentials import *
from comofas_settings import *

class Comofas(object):
    """
    Esta classe pretende cumprir a nobre missao de responder
    a todos os usuarios
    """
    def __init__(self,twitter_connection=None,answers_connection=None):
        """
        o twitter_connection pode ser substituido por outra classe
        para executar testes offline.
        """
        self.twitter_connection = twitter_connection
        self.twitter_connection.SetXTwitterHeaders('fasassim',"http://twitter.com/fasassim",'Ni!')
        self.answers_connection = answers_connection
        self.search_url = 'http://search.twitter.com/search.json?q=' + KEYWORD
        self.last_feeds = dict()
        self.last_users = dict()

        logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG,)

        if PICKLE_FILENAME in os.listdir('.'):
            self.picklefd = open(PICKLE_FILENAME,"rb")
            try:
                self.last_feeds = pickle.load(self.picklefd)
            except EOFError:
                self.last_feeds = dict()
            finally:
                self.picklefd.close()
        else:
            self.picklefd =open(PICKLE_FILENAME,'wb')
            pickle.dump(self.last_feeds,self.picklefd)
            self.picklefd.close()
            self.picklefd =open(PICKLE_FILENAME,'wb')
            self.last_feeds = pickle.load(self.picklefd)
            self.picklefd.close()

    def persist_last_feeds(self):
        self.picklefd =open(PICKLE_FILENAME,"wb")
        pickle.dump(self.last_feeds,self.picklefd)
        self.picklefd.close()

    def get_last_feed(self):
        """
        este metodo retorna o ultimo resultado, com os
        campos text e from_user apenas
        """
        feed = urllib.urlopen(self.search_url)
        updates = feed.read()
        updates = simplejson.loads(updates)

        return updates['results'][0]['text'].lower(), updates['results'][0]['from_user']

    def get_feeds(self):
        """
        Este metodo retorna todos os resultados, sem filtrar por text e from_user
        """
        feed = urllib.urlopen('http://search.twitter.com/search.json?q=' + KEYWORD)
        updates = feed.read()
        updates = simplejson.loads(updates)

        return updates['results']

    def clean_text(self,txt):
        """
        vou falar o obvio: este metodo limpa o texto
        """
        pergunta = re.sub('#\w+',' ', txt)
        pergunta = re.sub('@\w+',' ', pergunta)

        pergunta = pergunta.replace(KEYWORD,' ')
        s=string.find(pergunta,'http://')
        if s != -1:
            t = string.find(pergunta[s+7:], " ")
            pergunta = string.replace(pergunta,pergunta[s:t+s+7],' ')

        replacements = [(u'é','e'),(u'á','a'),(u'í','i'),(u'ó','o'),(u'ú','u'),(u'â','a'),(u'ê','e'),(u'ô','o'),(u'ã','a'),(u'õ','o'),(u'ç','c')]

        for a, b in replacements:
            pergunta = pergunta.replace(a, b)
        for punct in string.punctuation:
            pergunta = pergunta.replace(punct, ' ')

        pergunta = re.sub('\n','', pergunta)
        pergunta = re.sub('\s\s','', pergunta)

        pergunta = string.joinfields(pergunta.split(), ' OR ')

        return pergunta

    def nao_entendi(self,usuario):
        """
        este metodo pede que o usuario pergunte de outra forma
        """
        nao_entendi_lista=[u"Não entendi mt bem, vc poderia me explicar?",u"Vc poderia explicar melhor?", u"Hã? Repete aí, mas fala diferente."]
        mensagem = self.build_message(to_user=usuario, raw_answer=nao_entendi_lista[random.randint(0,len(nao_entendi_lista)-1)])
        status = "sem status"
        try:
            status = self.twitter_connection.PostUpdate(mensagem)
        except urllib2.HTTPError:
            logging.debug("answer exception:" + str(traceback.print_exc()))
        print '-' * 50
        print "twitter.fasassim(nao entendi) postUpdate: %s - status:%s" % (mensagem,status)

    def build_message(self,to_user, raw_answer):
        """
        este metodo:
        remove palavras que nao ficam bem no final da frase.
        limita o tamanho da frase
        """
        chars = 140
        msg = "@%s " % to_user
        chars = chars - len(msg)
        if len(raw_answer) < chars:
            return "%s%s" % (msg, raw_answer)
        last = 0
        for indice, c in enumerate(raw_answer):
            if indice > chars:
                break
            if c == " ":
                last = indice
        # Lista de palavras que não ficam bem no fim da frase.
        dispensaveis = ['e','a','o','de','da','do','na','no','para','pra','mas','se','que','como','porque']
        a = raw_answer[:last].split()
        while a[-1] in dispensaveis:
            a.pop(-1)
        return "%s%s" % (msg, " ".join(a))

    def add_new_friends(self):
        added_friends = list()
        try:
            added_friends = self.twitter_connection.GetFriends()
            for friend in added_friends:
                if not self.last_users.has_key(friend.name) and friend.name != 'fasassim':
                    try:
                        self.twitter_connection.CreateFriendship(friend)
                        print "twitter.fasassim.following %s" % friend.name
                        self.last_users.pop(friend.name)
                    except:
                        print "cannot create friendship for " + friend.name
        except urllib2.HTTPError:
            logging.debug("answer exception:" + str(traceback.print_exc()))
        finally:
            self.last_users = dict()

    def answer_tweets(self):
        """
        este metodo busca os feeds do twitter, limpa o texto e processa a resposta para cada feed
        ele nao foi testado e deve ser refatorado p/ dividir as responsabilidades

        twitter keys:
        [u'iso_language_code',
     u'text',
     u'created_at',
     u'profile_image_url',
     u'from_user',
     u'from_user_id',
     u'to_user_id',
     u'id']

        twitter user methods:
        AsDict
    AsJsonString
    GetDescription
    GetId
    GetLocation
    GetName
    GetProfileImageUrl
    GetScreenName
    GetStatus
    GetUrl
    NewFromJsonDict
    SetDescription
    SetId
    SetLocation
    SetName
    SetProfileImageUrl
    SetScreenName
    SetStatus
    SetUrl
        """
        try:
            while (True):
                
                time.sleep(random.randint(15,30)*2)
                feeds = self.get_feeds()
                for feed in feeds:
                    if feed.get('id') in self.last_feeds:
                        continue
                    else:

                        usuario = feed.get('from_user')
                        pergunta = feed.get('text')
                        self.last_users[usuario] = 1

                        pergunta = self.clean_text(pergunta)

                        try:
                            answers_for_questions = answers.questionSearch({'query':pergunta, 'search_in':'question', 'region': REGION, 'type':'resolved', 'results':'1'})
                        except:
                            logging.debug("answer exception:" + str(traceback.print_exc()))
                            continue

                        if len(self.last_users) == LAST_USERS_SIZE:
                            self.add_new_friends()

                        if answers_for_questions:
                            for question in answers_for_questions:
                                try:
                                    print '-'* 50
                                    print "Twitter: %s\nPergunta: %s \n\n" % (pergunta, question['Subject'])

                                    mensagem = self.build_message(to_user=usuario, raw_answer=question['ChosenAnswer'])
                                    meu_timeline =self.twitter_connection.GetUserTimeline(TWITTER_USERNAME)

                                    for msg in meu_timeline:
                                        if mensagem == msg.text:
                                            continue

                                    print '-' * 50
                                    print "twitter.fasassim postUpdate: %s" % mensagem

                                    status = self.twitter_connection.PostUpdate(mensagem)
                                    self.last_feeds[feed.get('id')] = feed
                                except (UnicodeEncodeError,urllib2.HTTPError):
                                    logging.debug("Error:" + str(traceback.print_exc()))
                                    continue
                        else:
                            logging.debug("Nao entendi:" + pergunta)
                            #self.nao_entendi(usuario)
        except KeyboardInterrupt:
            self.persist_last_feeds()
            sys.exit(1)



if __name__=='__main__':
    answers = Answers()
    answers.appid = answers_appid

    twitter_api = twitter.Api(username=TWITTER_USERNAME, password=TWITTER_PASSWORD)
    twitter_api.SetXTwitterHeaders('fasassim',"http://twitter.com/fasassim",'Ni!')

    comofas2 = Comofas(twitter_connection = twitter_api,answers_connection = answers)
    comofas2.answer_tweets()