Anonymous avatar Anonymous committed 4e88a83

Nova separacao de tarefas do comofas. Todas as acoes rodam uma vez soh e dependem da persistencia em arquivo e de um cron para chama-las. O arquivo comofas_sequencia executa as 3 etapas (puxar novas perguntas, encontrar resposta, mandar pro tuiter) de uma vez

Comments (0)

Files changed (4)

comofas_answers.py

+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Comofas - everyone deserves an answer from Yahoo!Answers and Twitter
+# Yahoo!Answers and Twitter are trademarks from Yahoo! Inc and Twitter respectively.
+#
+#Copyright (C) 2008  Pedro Valente
+#
+#This program is free software; you can redistribute it and/or
+#modify it under the terms of the GNU General Public License
+#as published by the Free Software Foundation; either version 2
+#of the License, or (at your option) any later version.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+#along with this program; if not, write to the Free Software
+#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+from pynswers.Answers import Answers
+
+import urllib
+import urllib2
+import string
+import random
+import re
+import simplejson
+import sys
+import logging
+import pickle
+import os
+import traceback
+
+from comofas_credentials import *
+from comofas_settings import *
+
+class ComofasAnswers(object):
+    """
+    Esta classe encontra a resposta e atualiza o arquivo de persistencia com as respostas
+    """
+    def __init__(self, answers_connection=None):
+    
+        self.answers_connection = answers_connection
+
+        logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG,)
+
+        if PICKLE_FILENAME in os.listdir('.'):
+            self.picklefd = open(PICKLE_FILENAME,"rb")
+            try:
+                self.persisted_questions = pickle.load(self.picklefd)
+            except EOFError:
+                self.persisted_questions = []
+            finally:
+                self.picklefd.close()
+        else:
+            self.picklefd =open(PICKLE_FILENAME,'wb')
+            pickle.dump(self.persisted_questions,self.picklefd)
+            self.picklefd.close()
+            self.picklefd =open(PICKLE_FILENAME,'wb')
+            self.persisted_questions = pickle.load(self.picklefd)
+            self.picklefd.close()
+
+    def persist_questions(self):
+        self.picklefd =open(PICKLE_FILENAME,"wb")
+        pickle.dump(self.persisted_questions,self.picklefd)
+        self.picklefd.close()
+
+  
+    def clean_text(self,txt):
+        """
+        vou falar o obvio: este metodo limpa o texto
+        """
+        pergunta = string.lower(txt)
+        
+        #pergunta = re.sub('#\w+',' ', txt)
+        pergunta = re.sub('@\w+',' ', pergunta)
+
+        pergunta = pergunta.replace(KEYWORD,' ')
+        
+        s=string.find(pergunta,'http://')
+        if s != -1:
+            t = string.find(pergunta[s+7:], " ")
+            pergunta = string.replace(pergunta,pergunta[s:t+s+7],' ')
+
+        replacements = [(u'é','e'),(u'á','a'),(u'í','i'),(u'ó','o'),(u'ú','u'),(u'â','a'),(u'ê','e'),(u'ô','o'),(u'ã','a'),(u'õ','o'),(u'ç','c')]
+
+        for a, b in replacements:
+            pergunta = pergunta.replace(a, b)
+        for punct in string.punctuation:
+            pergunta = pergunta.replace(punct, ' ')
+
+        pergunta = re.sub('\n',' ', pergunta)
+        pergunta = re.sub('\s\s',' ', pergunta)
+
+        pergunta = string.joinfields(pergunta.split(), ' OR ')
+
+        return pergunta.encode("utf-8")
+
+    def nao_entendi(self,usuario):
+        """
+        este metodo pede que o usuario pergunte de outra forma
+        """
+        nao_entendi_lista=[u"Não entendi mt bem, vc poderia me explicar?",u"Vc poderia explicar melhor?", u"Hã? Repete aí, mas fala diferente."]
+        mensagem = self.build_message(to_user=usuario, raw_answer=nao_entendi_lista[random.randint(0,len(nao_entendi_lista)-1)])
+        status = "sem status"
+        try:
+            status = self.twitter_connection.PostUpdate(mensagem)
+        except urllib2.HTTPError:
+            logging.debug("answer exception:" + str(traceback.print_exc()))
+        print '-' * 50
+        print "twitter.fasassim(nao entendi) postUpdate: %s - status:%s" % (mensagem,status)
+
+    def build_message(self,to_user, raw_answer):
+        """
+        este metodo:
+        remove palavras que nao ficam bem no final da frase.
+        limita o tamanho da frase
+        """
+        chars = 140
+        msg = "@%s " % to_user
+        chars = chars - len(msg)
+        if len(raw_answer) < chars:
+            return "%s%s" % (msg, raw_answer)
+        last = 0
+        for indice, c in enumerate(raw_answer):
+            if indice > chars:
+                break
+            if c == " ":
+                last = indice
+        # Lista de palavras que não ficam bem no fim da frase.
+        dispensaveis = ['e','a','o','de','da','do','na','no','para','pra','mas','se','que','como','porque']
+        a = raw_answer[:last].split()
+        while a[-1] in dispensaveis:
+            a.pop(-1)
+        return "%s%s" % (msg, " ".join(a))
+
+    
+    def answer_tweets(self):
+        """
+        este metodo busca as perguntas do twitter, limpa o texto e processa a
+        resposta para cada pergunta UMA única vez
+
+        """
+        try:
+            for question in self.persisted_questions:
+                #Detecta se a pergunta já foi respondida
+                try:
+                    if question['answered'] == True:                
+                        continue
+                except KeyError:
+                    question['answered'] = False
+                cleaned_question = self.clean_text(question['text'])                
+                
+                try:
+                    answer = self.answers_connection.questionSearch({'query':cleaned_question, 'search_in':'question', 'region': REGION, 'type':'resolved', 'results':'1'})
+                    question['answered'] = True
+                    question['answer'] = answer[0]
+                    question['twittered'] = False
+                    
+                    if DEBUG: print question                   
+
+                except:
+                    logging.debug("answer exception:" + str(traceback.print_exc()))
+                    continue
+            self.persist_questions()
+
+        except KeyboardInterrupt:
+            self.persist_questions()
+            sys.exit(1)
+
+
+if __name__=='__main__':
+    answers = Answers()
+    answers.appid = answers_appid
+
+    comofas = ComofasAnswers(answers_connection = answers)
+    comofas.answer_tweets()

comofas_questions.py

+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Comofas - everyone deserves an answer from Yahoo!Answers and Twitter
+# Yahoo!Answers and Twitter are trademarks from Yahoo! Inc and Twitter respectively.
+#
+#Copyright (C) 2008  Pedro Valente
+#
+#This program is free software; you can redistribute it and/or
+#modify it under the terms of the GNU General Public License
+#as published by the Free Software Foundation; either version 2
+#of the License, or (at your option) any later version.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+#along with this program; if not, write to the Free Software
+#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+import urllib
+import re
+import simplejson
+import sys
+import logging
+import pickle
+import os
+import traceback
+
+from twitter import twitter
+from comofas_credentials import *
+from comofas_settings import *
+
+
+class ComofasQuestions(object):
+    """
+    Esta classe cuida apenas de perguntas
+    """
+    def __init__(self,twitter_connection=None):
+        """
+        o twitter_connection pode ser substituido por outra classe
+        para executar testes offline.
+        """
+        self.twitter_connection = twitter_connection
+        # Esta linha não funciona, não descobrimos pq
+        # self.twitter_connection.SetXTwitterHeaders('fasassim',"http://twitter.com/fasassim",'Ni!')
+        self.search_url = 'http://search.twitter.com/search.json?q=' + KEYWORD
+
+        logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG,)
+
+        if PICKLE_FILENAME in os.listdir('.'):
+            self.picklefd = open(PICKLE_FILENAME,"rb")
+            try:
+                self.persisted_questions = pickle.load(self.picklefd)
+            except EOFError:
+                self.persisted_questions = []
+            finally:
+                self.picklefd.close()
+        else:
+            self.picklefd =open(PICKLE_FILENAME,'wb')
+            pickle.dump(self.persisted_questions,self.picklefd)
+            self.picklefd.close()
+            self.picklefd =open(PICKLE_FILENAME,'wb')
+            self.persisted_questions = pickle.load(self.picklefd)
+            self.picklefd.close()
+
+    def persist_questions(self):
+        self.picklefd =open(PICKLE_FILENAME,"wb")
+        pickle.dump(self.persisted_questions,self.picklefd)
+        self.picklefd.close()
+
+
+    def get_questions(self):
+        """
+        Este metodo retorna todos os resultados, sem filtrar por text e from_user
+        """
+        feed = urllib.urlopen('http://search.twitter.com/search.json?q=' + KEYWORD)
+        updates = simplejson.loads(feed.read())
+        return updates['results'] 
+
+            
+    def update_questions(self):
+        """
+            Este método vai ao twitter UMA vez, traz as perguntas novas e persiste no arquivo.
+        """
+        
+        #traz questions do twitter
+        questions = self.get_questions() 
+                 
+        
+        #descobre as não gravadas, ou seja, as novas
+        question_ids = []        
+        for q in questions:
+            question_ids.append(q['id'])
+        
+        persisted_ids = []
+        for p in self.persisted_questions:
+            persisted_ids.append(p['id'])
+        
+        new_ids = set(question_ids) - set(persisted_ids)
+        
+        if new_ids:
+            #persiste a lista incluindo as novas
+            for q in questions:
+                if q['id'] in new_ids:    
+                    self.persisted_questions.append(q)
+                    
+                    if DEBUG: print q
+            
+            self.persist_questions()
+        
+
+if __name__=='__main__':
+    twitter_api = twitter.Api(username=TWITTER_USERNAME, password=TWITTER_PASSWORD)
+
+    comofas = ComofasQuestions(twitter_connection = twitter_api)
+    comofas.update_questions()

comofas_sequencia.py

+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from comofas_questions import ComofasQuestions
+from comofas_answers import ComofasAnswers
+from comofas_twitterer import ComofasTwitterer
+from pynswers.Answers import Answers
+from twitter import twitter
+
+from comofas_credentials import *
+from comofas_settings import *
+
+if __name__=='__main__':
+    
+    #Puxa questions
+    twitter_api = twitter.Api(username=TWITTER_USERNAME, password=TWITTER_PASSWORD)
+
+    questions = ComofasQuestions(twitter_connection = twitter_api)
+    questions.update_questions()
+    
+    #Responde
+    answers = Answers()
+    answers.appid = answers_appid
+    comofas = ComofasAnswers(answers_connection = answers)
+    comofas.answer_tweets()
+
+    #Posta
+    comofas = ComofasTwitterer(twitter_connection = twitter_api)
+    comofas.post_tweets()

comofas_twitterer.py

+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Comofas - everyone deserves an answer from Yahoo!Answers and Twitter
+# Yahoo!Answers and Twitter are trademarks from Yahoo! Inc and Twitter respectively.
+#
+#Copyright (C) 2008  Pedro Valente
+#
+#This program is free software; you can redistribute it and/or
+#modify it under the terms of the GNU General Public License
+#as published by the Free Software Foundation; either version 2
+#of the License, or (at your option) any later version.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+#along with this program; if not, write to the Free Software
+#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+import urllib
+import re
+import simplejson
+import sys
+import logging
+import pickle
+import os
+import traceback
+
+from twitter import twitter
+from comofas_credentials import *
+from comofas_settings import *
+
+
+class ComofasTwitterer(object):
+    """
+    Esta classe cuida apenas formatar a resposta e mandar pro usuário via twitter
+    """
+    def __init__(self,twitter_connection=None):
+        """
+        o twitter_connection pode ser substituido por outra classe
+        para executar testes offline.
+        """
+        self.twitter_connection = twitter_connection
+
+        
+
+        logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG,)
+
+        if PICKLE_FILENAME in os.listdir('.'):
+            self.picklefd = open(PICKLE_FILENAME,"rb")
+            try:
+                self.persisted_questions = pickle.load(self.picklefd)
+            except EOFError:
+                self.persisted_questions = []
+            finally:
+                self.picklefd.close()
+        else:
+            self.picklefd =open(PICKLE_FILENAME,'wb')
+            pickle.dump(self.persisted_questions,self.picklefd)
+            self.picklefd.close()
+            self.picklefd =open(PICKLE_FILENAME,'wb')
+            self.persisted_questions = pickle.load(self.picklefd)
+            self.picklefd.close()
+
+    def persist_questions(self):
+        self.picklefd =open(PICKLE_FILENAME,"wb")
+        pickle.dump(self.persisted_questions,self.picklefd)
+        self.picklefd.close()
+
+
+    def build_message(self,to_user, raw_answer):
+        """
+        este metodo:
+        remove palavras que nao ficam bem no final da frase.
+        limita o tamanho da frase
+        """
+        chars = 140
+        msg = "@%s " % to_user
+        chars = chars - len(msg)
+        if len(raw_answer) < chars:
+            return "%s%s" % (msg, raw_answer)
+        last = 0
+        for indice, c in enumerate(raw_answer):
+            if indice > chars:
+                break
+            if c == " ":
+                last = indice
+        # Lista de palavras que não ficam bem no fim da frase.
+        dispensaveis = ['e','a','o','de','da','do','na','no','para','pra','mas',
+                        'em', u'até', 'se','que','como','porque', ',',':']
+        a = raw_answer[:last].split()
+        while a[-1] in dispensaveis:
+            a.pop(-1)
+        
+        return "%s%s" % (msg, " ".join(a))
+
+    def add_new_friends(self):
+        added_friends = list()
+        try:
+            added_friends = self.twitter_connection.GetFriends()
+            for friend in added_friends:
+                if not self.last_users.has_key(friend.name) and friend.name != 'fasassim':
+                    try:
+                        self.twitter_connection.CreateFriendship(friend)
+                        print "twitter.fasassim.following %s" % friend.name
+                        self.last_users.pop(friend.name)
+                    except:
+                        print "cannot create friendship for " + friend.name
+        except urllib2.HTTPError:
+            logging.debug("answer exception:" + str(traceback.print_exc()))
+        finally:
+            self.last_users = dict()
+
+    def post_tweets(self):
+        """
+        este metodo processa cada resposta e envia ao usuário
+        faz isso com apenas 3 por vez
+        """
+        try:      
+            contador_de_posts = 0
+            
+            fasassim_timeline =self.twitter_connection.GetUserTimeline(TWITTER_USERNAME)
+            fasassim_updates = [ up.text for up in fasassim_timeline ]
+          
+            for question in self.persisted_questions:
+                try:
+                # Estabelece que a pergunta tenha uma resposta
+                # Ignora o que ainda não foi processado
+                    if question['answered'] == False:                
+                        continue
+                except KeyError:
+                    continue
+                
+                try:
+                    if question['twittered'] == True:
+                        continue
+                except KeyError:
+                    question['twittered'] = False
+                
+                # Beleza, temos uma pergunta respondida e não tuitada.
+                # Vamos em frente
+                
+                if contador_de_posts <= 2:
+                    
+                    to_user = question['from_user']
+                    raw_answer = question['answer']['ChosenAnswer']
+                    update = self.build_message(to_user, raw_answer)
+                    
+                    # Salvaguarda antes de mandar, vamos checar se nos últimos
+                    # updates o fasassim já mandou um tweet idêntico.
+                    # Se tiver mandado, aborta o post e atualiza a persistencia
+
+                    if not update in fasassim_updates:
+                        status = self.twitter_connection.PostUpdate(update)
+                    question['twittered'] = True
+                    question['posted_tweet'] = update                    
+                    contador_de_posts = contador_de_posts + 1
+                    
+                    if DEBUG: print update
+                else:
+                    if DEBUG: print "Fica para a próxima iteração"
+            self.persist_questions()
+                   
+        except KeyboardInterrupt:
+            self.persist_questions()
+            sys.exit(1)
+        
+
+if __name__=='__main__':
+    twitter_api = twitter.Api(username=TWITTER_USERNAME, password=TWITTER_PASSWORD)
+
+    comofas = ComofasTwitterer(twitter_connection = twitter_api)
+    comofas.post_tweets()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.