# coding=utf-8

# Copyright (c) 2017 Malte Flender, Carsten Gips and Christian Carsten Sander
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this 
# software and associated documentation files (the "Software"), to deal in the Software 
# without restriction, including without limitation the rights to use, copy, modify, merge, 
# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons 
# to whom the Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all copies or 
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 
# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
# DEALINGS IN THE SOFTWARE.

# Python: Version 2.7.13
# NLTK: Version 3.0.3
# scikit-learn: Version 0.18

import nltk 
import sys
import collections
import gc
import time

from nltk.corpus import BracketParseCorpusReader
from nltk.corpus import stopwords
from nltk.classify.scikitlearn import SklearnClassifier

from multiprocessing import Pool

from sklearn import svm
from sklearn import model_selection
from sklearn import naive_bayes

# @ the moment not in use.
# from nltk import word_tokenize

# The Path to the corpus files and the filenames.
corpus_root_path = "/path/to/the/corpus/SentimentDataset/Data/"
file_pattern_names = ["de_sentiment.tsv", "de_sentiment_agree2.tsv", "de_sentiment_agree3.tsv"]

# To cope with the German umlauts.
reload(sys)
sys.setdefaultencoding("utf8")

stpwrds = stopwords.words("german")  # Use the German stopwords.

# We added some additional stopwords.
stpwrds.extend([',', '.', '(', ')', '!', ';', '-', ':', '...',
                "''", "'", "``", '~http', '?', '@', '#', '/',
                '&', '--', '[', ']', '=', '||', '+', '_', '..',
                '%', '|', '~', 'http'])

# Use the German version of the snowball stemmer.
stemmer = nltk.stem.SnowballStemmer("german")

NUM_MOST_COMMON_WORDS = 2000  # The number of the most common n-grams to be used.
NUM_FOLDS = 10  # The number of folds used for the crossvalidation

# Generate from min- to max-grams, e.g. uni- and bi-grams.
MIN_N_GRAM = 1  # The starting point of the n-gram generation, e.g. 1 -> unigram.
MAX_N_GRAM = 2  # The end point of the n-gram generation, e.g. 2 -> bigram.

MAXENT_MAX_ITER = 20  # The max number of iterations the max entropy classifier performs.

SHUFFLE = True  # If True the features will be shuffled before classification.
SEED = 4711  # The seed used in the shuffle prior to the classification.

CORES = 8 # The number of logical CPU-cores.

# Open the file with the given ID and return it.
def open_corpus(file_id):
    reader = BracketParseCorpusReader(corpus_root_path, file_pattern_names)
    file_content = reader.raw(fileids=reader.fileids()[file_id]).splitlines()
    return file_content

# Builds the sparse feature vector for a single Tweet from the Tweet,
#  the sentiment and all words from the corpus.
# The tweet contains of a list where the first part is the list of n-grams and
#  the second or last part is the given sentiment.
# The returning vector contains mostly n-grams that are not in the Tweet so most of it will be False.
def feature_extractor_sparse(tweet, all_words_from_the_corpus):
    sentiment = tweet[-1]
    dictinary = {}
    for word in all_words_from_the_corpus:
        dictinary[word] = (word in tweet[0])
    return (dictinary,) + (sentiment,)

# Builds the dense feature vector for a single Tweet from the Tweet and the sentiment.
# The returning vector contains only the n-grams from the given Tweet.
def feature_extractor_dense(n_grams, sentiment):
    dictinary = {}
    for n_gram in n_grams:
        dictinary.update({n_gram : True})
    return (dictinary,) + (sentiment,)

# Generate the from min_len to max_len grams, e.g. uni- and bi-grams.
def everygrams(sequence, min_len=1, max_len=-1):
    if max_len == -1:
        max_len = len(sequence)
    for n in range(min_len, max_len + 1):
        for ng in nltk.ngrams(sequence, n):
            yield ng

# Creates the train and test set from the corpus.
def generate_sets_sparse(file_id):
    features = []
    all_words = []
    tweets = []
    tweet_ID_last = ""

    for line in open_corpus(file_id):
        ignore = False
        parts_of_the_table = line.split("\t")
        
        if len(parts_of_the_table) == 4:  # some lines from the corpus are broken and lead to a fourth class. So only perform on working lines.
            sentiment = parts_of_the_table[0]  # split into columns.
            tweet_ID = parts_of_the_table[2]  # save the sentiment.
            
            if (tweet_ID == tweet_ID_last) and (tweets[-1][-1] == sentiment):
                ignore = True  # In the first set the sentiments are duplicate, so remove them.
                               # In the other files it woun'd change a bit.
            tweet_ID_last = tweet_ID
            
            if (sentiment != "na") and not ignore:
                tokens = nltk.tokenize.WordPunctTokenizer().tokenize(parts_of_the_table[-1].lower())  # Tokenize the feature and set it to lowercase.
                # tokens = nltk.word_tokenize(parts_of_the_table[-1].lower()) # Alternative tokenizer, worse performance.
                small_tokens = [i for i in tokens if i not in stpwrds]  # Remove all stopwords and additional stopwords.
                stem_tokens = [stemmer.stem(i) for i in small_tokens]  # Stemming the feature optional step.
                n_grams = everygrams(stem_tokens, MIN_N_GRAM, MAX_N_GRAM)  # Generate the n-grams.
                
                tmp_tweet_words = []
                for n_gram in n_grams:
                    tmp_tweet_words.append(n_gram)
                    if n_gram not in all_words:
                        all_words.append(n_gram)  # Append to the list with all words.

                tweets.append([tmp_tweet_words, sentiment])  # Generate a list of the feature and the sentiment.
            
    f_dist = list(nltk.FreqDist(all_words))[:NUM_MOST_COMMON_WORDS]  # Use only the k most common n-grams.
    tweets.pop(0)  # Delete the first line since it contains the header.
    
    for tweet in tweets:  # Generate the feature vector for every feature.
        for n_gram in tweet[0]:
            if n_gram not in f_dist:  # Shrink the soon to be feature vector to the k most common n-grams.
                tweet[0].remove(n_gram)
        feature = feature_extractor_sparse(tweet, f_dist)
        features.append(feature)

    return features

# Creates the train and test set from the corpus.
def generate_sets_dense(file_id):
    features = []
    tweet_ID_last = ""
    
    for line in open_corpus(file_id):
        ignore = False
        parts_of_the_table = line.split("\t")
        if len(parts_of_the_table) == 4:
            sentiment = parts_of_the_table[0]
            tweet_ID = parts_of_the_table[2]
            
            if (tweet_ID == tweet_ID_last) and (features[-1][-1] == sentiment):
                ignore = True
            
            tweet_ID_last = tweet_ID
            
            if (sentiment != "na") and not ignore:
                tokens = nltk.tokenize.WordPunctTokenizer().tokenize(parts_of_the_table[-1].lower())
                small_tokens = [i for i in tokens if i not in stpwrds]
                stem_tokens = [stemmer.stem(i) for i in small_tokens]
                n_grams = everygrams(stem_tokens, MIN_N_GRAM, MAX_N_GRAM)
                feature = feature_extractor_dense(n_grams, sentiment)
                features.append(feature)
            
    features.pop(0)
    return features

# Generate some statistics from the classification
def get_stats_cross(train_set, test_set, classifier, out_list):
    ref_set = collections.defaultdict(set)
    test_set_new = collections.defaultdict(set)
 
    for i, (feature_vector, label) in enumerate(test_set):
        ref_set[label].add(i)
        observed = classifier.classify(feature_vector)
        test_set_new[observed].add(i)
        
# Print wrong labeled occurrences:
#        if observed != label:
#            print(str(observed) + ":\t" + str(label))

    out_list[0] = out_list[0] + float(len(train_set) or 0.0)
    out_list[1] = out_list[1] + float(len(test_set) or 0.0)
    out_list[2] = out_list[2] + float(nltk.classify.accuracy(classifier, test_set) or 0.0)

    out_list[3] = out_list[3] + float(nltk.metrics.precision(ref_set["positive"], test_set_new["positive"]) or 0.0)
    out_list[4] = out_list[4] + float(nltk.metrics.recall(ref_set["positive"], test_set_new["positive"]) or 0.0)
    out_list[5] = out_list[5] + float(nltk.metrics.f_measure(ref_set["positive"], test_set_new["positive"]) or 0.0)

    out_list[6] = out_list[6] + float(nltk.metrics.precision(ref_set["negative"], test_set_new["negative"]) or 0.0)
    out_list[7] = out_list[7] + float(nltk.metrics.recall(ref_set["negative"], test_set_new["negative"]) or 0.0)
    out_list[8] = out_list[8] + float(nltk.metrics.f_measure(ref_set["negative"], test_set_new["negative"]) or 0.0)

    out_list[9] = out_list[9] + float(nltk.metrics.precision(ref_set["neutral"], test_set_new["neutral"]) or 0.0)
    out_list[10] = out_list[10] + float(nltk.metrics.recall(ref_set["neutral"], test_set_new["neutral"]) or 0.0)
    out_list[11] = out_list[11] + float(nltk.metrics.f_measure(ref_set["neutral"], test_set_new["neutral"]) or 0.0)

# Print the generated statistics to stdout
def print_stats(txt, results):
    out = txt + "\n"
    out += "Num Train: " + str(results[0] / NUM_FOLDS) + "\n"
    out += "Num Test: " + str(results[1] / NUM_FOLDS) + "\n"
    out += "Accuracy: " + str(results[2] / NUM_FOLDS) + "\n"
    out += "\n"
    out += "Time used: " + str(time.time() - start_time) + "\n"
    out += "\n"
    out += "positive precision: " + str(results[3] / NUM_FOLDS) + "\n"
    out += "positive recall:    " + str(results[4] / NUM_FOLDS) + "\n"
    out += "positive F-measure: " + str(results[5] / NUM_FOLDS) + "\n"
    out += "\n"
    out += "negative precision: " + str(results[6] / NUM_FOLDS) + "\n"
    out += "negative recall:    " + str(results[7] / NUM_FOLDS) + "\n"
    out += "negative F-measure: " + str(results[8] / NUM_FOLDS) + "\n"
    out += "\n"
    out += "neutral precision: " + str(results[9] / NUM_FOLDS) + "\n"
    out += "neutral recall:    " + str(results[10] / NUM_FOLDS) + "\n"
    out += "neutral F-measure: " + str(results[11] / NUM_FOLDS) + "\n"
    out += "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n"
    print(out)

# The general classification process.
# The concrete algorithm will be provided external
def classify(content):
    
    # To use the map function only one argument can be passed, so it needs to be splitted in its parts.
    classifier_blank = content[0] # The encapsulated train function of the actual classifier.
    corpus = content[1]
    txt = content[2]
    
    results = [0.0] * 12  # The amount of the statistic values

    cv = model_selection.KFold(n_splits=NUM_FOLDS, shuffle=SHUFFLE, random_state=SEED)
    
    for train_index, test_index in cv.split(corpus):
 
        train_values = []
        test_values = []
        
        for i in train_index:
            train_values.append(corpus[i])
        
        for i in test_index:
            test_values.append(corpus[i])
        
        classifier = classifier_blank(train_values)
        
        get_stats_cross(train_values, test_values, classifier, results)
    
    print_stats(txt, results)

# In order to get the baseline performance for the other classifiers
#  this function computes a sort of ZeroR-classification for all sentiments.
def classify_ZeroR(content):
    corpus = content[1]
    txt = content[2]
    
    neutral = 0
    positive = 0
    negative = 0
    
    for i in corpus:
        if i[1] == "neutral":
            neutral += 1
        elif i[1] == "positive":
            positive += 1
        elif i[1] == "negative":
            negative += 1
        else:
            print("ERROR: classify_ZeroR")
            return
 
    out = txt + "\n"
    out += "Amount neutral: " + str(float(neutral) / len(corpus)) + "\n"
    out += "Amount positive:  " + str(float(positive) / len(corpus)) + "\n"
    out += "Amount negative: " + str(float(negative) / len(corpus)) + "\n"
    out += "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––"
    print(out)

# To test the distribution of the classes for every fold
def classify_distribution_test(content):
    corpus = content[1]
    txt = content[2]

    cv = model_selection.KFold(n_splits=NUM_FOLDS, shuffle=SHUFFLE, random_state=SEED)
    
    for train_index, test_index in cv.split(corpus):
 
        train_values = []
        test_values = []
        
        neutral_train = 0
        positive_train = 0
        negative_train = 0
        
        neutral_test = 0
        positive_test = 0
        negative_test = 0
        
        for i in train_index:
            train_values.append(corpus[i])
        
        for i in test_index:
            test_values.append(corpus[i])

        for i in train_values:
            if i[1] == "neutral":
                neutral_train += 1
            elif i[1] == "positive":
                positive_train += 1
            elif i[1] == "negative":
                negative_train += 1
            else:
                print("ERROR: classify_classify_distribution_test train_values")
                return

        for i in test_values:
            if i[1] == "neutral":
                neutral_test += 1
            elif i[1] == "positive":
                positive_test += 1
            elif i[1] == "negative":
                negative_test += 1
            else:
                print("ERROR: classify_classify_distribution_test test_values")
                return

        out = "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n"
        out += txt + "\n"
        out += "Amount neutral train: " + str(float(neutral_train) / len(train_values)) + "\n"
        out += "Amount positive train:  " + str(float(positive_train) / len(train_values)) + "\n"
        out += "Amount negative train: " + str(float(negative_train) / len(train_values)) + "\n"
        out += "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n"
        out += "Amount neutral test: " + str(float(neutral_test) / len(test_values)) + "\n"
        out += "Amount positive test:  " + str(float(positive_test) / len(test_values)) + "\n"
        out += "Amount negative test: " + str(float(negative_test) / len(test_values)) + "\n"
        out += "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n"
        
        print(out)

# The encapsulated train function of the Naive Bayes classifier
def NB_classifier(train_set):
    classifier = SklearnClassifier(naive_bayes.MultinomialNB(
        alpha = 1.0,
        fit_prior = True
    ))
    classifier.train(train_set)
    
    return classifier

# The encapsulated train function of the decision tree classifier
def DT_classifier(train_set):
    classifier = nltk.DecisionTreeClassifier.train(train_set)
    return classifier

# The encapsulated train function of the maximum entropy classifier
def ME_classifier(train_set):

    classifier = nltk.MaxentClassifier.train(
        train_set, 
        max_iter = MAXENT_MAX_ITER,
        trace = 0,
        algorithm = 'MEGAM'
    )
    return classifier

# The encapsulated train function of the support vector machine classifier
def SVM_classifier(train_set):
    
# Alternative SVM wit slightly worse performance
#     classifier = SklearnClassifier(svm.SVC(
#         C = 100000.0, 
#         kernel = 'linear', 
#         degree = 3, 
#         gamma = 'auto', 
#         coef0 = 0.0, 
#         shrinking = True, 
#         probability = False, 
#         tol = 1.0, 
#         cache_size = 200, 
#         class_weight = None, 
#         verbose = False, 
#         max_iter = -1, 
#         decision_function_shape = None, 
#         random_state = None
#     ))

    classifier = SklearnClassifier(svm.LinearSVC(
        C = 1.0,
        class_weight = None,
        dual = True,
        fit_intercept = True,
        intercept_scaling = 0.1,
        loss = 'squared_hinge',
        max_iter = 1000,
        multi_class = 'ovr',
        penalty = 'l2',
        random_state = None,
        tol = 1.0,
        verbose = 0
    ))
    
    classifier.train(train_set)
    return classifier

# Get the sparse and dense features and perform the classification with multicore support.
# To save some RAM one could perform the sparse and dense features in two separate runs.

start_time = time.time()
de_sentiment_sparse = generate_sets_sparse(0)
print("de_sentiment_sparse generation: " + str(time.time()-start_time))

start_time = time.time()
de_sentiment_dense = generate_sets_dense(0)
print("de_sentiment_dense generation: " + str(time.time()-start_time))

start_time = time.time()
de_sentiment_agree2_sparse = generate_sets_sparse(1)
print("de_sentiment_agree2_sparse generation: " + str(time.time()-start_time))

start_time = time.time()
de_sentiment_agree2_dense = generate_sets_dense(1)
print("de_sentiment_agree2_dense generation: " + str(time.time()-start_time))

start_time = time.time()
de_sentiment_agree3_sparse = generate_sets_sparse(2)
print("de_sentiment_agree3_sparse generation: " + str(time.time() - start_time))

start_time = time.time()
de_sentiment_agree3_dense = generate_sets_dense(2)
print("de_sentiment_agree3_dense generation: " + str(time.time() - start_time) + "\n")

gc.collect()  # Trigger the garbage collector to free a little RAM.
pool = Pool(CORES)

start_time = time.time()

pool.map_async(classify, [
    (NB_classifier, de_sentiment_sparse, "Naive sparse: de_sentiment.tsv",),
    (NB_classifier, de_sentiment_agree2_sparse, "Naive sparse: de_sentiment_agree2.tsv",),
    (NB_classifier, de_sentiment_agree3_sparse, "Naive sparse: de_sentiment_agree3.tsv",),
    (NB_classifier, de_sentiment_dense, "Naive dense: de_sentiment.tsv",),
    (NB_classifier, de_sentiment_agree2_dense, "Naive dense: de_sentiment_agree2.tsv",),
    (NB_classifier, de_sentiment_agree3_dense, "Naive dense: de_sentiment_agree3.tsv",),
    (SVM_classifier, de_sentiment_sparse, "SVM sparse: de_sentiment.tsv",),
    (SVM_classifier, de_sentiment_agree2_sparse, "SVM sparse: de_sentiment_agree2.tsv",),
    (SVM_classifier, de_sentiment_agree3_sparse, "SVM sparse: de_sentiment_agree3.tsv",),
    (SVM_classifier, de_sentiment_dense, "SVM dense: de_sentiment.tsv",),
    (SVM_classifier, de_sentiment_agree2_dense, "SVM dense: de_sentiment_agree2.tsv",),
    (SVM_classifier, de_sentiment_agree3_dense, "SVM dense: de_sentiment_agree3.tsv",),
    (ME_classifier, de_sentiment_sparse, "MaxEnt sparse: de_sentiment.tsv",),
    (ME_classifier, de_sentiment_agree2_sparse, "MaxEnt sparse: de_sentiment_agree2.tsv",),
    (ME_classifier, de_sentiment_agree3_sparse, "MaxEnt sparse: de_sentiment_agree3.tsv",),
    (ME_classifier, de_sentiment_dense, "MaxEnt dense: de_sentiment.tsv",),
    (ME_classifier, de_sentiment_agree2_dense, "MaxEnt dense: de_sentiment_agree2.tsv",),
    (ME_classifier, de_sentiment_agree3_dense, "MaxEnt dense: de_sentiment_agree3.tsv",),
    (DT_classifier, de_sentiment_sparse, "Tree sparse: de_sentiment.tsv",),
    (DT_classifier, de_sentiment_agree2_sparse, "Tree sparse: de_sentiment_agree2.tsv",),
    (DT_classifier, de_sentiment_agree3_sparse, "Tree sparse: de_sentiment_agree3.tsv",),
    (DT_classifier, de_sentiment_dense, "Tree dense: de_sentiment.tsv",),
    (DT_classifier, de_sentiment_agree2_dense, "Tree dense: de_sentiment_agree2.tsv",),
    (DT_classifier, de_sentiment_agree3_dense, "Tree dense: de_sentiment_agree3.tsv",)
])

pool.close()
pool.join()