__author__ = 'kin_chun'
import numpy as np
from flask import Flask, request, jsonify, render_template, make_response , url_for,redirect
import joblib
import os
import io
import base64
import csv
import pandas
import pydotplus
from pandas.plotting import scatter_matrix
import matplotlib.pyplot as plt
from sklearn import model_selection
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.ensemble import RandomForestClassifier
from six import StringIO
from IPython.display import Image
from sklearn.tree import export_graphviz
import pydotplus
import warnings
import os.path
from sklearn import metrics
from sklearn.metrics import plot_confusion_matrix

warnings.filterwarnings('always')  # "error", "ignore", "always", "default", "module" or "once"
warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning)
warnings.filterwarnings("ignore")

#python to open csv for MAC
folder_MAC = 'C:/Users/Admin/Desktop/CSV file/Backup/Flask Testing/Display'
file_name_MAC = 'Master_MAC_Attack.csv'
file_path_MAC = os.path.join(folder_MAC, file_name_MAC)

#python to open csv for DHCP
folder_DHCP = 'C:/Users/Admin/Desktop/CSV file/Backup/Flask Testing/Display'
file_name_DHCP = 'Master_Spoofing_Attack.csv'
file_path_DHCP = os.path.join(folder_DHCP, file_name_DHCP)

#python to open csv for ICMP
folder_ICMP = 'C:/Users/Admin/Desktop/CSV file/Backup/Flask Testing/Display'
file_name_ICMP = 'Master_ICMP_Attack.csv'
file_path_ICMP = os.path.join(folder_ICMP, file_name_ICMP)

#python to open csv for SYN
folder_SYN = 'C:/Users/Admin/Desktop/CSV file/Backup/Flask Testing/Display'
file_name_SYN = 'Master_SYN_Attack.csv'
file_path_SYN = os.path.join(folder_SYN, file_name_SYN)

app = Flask(__name__)

def plot_classification_report(classificationReport,
                               title='Classification report',
                               cmap='RdBu'):
            classificationReport = classification_report(Y_validation, newpredictions_rfc)
            classificationReport = classificationReport.replace('\n\n', '\n')
            classificationReport = classificationReport.replace(' / ', '/')
            lines = classificationReport.split('\n')

            classes, plotMat, support, class_names = [], [], [], []
            for line in lines[1:]:  # if you don't want avg/total result, then change [1:] into [1:-1]
                t = line.strip().split()
                if len(t) < 2:
                    continue
                classes.append(t[0])
                v = [float(x) for x in t[1: len(t) - 1]]
                support.append(int(t[-1]))
                class_names.append(t[0])
                plotMat.append(v)

            plotMat = np.array(plotMat)
            xticklabels = ['Precision', 'Recall', 'F1-score']
            yticklabels = ['{0} ({1})'.format(class_names[idx], sup)
                           for idx, sup in enumerate(support)]

            plt.imshow(plotMat, interpolation='nearest', cmap='RdBu', aspect='auto')
            plt.title('Classification Report')
            plt.colorbar()
            plt.xticks(np.arange(3), xticklabels, rotation=45)
            plt.yticks(np.arange(len(classes)), yticklabels)

            upper_thresh = plotMat.min() + (plotMat.max() - plotMat.min()) / 10 * 8
            lower_thresh = plotMat.min() + (plotMat.max() - plotMat.min()) / 10 * 2
            for i, j in itertools.product(range(plotMat.shape[0]), range(plotMat.shape[1])):
                plt.text(j, i, format(plotMat[i, j], '.2f'),
                         horizontalalignment="center",
                         color="white" if (plotMat[i, j] > upper_thresh or plotMat[i, j] < lower_thresh) else "black")

            plt.ylabel('Metrics')
            plt.xlabel('Classes')
            plt.tight_layout()
            plt.show()

def plot_classification_report(cr, title='Classification report ', with_avg_total=False, cmap=plt.cm.Blues):

    lines = cr.split('\n')

    classes = []
    plotMat = []

    for line in lines[2 : (len(lines) - 3)]:
        #print(line)
        t = line.split()
        # print(t)
        if(len(t)==0):
            break
        else:
            classes.append(t[0])
            v = [float(x) for x in t[1: len(t) - 1]]
            print(v)
            plotMat.append(v)

    if with_avg_total:
        aveTotal = lines[len(lines) - 1].split()
        classes.append('avg/total')
        vAveTotal = [float(x) for x in t[1:len(aveTotal) - 1]]
        plotMat.append(vAveTotal)

    plt.imshow(plotMat, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    x_tick_marks = np.arange(3)
    y_tick_marks = np.arange(len(classes))
    plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
    plt.yticks(y_tick_marks, classes)
    plt.tight_layout()
    plt.ylabel('Classes')
    plt.xlabel('Measures')

@app.route('/')
def main():
    return render_template('main.html')

@app.route('/deploy', methods=['POST'])
def deploy():

    attackType = request.form["attack"]
    modelType = request.form['model']
    ratio = request.form['ratio']
    dataset = request.form['dataset']

    if modelType == "lr":
        classifier = LogisticRegression()
    elif modelType == "lda":
        classifier = LinearDiscriminantAnalysis()
    elif modelType == "knn":
        classifier = KNeighborsClassifier()
    elif modelType == "dt":
        classifier = DecisionTreeClassifier()

    if ratio == "ratio70":
        validation_size = 0.30
    elif ratio == "ratio80":
        validation_size = 0.20

    if dataset == "mac_dataset":
        file_path = file_path_MAC
    elif dataset == "dhcp_dataset":
        file_path = file_path_DHCP
    elif dataset == "syn_dataset":
        file_path = file_path_SYN
    elif dataset == "icmp_dataset":
        file_path = file_name_ICMP

    # import dataset
    if dataset == "mac_dataset":
        dataset = pandas.read_csv("Master_MAC_Attack.csv")
    elif dataset == "dhcp_dataset":
        dataset = pandas.read_csv("Master_Spoofing_Attack.csv")
    elif dataset == "syn_dataset":
        dataset = pandas.read_csv("Master_SYN_Attack.csv")
    elif dataset == "icmp_dataset":
        dataset = pandas.read_csv("Master_ICMP_Attack.csv")

    #(Array has to be defined to avoid UnboundLocalError)
    # split dataset
    array = dataset.values
    print (array)
    if attackType == "mac":
        X = array[:,0:6]
        Y = array[:,6]
    elif attackType == "dhcp":
        X = array[:,0:8]
        Y = array[:,8]
    elif attackType == "syn":
        X = array[:,0:8]
        Y = array[:,8]
    elif attackType == "icmp":
        X = array[:,0:8]
        Y = array[:,8]

    # split dataset
    # array = dataset.values
    # print (array)
    # X and Y has already been configure during the attackType
    #X = array[:,0:8]
    #Y = array[:,8]
    # Validation size has been configure during the ratio selection
    #validation_size = 0.30
    seed = 42

    # Split dataset into training set and test set
    X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=validation_size, random_state=seed)
    X_train_set, X_test, Y_train_set, Y_test = model_selection.train_test_split(X_train, Y_train, test_size=validation_size, random_state=seed)

    # Test options and evaluation metric
    scoring = 'accuracy'

    # Fit algorithm model

    models = []
    models.append(('LR', LogisticRegression()))
    models.append(('LDA', LinearDiscriminantAnalysis()))
    models.append(('KNN', KNeighborsClassifier()))
    models.append(('CART', DecisionTreeClassifier()))

    # evaluate each model in turn
    results = []
    names = []
    for name, model in models:
            kfold = model_selection.KFold(n_splits=5, shuffle=True, random_state=seed)
            cv_results = model_selection.cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring)
            results.append(cv_results)
            names.append(name)
            msg = "Highest Accuracy is %s with Accuracy: %f (+/- %f)" % (name, cv_results.mean(), cv_results.std())
            print(msg)

    #Compare Algorithms
    fig = plt.figure()
    fig.suptitle('Algorithm Comparison')
    ax = fig.add_subplot(111)
    plt.boxplot(results)
    ax.set_xticklabels(names)
    # plt.clf()
    # plt.cla()
    # plt.close()

    my_stringIObytes = io.BytesIO()
    plt.savefig(my_stringIObytes, format='png')
    my_stringIObytes.seek(0)
    my_base64_pngData = base64.b64encode(my_stringIObytes.read())
    data_uri = "data:image/png;base64," + my_base64_pngData.decode("ascii")

    #plt.show()

    if modelType == "lr":
        # Testing For Logistic Regression
            print("-------------------------------------------------------")

        # Make predictions on validation dataset
            print("\n Logistic Regression results on test set \n")
        # Create Logistic Regression classifer object
            logistic = LogisticRegression()
        # Train Logistic Regression Classifer
            logistic.fit(X_train_set, Y_train_set)
        #saving the model using joblib
            filename = 'logistic_finalized_DT_model.sav'
            joblib.dump(logistic, filename)
        # load the model from disk
            loaded_model = joblib.load(filename)
            result = loaded_model.score(X_test, Y_test)
            print (result)
        #Predict the response for test dataset
            predictions_rfc = logistic.predict(X_test)
            print("\nLogistic Regression accuracy test: \n")
            print(accuracy_score(Y_test, predictions_rfc))
            print(confusion_matrix(Y_test, predictions_rfc))
            print(classification_report(Y_test, predictions_rfc, zero_division=0))

            Train_Accuracy = accuracy_score(Y_test, predictions_rfc)
            Train_Confusion_Matrix = confusion_matrix(Y_test, predictions_rfc)
            Train_Classification_Report = classification_report(Y_test, predictions_rfc)
            # matrix = plot_confusion_matrix(logistic, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

        # Make predictions on test dataset
            print("\nLogistic Regression results on final validation \n")
            newlogistic = DecisionTreeClassifier()
            newlogistic.fit(X_train_set, Y_train_set)
            newpredictions_rfc = newlogistic.predict(X_validation)
            print("\nLogistic Regression accuracy validation: \n")
            print(accuracy_score(Y_validation, newpredictions_rfc))
            print(confusion_matrix(Y_validation, newpredictions_rfc))
            print(classification_report(Y_validation, newpredictions_rfc))
            df = dataset.reset_index(drop = False)

            Test_Accuracy = accuracy_score(Y_validation, newpredictions_rfc)
            Test_Confussion_Matrix = confusion_matrix(Y_validation, newpredictions_rfc)
            Test_Classification_Report = classification_report(Y_validation, newpredictions_rfc)
            matrix = plot_confusion_matrix(newlogistic, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

            # #Compare Algorithms
            # fig = plt.figure()
            # fig.suptitle('Confusion Matrix')
            # ax = fig.add_subplot(111)
            # plt.boxplot(results)
            # ax.set_xticklabels(names)

            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_matrix = "data:image/png;base64," + my_base64_pngData.decode("ascii")
            plt.close()

            # plot_classification_report(classification_report(Y_validation, newpredictions_rfc) )
            # The following is classification report coding to display visualization (heatmap only)

            lines = classification_report(Y_validation, newpredictions_rfc).split('\n')

            classes = []
            plotMat = []

            for line in lines[2 : (len(lines) - 3)]:
                #print(line)
                t = line.split()
                print(t)
                if(len(t)==0):
                    break
                else:
                    classes.append(t[0])
                    v = [float(x) for x in t[1: len(t) - 1]]
                    print(v)
                    plotMat.append(v)

            # if with_avg_total:
                # aveTotal = lines[len(lines) - 1].split()
                # classes.append('avg/total')
                # vAveTotal = [float(x) for x in t[1:len(aveTotal) - 1]]
                # plotMat.append(vAveTotal)

            plt.imshow(plotMat, interpolation='nearest', cmap=plt.cm.Blues)
            plt.title('Classification report ')
            plt.colorbar()
            x_tick_marks = np.arange(3)
            y_tick_marks = np.arange(len(classes))
            plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
            plt.yticks(y_tick_marks, classes)
            plt.tight_layout()
            plt.ylabel('Classes')
            plt.xlabel('Measures')
            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_report = "data:image/png;base64," + my_base64_pngData.decode("ascii")

            # plt.close()
            # Classification Report (Not accurate)
            #-----------------------------------------------------------------------------------
            #-----------------------------------------------------------------------------------
            # classificationReport = classification_report(Y_validation, newpredictions_rfc)
            # plot_classification_report(classificationReport)
            # plt.show()

            # Instantiate the classification model and visualizer

    elif modelType == "lda":
        # Testing For Linear Discriminant Analysis
            print("-------------------------------------------------------")
        # Make predictions on validation dataset
            print("\n Linear Discriminant Analysis results on 30% test set \n")
        # Create Logistic Regression classifer object
            lda = LinearDiscriminantAnalysis()
        # Train Linear Discriminant Analysis Classifer
            lda.fit(X_train_set, Y_train_set)
        #saving the model using joblib
            filename = 'lda_finalized_DT_model.sav'
            joblib.dump(lda, filename)
        # load the model from disk
            loaded_model = joblib.load(filename)
            result = loaded_model.score(X_test, Y_test)
            print (result)
        #Predict the response for test dataset
            predictions_rfc = lda.predict(X_test)
            print("\nLinear Discriminant Analysis accuracy test: \n")
            print(accuracy_score(Y_test, predictions_rfc))
            print(confusion_matrix(Y_test, predictions_rfc))
            print(classification_report(Y_test, predictions_rfc, zero_division=0))

            Train_Accuracy = accuracy_score(Y_test, predictions_rfc)
            Train_Confusion_Matrix = confusion_matrix(Y_test, predictions_rfc)
            Train_Classification_Report = classification_report(Y_test, predictions_rfc)
            # matrix = plot_confusion_matrix(lda, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

        # Make predictions on test dataset
            print("\nLinear Discriminant Analysis results on final 30% validation \n")
            newlda = DecisionTreeClassifier()
            newlda.fit(X_train_set, Y_train_set)
            newpredictions_rfc = newlda.predict(X_validation)
            print("\nLinear Discriminant Analysis accuracy validation: \n")
            print(accuracy_score(Y_validation, newpredictions_rfc))
            print(confusion_matrix(Y_validation, newpredictions_rfc))
            print(classification_report(Y_validation, newpredictions_rfc))
            df = dataset.reset_index(drop = False)

            Test_Accuracy = accuracy_score(Y_validation, newpredictions_rfc)
            Test_Confussion_Matrix = confusion_matrix(Y_validation, newpredictions_rfc)
            Test_Classification_Report = classification_report(Y_validation, newpredictions_rfc)
            matrix = plot_confusion_matrix(newlda, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

            # #Compare Algorithms
            # fig = plt.figure()
            # fig.suptitle('Confusion Matrix')
            # ax = fig.add_subplot(111)
            # plt.boxplot(results)
            # ax.set_xticklabels(names)

            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_matrix = "data:image/png;base64," + my_base64_pngData.decode("ascii")
            plt.close()

            lines = classification_report(Y_validation, newpredictions_rfc).split('\n')

            classes = []
            plotMat = []

            for line in lines[2 : (len(lines) - 3)]:
                #print(line)
                t = line.split()
                print(t)
                if(len(t)==0):
                    break
                else:
                    classes.append(t[0])
                    v = [float(x) for x in t[1: len(t) - 1]]
                    print(v)
                    plotMat.append(v)

            plt.imshow(plotMat, interpolation='nearest', cmap=plt.cm.Blues)
            plt.title('Classification report ')
            plt.colorbar()
            x_tick_marks = np.arange(3)
            y_tick_marks = np.arange(len(classes))
            plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
            plt.yticks(y_tick_marks, classes)
            plt.tight_layout()
            plt.ylabel('Classes')
            plt.xlabel('Measures')
            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_report = "data:image/png;base64," + my_base64_pngData.decode("ascii")

    elif modelType == "knn":
        # Testing For KNeighbors
            print("-------------------------------------------------------")

        # Make predictions on validation dataset
            print("\n KNeighbors results on 30% test set \n")
        # Create K Neighbors classifer object
            KNeighbors = KNeighborsClassifier()
        # Train KNeighbors Classifer
            KNeighbors.fit(X_train_set, Y_train_set)
        #saving the model using joblib
            filename = 'kneighbors_finalized_DT_model.sav'
            joblib.dump(KNeighbors, filename)
        # load the model from disk
            loaded_model = joblib.load(filename)
            result = loaded_model.score(X_test, Y_test)
            print (result)
        #Predict the response for test dataset
            predictions_rfc = KNeighbors.predict(X_test)
            print("\nK Neighbors accuracy test: \n")
            print(accuracy_score(Y_test, predictions_rfc))
            print(confusion_matrix(Y_test, predictions_rfc))
            print(classification_report(Y_test, predictions_rfc, zero_division=0))

            Train_Accuracy = accuracy_score(Y_test, predictions_rfc)
            Train_Confusion_Matrix = confusion_matrix(Y_test, predictions_rfc)
            Train_Classification_Report = classification_report(Y_test, predictions_rfc)
            # matrix = plot_confusion_matrix(KNeighbors, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

        # Make predictions on test dataset
            print("\nK Neighbors results on final 30% validation \n")
            newKNeighbors = DecisionTreeClassifier()
            newKNeighbors.fit(X_train_set, Y_train_set)
            newpredictions_rfc = newKNeighbors.predict(X_validation)
            print("\nK Neighbors accuracy validation: \n")
            print(accuracy_score(Y_validation, newpredictions_rfc))
            print(confusion_matrix(Y_validation, newpredictions_rfc))
            print(classification_report(Y_validation, newpredictions_rfc))
            df = dataset.reset_index(drop = False)

            Test_Accuracy = accuracy_score(Y_validation, newpredictions_rfc)
            Test_Confussion_Matrix = confusion_matrix(Y_validation, newpredictions_rfc)
            Test_Classification_Report = classification_report(Y_validation, newpredictions_rfc)
            matrix = plot_confusion_matrix(newKNeighbors, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

            # #Compare Algorithms
            # fig = plt.figure()
            # fig.suptitle('Confusion Matrix')
            # ax = fig.add_subplot(111)
            # plt.boxplot(results)
            # ax.set_xticklabels(names)

            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_matrix = "data:image/png;base64," + my_base64_pngData.decode("ascii")
            plt.close()

            lines = classification_report(Y_validation, newpredictions_rfc).split('\n')

            classes = []
            plotMat = []

            for line in lines[2 : (len(lines) - 3)]:
                #print(line)
                t = line.split()
                print(t)
                if(len(t)==0):
                    break
                else:
                    classes.append(t[0])
                    v = [float(x) for x in t[1: len(t) - 1]]
                    print(v)
                    plotMat.append(v)

            plt.imshow(plotMat, interpolation='nearest', cmap=plt.cm.Blues)
            plt.title('Classification report ')
            plt.colorbar()
            x_tick_marks = np.arange(3)
            y_tick_marks = np.arange(len(classes))
            plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
            plt.yticks(y_tick_marks, classes)
            plt.tight_layout()
            plt.ylabel('Classes')
            plt.xlabel('Measures')
            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_report = "data:image/png;base64," + my_base64_pngData.decode("ascii")

    elif modelType == "dt":
        # Make predictions on validation dataset
            print("\n CART results on 30% test set \n")
        # Create Decision Tree classifer object
            cart = DecisionTreeClassifier()
        # Train Decision Tree Classifer
            cart.fit(X_train_set, Y_train_set)
        #saving the model using joblib
            filename = 'decision_finalized_DT_model.sav'
            joblib.dump(cart, filename)
        # load the model from disk
            loaded_model = joblib.load(filename)
            result = loaded_model.score(X_test, Y_test)
            print (result)
        #Predict the response for test dataset
            predictions_rfc = cart.predict(X_test)
            print("\nCART accuracy test: \n")
            print(accuracy_score(Y_test, predictions_rfc))
            print(confusion_matrix(Y_test, predictions_rfc))
            print(classification_report(Y_test, predictions_rfc))

            Train_Accuracy = accuracy_score(Y_test, predictions_rfc)
            Train_Confusion_Matrix = confusion_matrix(Y_test, predictions_rfc)
            Train_Classification_Report = classification_report(Y_test, predictions_rfc)
            # matrix = plot_confusion_matrix(cart, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

        # Make predictions on test dataset
            print("\nCART results on final 30% validation \n")
            newcart = DecisionTreeClassifier()
            newcart.fit(X_train_set, Y_train_set)
            newpredictions_rfc = newcart.predict(X_validation)
            print("\nCART accuracy validation: \n")
            print(accuracy_score(Y_validation, newpredictions_rfc))
            print(confusion_matrix(Y_validation, newpredictions_rfc))
            print(classification_report(Y_validation, newpredictions_rfc))
            df = dataset.reset_index(drop = False)

            Test_Accuracy = accuracy_score(Y_validation, newpredictions_rfc)
            Test_Confussion_Matrix = confusion_matrix(Y_validation, newpredictions_rfc)
            Test_Classification_Report = classification_report(Y_validation, newpredictions_rfc)
            matrix = plot_confusion_matrix(newcart, X_test, Y_test, cmap=plt.cm.Blues)
            # plt.show()

            # #Compare Algorithms
            # fig = plt.figure()
            # fig.suptitle('Confusion Matrix')
            # ax = fig.add_subplot(111)
            # plt.boxplot(results)
            # ax.set_xticklabels(names)

            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_matrix = "data:image/png;base64," + my_base64_pngData.decode("ascii")
            plt.close()

            lines = classification_report(Y_validation, newpredictions_rfc).split('\n')

            classes = []
            plotMat = []

            for line in lines[2 : (len(lines) - 3)]:
                #print(line)
                t = line.split()
                print(t)
                if(len(t)==0):
                    break
                else:
                    classes.append(t[0])
                    v = [float(x) for x in t[1: len(t) - 1]]
                    print(v)
                    plotMat.append(v)
            plt.imshow(plotMat, interpolation='nearest', cmap=plt.cm.Blues)
            plt.title('Classification report ')
            plt.colorbar()
            x_tick_marks = np.arange(3)
            y_tick_marks = np.arange(len(classes))
            plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
            plt.yticks(y_tick_marks, classes)
            plt.tight_layout()
            plt.ylabel('Classes')
            plt.xlabel('Measures')
            my_stringIObytes = io.BytesIO()
            plt.savefig(my_stringIObytes, format='png')
            my_stringIObytes.seek(0)
            my_base64_pngData = base64.b64encode(my_stringIObytes.read())
            data_uri_report = "data:image/png;base64," + my_base64_pngData.decode("ascii")

    return render_template('deploy.html', data=msg, data2=Train_Accuracy, data3=data_uri_matrix, data4=Test_Classification_Report, comparison_uri=data_uri, data5=Test_Accuracy, data6=Test_Confussion_Matrix, data7=Test_Classification_Report, data8=data_uri_report)

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=5000, debug=True)


from django.http import HttpResponseRedirect, HttpResponse
import mimetypes, rfc822, os, stat
from pdb import set_trace
from django.shortcuts import get_object_or_404
from models import RedirCounter, Referer, ViewCounter

try:
    from PIL import Image
except ImportError:
    import Image

from StringIO import StringIO

_img  = Image.fromstring('RGB', (1,1), '000')
_ff = StringIO()
_img.save(_ff, format='GIF')
_ff.seek(0)

IMG_MIMETYPE = 'image/gif'
IMG_DATA = _ff.read()

def count(request, ctype_id, object_id):
    ctype_id = int(ctype_id)
    object_id = int(object_id)
    counter = ViewCounter.objects.increment(ctype_id, object_id)

    response = HttpResponse(IMG_DATA, mimetype=IMG_MIMETYPE)
    return response

# We may use
# update counter_downloadcounter as t1 set count = (select sum(t2.count) from counter_referer as t2 where t1.id = t2.counter_id);
# to calculate all total stats, if needed.
#
def redir(request, counter_id):
    referer = request.environ.get('HTTP_REFERER') or 'none'
    counter = get_object_or_404(RedirCounter, id = counter_id)
    ref = counter.referers.filter(url = referer)[:1]
    if not ref:
        ref = Referer(counter = counter, url = referer, count = 0)
    else:
        ref = ref[0]
    ref.count += 1
    ref.save()
    counter.count += 1
    counter.save()
    return HttpResponseRedirect(counter.url)


from telegram import Update
from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters
from excel_parser import SNILS_ERROR_TEXT, get_message
from snils_chek import snils_chek
from dotenv import load_dotenv
import os

TOKEN = ""

async def start(update: Update, context: ContextTypes.DEFAULT_TYPE):
    user = update.effective_user
    await update.message.reply_html(
            text=fr"Hi {user.mention_html()}!",
            # reply_markup=ForceReply(selective=True),
            )

async def send_data_from_snils(update: Update, context: ContextTypes.DEFAULT_TYPE):
    snils = snils_chek(update.message.text)
    if snils and len(snils) == 14:
        message = get_message(snils)
    else:
        message = SNILS_ERROR_TEXT

    await update.message.reply_html(text=message)

def main() -> None:
    load_dotenv()
    TOKEN = str(os.getenv("BOT_TOKEN"))

    app = Application.builder().token(TOKEN).build()

    app.add_handler(CommandHandler("start", start))
    app.add_handler(MessageHandler(filters.TEXT, callback=send_data_from_snils))

    app.run_polling()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# coding: utf-8

# In[30]:

import boto3

# In[31]:

po = boto3.client('polly')

# In[55]:

res=po.synthesize_speech(Text="this is srinivas,completing AWS workshop",OutputFormat='mp3',VoiceId='Russell')

# In[56]:

res

# In[57]:

#res['AudioStream'].read()

# In[58]:

file=open('myaudio.mp3','wb')

# In[59]:

file.write(res['AudioStream'].read())

# In[60]:

file.close()

# In[61]:

import IPython

# In[62]:

IPython.display.Audio("myaudio.mp3")

# In[ ]:


'''
Created on Sep 16, 2016

@author: evan.jensen

Evanmj@gmail.com
'''
from lxml import html
from lxml import etree
import requests        # pip install requests
import time
import os
import random

from bokeh.plotting import figure, output_file, show
from bokeh.models import Range1d

custom_y_scale = []

#RegistersToMonitor = [302,303,304,305]  # wings runout
#RegistersToMonitor = [308,309]  # Stem runout
#RegistersToMonitor = [306,307]  # Diameter Stem
#RegistersToMonitor = [331]  # Oven Conveyor Index Accuracy
RegistersToMonitor = [252,253]
#RegistersToMonitor = [336]  # oven conv clmpl

#custom_y_scale = [18000,20000]  #wings runout
#custom_y_scale = [45000,55000]  #Stem runout
#custom_y_scale = [125500,126500]  #Stem Diameter
#custom_y_scale = [30,40]  #Stem Diameter

clear = lambda: os.system('cls')

random.seed()

free_run = False

data_points_stored = 0

prog_start_time = time.time() * 1000

#setup dict
myData = {}
myData2 = {}
for r in RegistersToMonitor:
    myData[str(r)] = {}

print(myData)

def plot_now(data,use_measured_time=False):

    # build bokeh graph
    TOOLS = 'box_zoom,box_select,hover,resize,reset,save'
    if use_measured_time:
        x_label = 'Time'
    else:
        x_label = 'Data Collection Request #'

    p = figure(title="Fanuc Data", x_axis_label=x_label, y_axis_label='Value',plot_width=1300, plot_height=500, tools=TOOLS)
    if custom_y_scale != []:
        p.y_range = Range1d(custom_y_scale[0],custom_y_scale[1]) # force scale for now (TEMPORARY!!!!)

    for trace in data:
        #print(data[trace])
        i = 1  #start at fake time 1
        color = (random.random()*170,random.random()*170,random.random()*170,1)
        color_line = (color[0],color[1],color[2],0.25)
        print(color)

        trace_name = data[trace]['NAME']

        this_trace_x = []
        this_trace_y = []

        for pt in data[trace]['DATA']:

            # now pt is a (time,value) tuple.
            if use_measured_time:
                this_trace_x.append(pt[0])
            else:
                this_trace_x.append(i)
                i = i + 1
            this_trace_y.append(pt[1])

        print("x: " + repr(this_trace_x))
        print("y: " + repr(this_trace_y))

        # build a trace for this point.
        p.circle(this_trace_x, this_trace_y, fill_color="white", size=8, line_color = color)
        p.line(this_trace_x, this_trace_y, legend=trace_name, line_width=2, line_color = color_line)

    # output to static HTML file
    output_file("lines.html", title="Fanuc Data")

    # show the results
    show(p)

while True:

    if not free_run:

        print()
        users_input = input("Data Points Stored: %d, Press Enter to pull a data field, F to free run, Y for y scale, or P for plot stored data..." % data_points_stored)

        if users_input == 'f' or users_input == 'F':
            print("Starting Free Run.")
            free_run = True
        elif users_input == 'p' or users_input == 'P':
            print("Plotting Data.")
            plot_now(myData2)
        elif users_input == 'y' or users_input == 'Y':
            users_input = input("Enter new Min Y...")
            custom_y_scale[0] = int(users_input)
            cusers_input = input("Enter new Max Y...")
            custom_y_scale[1] = int(users_input)
            print("Got it.")
        else:
            #grab new data now.
            page = requests.get('http://192.168.3.11/karel/ComGet?sFc=28')

            tree = html.fromstring(page.text)

            clear() # clear monitor

            this_data = {}
            time_now = int((time.time() * 1000) - prog_start_time)

            for R in RegistersToMonitor:

                time_series = {}
                time_series[time_now] = ''

                td_row = R + 1 #avoid header.

                Comment_Val = tree.xpath('//*[@id="formTable"]/table/tr[' + str(td_row) + ']/td[2]/p/input')
                InputBox = tree.xpath('//*[@id="formTable"]/table/tr[' + str(td_row) + ']/td[3]/p/input')
                if Comment_Val[0].value != '':
                    Comment_Val[0].value = ':' + Comment_Val[0].value

                point_name = 'R[' + str(R) + Comment_Val[0].value + ']'
                point_value = InputBox[0].value

                this_data['value'] = point_value
                this_data['name'] = point_name

                time_series[time_now] = this_data

                print(time_series)

                if R in myData2:
                    myData2[R]['NAME']=point_name
                else:
                    #print ("Initializing for %d" % R)
                    #print ("Name is: " + point_name)
                    myData2[R]={'NAME':point_name, 'DATA':[]}
                if 'DATA' in myData2[R]:
                    myData2[R]['DATA'].append((time_now, point_value))

            data_points_stored = data_points_stored + 1
            #print(myData)
            #print(myData2)
    else:

        page = requests.get('http://192.168.3.11/karel/ComGet?sFc=28')

        tree = html.fromstring(page.text)

        clear() # clear monitor

        for R in RegistersToMonitor:

            td_row = R + 1 #avoid header.

            Comment_Val = tree.xpath('//*[@id="formTable"]/table/tr[' + str(td_row) + ']/td[2]/p/input')
            InputBox = tree.xpath('//*[@id="formTable"]/table/tr[' + str(td_row) + ']/td[3]/p/input')
            if Comment_Val[0].value != '':
                Comment_Val[0].value = ':' + Comment_Val[0].value

            print('R[' + str(R) + Comment_Val[0].value + '] = ' + InputBox[0].value)

        time.sleep(2)


import time
import logging
import sys
import argparse
from argparse import RawTextHelpFormatter
import operator
import csv
from flask import Flask, request, render_template, abort, Response
import networkx as nx
from bravo.regulation import upstream_regulation
from bravo.signaling import upstream_signaling
import bravo.config as config
import bravo.util as util

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s: %(message)s', stream=sys.stdout)

app = Flask("pyBravo")

data_sources = ['bind', 'biogrid', 'corum',
                'ctd', 'dip', 'drugbank', 'hprd', 'humancyc', 'inoh',
                'intact', 'kegg', 'mirtarbase', 'netpath', 'panther',
                'pid', 'psp', 'reactome', 'reconx', 'smpdb', 'wp',
                'intact_complex', 'msigdb', 'reach', 'wikipathways']

parser = argparse.ArgumentParser(description="""
BRAvo upstream regulation network reconstruction.
Here are some possible command lines :
    python pyBravo.py --web
    python pyBravo.py --regulation --input_genes JUN/FOS SCN5A -md 2 -co -su -sy
    python pyBravo.py --regulation --input_genes JUN/FOS SCN5A -md 2 -excl mirtarbase -co -su -sy
    python pyBravo.py --regulation --input_file myGenes.csv -md 2 -incl pid panther msigdb kegg -co -su -sy

Please report any issue to alban.gaignard@univ-nantes.fr.
""", formatter_class=RawTextHelpFormatter)
parser.add_argument('-w', '--web', action='store_true', required=False, help='to launch pyBravo as a web server', dest='web')
parser.add_argument('-reg', '--regulation', action='store_true', required=False, help='to assemble a regulation network', dest='reg')
parser.add_argument('-sig', '--signaling', action='store_true', required=False, help='to assemble a signaling network', dest='sig')
parser.add_argument('-sigd', '--signaling-detailed', action='store_true', required=False, help='to assemble a signaling network with detailed reactions', dest='sigd')
parser.add_argument('-md', '--max_depth', type=int, required=False, help='the maximum exploration depth', dest='md', default=1)
parser.add_argument('-sy', '--extend_with_synonyms', action='store_true', required=False, help='if specified, explore also synonyms', dest='s')
parser.add_argument('-su', '--extend_with_rna_protein_suffixes', action='store_true', required=False, help='if specified, explore also names suffixed with " rna" or " protein"', dest='su')
parser.add_argument('-co', '--decompose_complexes', required=False, action='store_true', help='if specified, decompose protein complexes', dest='c')
parser.add_argument('-fa', '--fast', required=False, action='store_true', help='if specified, only explore biopax display names', dest='fast')
parser.add_argument('-i', '--input_genes', nargs='+', required=False, help='the input gene list', dest='i')
parser.add_argument('-f', '--input_file', required=False, help='the input file, one gene per line', dest='f')
parser.add_argument('-o', '--output_file', required=False, help='the output files path and prefix', dest='o', default='out')
parser.add_argument('-incl', '--include_sources', nargs='+', required=False, help='the data sources to include', dest='incl')
parser.add_argument('-excl', '--exclude_sources', nargs='+', required=False, help='the data sources to exclude', dest='excl')
parser.add_argument('-e', '--endpoint', required=False, help='the endpoint to query (default: {})'.format(config.SPARQL_ENDPOINT), dest='endpoint')
parser.add_argument('-unk', '--unknown', required=False, action='store_true', help='if specified, do not consider unsigned edges', dest='unk')
parser.add_argument('-v', '--verbose', action='store_true', required=False, help='print debug information', dest='v')

@app.route("/test")
def test():
    return "test"

@app.route('/')
def home():
    return render_template('index.html')

def read_input_genes(filename):
    """

    :param filename:
    :return:
    """
    res = []
    with open(filename, newline='') as csvfile:
        reader = csv.reader(csvfile, delimiter=' ', quotechar='|')
        for row in reader:
            res.append(''.join(row))
    return res

def write_to_SIF(G, filename):
    """

    :param graph:
    :param filename:
    :return:
    """
    with open(filename, 'w', newline='') as csvfile:
        sif_writer = csv.writer(csvfile, delimiter='\t')
        for e in G.edges(data='attr'):
            try:
                sif_writer.writerow([e[0], e[2]['label'], e[1]])
            except TypeError:
                logging.error('Error for '+str(e))

    print('SIF network written to ' + filename)

def write_provenance(G, filename):
    """

    :param graph:
    :param filename:
    :return:
    """
    with open(filename, 'w', newline='') as csvfile:
        provenance_writer = csv.writer(csvfile, delimiter=',')
        for e in G.edges(data='attr'):
            reac = str(e[0]) + '\t' + str(e[2]['label']) + '\t' + str(e[1])
            p = e[2]['provenance']
            if 'http://pathwaycommons.org/pc2/' in p:
                p = p.split('http://pathwaycommons.org/pc2/')[1]
            provenance_writer.writerow([reac, p])
    print('Basic regulation reaction provenance written to ' + filename)

def get_centrality_as_md(G):
    """

    :param graph:
    :return:
    """
    centrality = nx.degree_centrality(G)
    # centrality = nx.closeness_centrality(G)
    # centrality = nx.betweenness_centrality(G)
    sorted_centrality = reversed(sorted(centrality.items(), key=operator.itemgetter(1)))
    sorted_centrality = list(sorted_centrality)
    cpt = 0
    md = """
| Node | Degree Centrality |
|------|------|
"""
    for g in sorted_centrality:
        md += "| " + g[0] + " | " + str(round(g[1], 3)) + " | \n"
        cpt += 1
        if cpt > 9:
            break
    return(md)

def build_nx_digraph(reconstructed_network):
    """

    :param reconstructed_network:
    :return:
    """
    G = nx.DiGraph()
    for e in reconstructed_network:
        #print(e)
        #print(e['source'] + ' --- ' + e['relation'] + ' --> ' + e['target'] + ' | ' + e['provenance'])
        G.add_edge(e['source'], e['target'],
                   color='g' if (e['relation'] in 'ACTIVATION') else 'r',
                   attr={'label': e['relation'], 'provenance': e['provenance']})
    print('Number of nodes = ' + str(len(G.nodes())))
    print('Number of edges = ' + str(len(G.edges())))
    return G

def main():
# def main(args):
    # args = parser.parse_args(args)
    args = parser.parse_args()

    if (args.md == None):
        print('please specify the maximum depth of exploration -md (--max_depth), 0 means complete exploration, 1 means '
              'first level of neighborhood.\n')
        parser.print_help()
        exit(0)

    if args.v:
        config.VERBOSE = True
        logging.getLogger().setLevel(logging.INFO)
    else:
        config.VERBOSE = False
        logging.getLogger().setLevel(logging.CRITICAL)

    if (not args.sig) and (not args.reg) and (not args.web):
        print('please specify one of -reg (--regulation), -sig (--signaling), or -w (--web) option\n')
        parser.print_help()
        exit(0)

    if (args.i is None) and (args.f is None) and (not args.web):
        print('please fill the -i (--input_genes) or -f (--input_file) parameter\n')
        parser.print_help()
        exit(0)

    if args.i and args.f:
        print('--input_genes and --input_file parameters are mutually exclusive : please provide only one of them\n')
        parser.print_help()
        exit(0)

    if args.sigd:
        config.FINE_GRAINED_SIGNALING_SIF = True
    else:
        config.FINE_GRAINED_SIGNALING_SIF = False

    input_genes_parameter = []
    if args.i:
        input_genes_parameter = args.i

    if args.f:
        input_genes_parameter = read_input_genes(args.f)

    if args.incl and args.excl:
        print('--include_sources and --exclude_sources parameters are mutually exclusive : please provide only one of them\\')
        parser.print_help()
        exit(0)

    if args.md > 0:
        config.MAX_DEPTH = args.md
        config.HAS_MAX_DEPTH = True
    else:
        logging.info('pyBravo is going for complete reconstruction')
        config.HAS_MAX_DEPTH = False

    if args.endpoint:
        if args.endpoint == 'pc':
            config.SPARQL_ENDPOINT = "http://rdf.pathwaycommons.org/sparql/"
        elif args.endpoint == 'ifb':
            config.SPARQL_ENDPOINT = "http://134.158.247.161/sparql/"
        else:
            config.SPARQL_ENDPOINT = args.endpoint

    config.DATA_SOURCES = data_sources
    if args.excl:
        for ds in args.excl:
            data_sources.remove(ds)
        config.DATA_SOURCES = data_sources
    if args.incl:
        config.DATA_SOURCES = args.incl

    logging.info("BRAvo will explore the following data sources:\n" + str(config.DATA_SOURCES))

    if args.s:
        config.EXTEND_WITH_SYNONYMS = True
    else:
        config.EXTEND_WITH_SYNONYMS = False

    if args.su:
        config.EXTEND_WITH_SUFFIXES = True
    else:
        config.EXTEND_WITH_SUFFIXES = False

    if args.c:
        config.DECOMPOSE_COMPLEXES = True
    else:
        config.DECOMPOSE_COMPLEXES = False

    if args.fast:
        config.FAST = True
    else:
        config.FAST = False

    if args.unk:
        config.UNKNOWN = True
    else:
        config.UNKNOWN = False

    logging.info("pyBravo version 1.0")

    if args.reg:
        start_time = time.time()
        # reconstructed_network = bravo.upstream_regulation(["JUN/FOS", "SCN5A"], max_depth=1)
        # reconstructed_network = bravo.upstream_regulation(["JUN/FOS", "SCN5A"], max_depth=2, data_sources = data_sources)
        # reconstructed_network = bravo.upstream_regulation(args.i, args.md, data_sources=data_sources)
        reconstructed_network = upstream_regulation(input_genes_parameter)
        elapsed_time = round((time.time() - start_time), 2)

        print("--- Upstream regulation network in %s seconds ---" % elapsed_time)

        G = build_nx_digraph(reconstructed_network)
        write_to_SIF(G, args.o + '.sif')
        write_provenance(G, args.o + '-provenance.csv')
        md = get_centrality_as_md(G)
        print(md)

        start_time = time.time()
        G_prime = util.fast_reg_network_unification(G, util.index_syn)
        elapsed_time = round((time.time() - start_time), 2)
        print("--- Network simplification in %s seconds ---" % elapsed_time)
        write_to_SIF(G_prime, args.o + '-unified.sif')
        write_provenance(G_prime, args.o + '-unified-provenance.csv')
        print('Nodes after simplification = ' + str(len(G_prime.nodes())))
        print('Edges after simplification = ' + str(len(G_prime.edges())))
        md = get_centrality_as_md(G_prime)
        print(md)

    elif args.sig:
        start_time = time.time()
        reconstructed_network = upstream_signaling(input_genes_parameter)
        elapsed_time = round((time.time() - start_time), 2)

        print("--- Upstream regulation network in %s seconds ---" % elapsed_time)

        G = build_nx_digraph(reconstructed_network)
        write_to_SIF(G, args.o + '.sif')
        write_provenance(G, args.o + '-provenance.csv')
        md = get_centrality_as_md(G)
        print(md)

        start_time = time.time()
        G_prime = util.fast_reg_network_unification(G, util.index_syn)
        elapsed_time = round((time.time() - start_time), 2)
        print("--- Network simplification in %s seconds ---" % elapsed_time)
        write_to_SIF(G_prime, args.o + '-unified.sif')
        write_provenance(G_prime, args.o + '-unified-provenance.csv')
        print('Nodes after simplification = ' + str(len(G_prime.nodes())))
        print('Edges after simplification = ' + str(len(G_prime.edges())))

        md = get_centrality_as_md(G_prime)
        print(md)

    elif args.web:
        app.run(host='0.0.0.0', port=9000, debug=True)
        logging.info("pyBravo launched as a web server")

if __name__ == "__main__":
    #args = ['--web']
    # args = ['--input_genes', 'HEY2', 'SCN5A', 'SCN3A', '-md', '1']
    # args = ['--input_genes', 'HEY2', 'SCN5A', '-md', '2']
    # args = ['--input_genes', 'HEY2', 'SCN5A', '-md', '2', '-co', '-sy', '-su']
    #         '-excl', 'mirtarbase', 'kegg']
    # args = ['-f', '../test-complex.csv', '-md', '1',
    #         '-incl', 'pid', 'panther', 'msigdb', 'kegg']
    # args = ['-f', '../test-complex.csv',
    #         '-incl', 'pid', 'panther', 'kegg']
    # args = ['-f', '../test-complex.csv', '-excl', 'mirtarbase']
    # args = ['--input_genes', 'JUN/FOS', 'SCN5A', '-md', '1',
    #        '-incl', 'pid', 'msigdb']
    # main(args = args)
    main()


# tạo ra file json - list of ground truth embedding vectors
# thử nghiệm trên backend trước khi deploy

import cv2
from keras_vggface.vggface import VGGFace
from tensorflow.python.keras.utils import generic_utils
from tensorflow.keras.models import load_model
from keras_vggface.utils import preprocess_input
import tensorflow as tf
from imutils import paths
import json
import numpy as np
from scipy.spatial.distance import cosine
from face_extraction import extract_face
import time

# model = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg')
# model.save('pretrained_model')

model = load_model('models/embedding')

"""1"""
# print(model.summary())

"""2"""
# img_paths = sorted(paths.list_images('anchor'))

# feature_vectors = []

# for img_path in img_paths:
#     img = extract_face(img_path)

#     img = preprocess_input(img, version=2)
#     feature_vector = model.predict(tf.expand_dims(img, axis=0))
#     feature_vectors.append(feature_vector.tolist())

# json_object = json.dumps(feature_vectors)
# with open('embedding_database.json', 'w') as outfile:
#     outfile.write(json_object)

# print('success')

""""3"""
with open('embedding_database.json', 'r') as openfile:

    # Reading from json file
    json_object = json.load(openfile)

face_embeddings = list(map(np.array, json_object))

labels = [
    "Dang Nguyen Anh Tuan",
    "Dinh Xuan Vu",
    "Do Manh Quan",
    "Nguyen Anh Tuan",
    "Nguyen Duy Phuc",
    "Ngo Hong Quang",
    "Nguyen Thanh Vinh",
]

t1 = time.time()

def test(imgPath):
    img = extract_face(imgPath)
    img = preprocess_input(img, version=2)
    img_embedding = model.predict(tf.expand_dims(img, axis=0))

    dis_vec = [np.linalg.norm(face_embedding - img_embedding)
               for face_embedding in face_embeddings]
    idx = np.argmin(dis_vec)

    return labels[idx], dis_vec

t2 = time.time()

imgPath = 'test/5.jpg'

face, dis_vec = test(imgPath)
img = cv2.imread(imgPath)
cv2.putText(img, face, (100, 100), cv2.FONT_HERSHEY_SIMPLEX,
            4, (250, 0, 0), 2, cv2.LINE_AA)
cv2.imshow('result', img)
cv2.waitKey()

print(f'[INFO] time: {t2 - t1}')
print(f'[INFO] distance vector: {dis_vec}')


# SourcePawn Completions is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import string
import sys
import sublime, sublime_plugin
import codecs
from collections import defaultdict
from queue import *
from threading import Timer, Thread

sys.path.append(os.path.dirname(__file__))
import watchdog.events
import watchdog.observers
import watchdog.utils
from watchdog.utils.bricks import OrderedSetQueue

class Wrapper:
    def __init__(self):
        self.value = []
    def set(self, newval):
        self.value = newval
    def get(self):
        return self.value

def plugin_loaded():
    load_include_dir(True)

def unload_handler():
    file_observer.stop()
    process_thread.stop()
    # Get process_thread to stop by adding something to the queue
    to_process.put(('', ''))
    # remove callback
    _get_settings().clear_on_change('SourcePawn Completions')

class SPCompletions(sublime_plugin.EventListener):
    def __init__(self):
        process_thread.start()
        self.delay_queue = None
        file_observer.start()

    def on_activated(self, view):
        if not self.is_sourcepawn_file(view):
            return
        file_name = view.file_name()
        if file_name is not None and file_name and not file_name in nodes:
            add_to_queue(view)

    def on_activated_async(self, view):
        _save_user_settings()

    def on_modified(self, view):
        self.add_to_queue_delayed(view)

    def on_post_save(self, view):
        self.add_to_queue_now(view)

    def on_load(self, view):
        self.add_to_queue_now(view)

    def add_to_queue_now(self, view):
        if not self.is_sourcepawn_file(view):
            return
        add_to_queue(view)

    def add_to_queue_delayed(self, view):
        if not self.is_sourcepawn_file(view):
            return

        if self.delay_queue is not None:
            self.delay_queue.cancel()

        delay_time = _get_settings().get('live_refresh_delay', 1.0)
        self.delay_queue = Timer(float(delay_time), add_to_queue_forward, [ view ])
        self.delay_queue.start()

    def is_sourcepawn_file(self, view):
        return view.file_name() is not None and (view.match_selector(0, 'source.sp') or view.match_selector(0, 'source.inc'))

    def on_query_completions(self, view, prefix, locations):
        # print('queried')
        if not view.match_selector(locations[0], 'source.sp -string -comment -constant') \
        and not view.match_selector(locations[0], 'source.inc -string -comment -constant'):
            return []

        file_name = view.file_name()
        if file_name is None:
            return []

        point = locations[0]-1;
        pointFound = False
        char = view.substr(point)
        if char == '.':
            pointFound = True
        else:
            while char.isalpha() and point > 1:
                point -= 1
                char = view.substr(point)
            if char == '.':
                pointFound = True

        if pointFound:
            current_node = nodes[file_name]

            classes = self.generate_classes(file_name)
            print('classes:')
            keys = classes.keys()
            if keys:
                for key in classes.keys():
                    print(' ' + key)
            else:
                print(' none')

            print('point found')
            wordpoint = point-1
            char = view.substr(point-1)
            hit_brace = False

            # do some stupid stuff if there are brackets.
            while char == ']':
                hit_brace = True
                bracelevel = 1
                wordpoint -= 1

                while bracelevel > 0 and point > 0:
                    char = view.substr(wordpoint)

                    if char == '[':
                        bracelevel -= 1
                    elif char == ']':
                        bracelevel += 1

                    wordpoint -= 1

                char = view.substr(wordpoint)

            if not char.isalpha():
                print('Could not find valid char: ' + char)
                return (
                    None,
                    sublime.INHIBIT_WORD_COMPLETIONS|
                    sublime.INHIBIT_EXPLICIT_COMPLETIONS
                )

            print('Found valid char')

            var_region = view.word(wordpoint)
            var = view.substr(var_region).strip()
            print('variable is: ' + var)

            if not var:
                return (
                    None,
                    sublime.INHIBIT_WORD_COMPLETIONS|
                    sublime.INHIBIT_EXPLICIT_COMPLETIONS
                )

            if re.match(r'\b[a-zA-Z_]\w*\b', var):
                # check if static access
                varstatic = var + '.static'
                # if varstatic in all_classes.keys():
                c = current_node.find_class(varstatic)
                if c is not None:
                    if c:
                        return (
                            c,
                            sublime.INHIBIT_WORD_COMPLETIONS|
                            sublime.INHIBIT_EXPLICIT_COMPLETIONS|
                            sublime.INHIBIT_REORDER
                        )
                    return (
                        None,
                        sublime.INHIBIT_WORD_COMPLETIONS|
                        sublime.INHIBIT_EXPLICIT_COMPLETIONS
                    )

                # check if local
                print(view.scope_name(point))
                begin = var_region.begin() - 1
                while view.match_selector(begin, 'source.sp meta.function | meta.function.parameters'):
                    begin -= 1;
                begin += 1

                region = view.substr(sublime.Region(begin, point))
                if region:
                    print('Found beginning at {}'.format(begin))
                    print(region)

                    print('searching')
                    m = re.search(r'\b([a-zA-Z_]\w+)\b\s+\b{}\b'.format(var), region)
                    if m:
                        retval = m.group(1)
                        # if retval in all_classes.keys():
                        c = current_node.find_class(retval)
                        if c is not None:
                            print('type found: ' + retval)
                            if c:
                                return (
                                    c,
                                    sublime.INHIBIT_WORD_COMPLETIONS|
                                    sublime.INHIBIT_EXPLICIT_COMPLETIONS|
                                    sublime.INHIBIT_REORDER
                                )
                            return (
                                None,
                                sublime.INHIBIT_WORD_COMPLETIONS|
                                sublime.INHIBIT_EXPLICIT_COMPLETIONS
                            )

                    # output text to a new view, apply syntax,
                    window = sublime.active_window()
                    panel = window.create_output_panel('temp_check_sp', unlisted=True)
                    panel.assign_syntax('Packages/SourcePawn/SourcePawn.sublime-syntax')
                    panel.run_command('append', {'characters': region})

                    for r in panel.find_by_selector('source.sp meta.variable.sp'):
                        region = re.sub('[ \t]{2,}', ' ', panel.substr(r).strip().replace('\n', ''))

                        print(region)

                        m = re.search(r'^([a-zA-Z_]\w+)\b', region)
                        if m:
                            retval = m.group(1)
                            if re.search(r'(?:{}|,) +{}'.format(retval, var), region):
                                print('Found local type. It\'s {}'.format(retval))

                                window.destroy_output_panel('temp_check_sp')

                                c = current_node.find_class(retval)
                                print(c)
                                if c is not None:
                                    if c:
                                        return (
                                            all_classes[retval],
                                            sublime.INHIBIT_WORD_COMPLETIONS|
                                            sublime.INHIBIT_EXPLICIT_COMPLETIONS|
                                            sublime.INHIBIT_REORDER
                                        )
                                return (
                                    None,
                                    sublime.INHIBIT_WORD_COMPLETIONS|
                                    sublime.INHIBIT_EXPLICIT_COMPLETIONS
                                )

                    window.destroy_output_panel('temp_check_sp')

                # check if global
                print('searching files for type')
                for v in sublime.active_window().views():
                    for r in v.find_by_selector('source.sp meta.variable.sp -meta.block.sp'):
                        region = re.sub('[ \t]{2,}', ' ', v.substr(r).strip().replace('\n', ''))
                        print(region)
                        m = re.search(r'^([a-zA-Z_]\w+)\b', region)
                        if m:
                            retval = m.group(1)
                            if re.search(r'(?:{}|,) +{}'.format(retval, var), region):

                                print('Found type. It\'s {}'.format(retval))
                                c = current_node.find_class(retval)
                                if c is not None:
                                    if c:
                                        return (
                                            c,
                                            sublime.INHIBIT_WORD_COMPLETIONS|
                                            sublime.INHIBIT_EXPLICIT_COMPLETIONS|
                                            sublime.INHIBIT_REORDER
                                        )
                                    return (
                                        None,
                                        sublime.INHIBIT_WORD_COMPLETIONS|
                                        sublime.INHIBIT_EXPLICIT_COMPLETIONS
                                    )

            return ( # Found a dot but unknown type
                None,
                sublime.INHIBIT_WORD_COMPLETIONS|
                sublime.INHIBIT_EXPLICIT_COMPLETIONS
            )

        # return (self.generate_funcset(view.file_name()), sublime.INHIBIT_WORD_COMPLETIONS | sublime.INHIBIT_EXPLICIT_COMPLETIONS)
        return (
            self.generate_funcset(file_name),
            sublime.INHIBIT_WORD_COMPLETIONS|
            sublime.INHIBIT_REORDER
        )

    def generate_funcset(self, file_name):
        funcset = set()
        visited = set()

        node = nodes[file_name]

        self.generate_funcset_recur(node, funcset, visited)
        return sorted_nicely(funcset)

    def generate_funcset_recur(self, node, funcset, visited):
        if node in visited:
            return

        visited.add(node)
        for child in node.children:
            self.generate_funcset_recur(child, funcset, visited)

        funcset.update(node.funcs)

    def generate_classes(self, file_name):
        classes = dict(
            {
                'Handle': [('Close()\t(function: void) [handles]', 'Close()')],
                'Handle.static':[]
            }
        )

        visited = set()

        node = nodes[file_name]
        self.generate_classes_recur(node, classes, visited)
        return classes

    def generate_classes_recur(self, node, classes, visited):
        if node in visited:
            return

        visited.add(node)
        for child in node.children:
            self.generate_classes_recur(child, classes, visited)

        classes.update(node.classes)

def _settings_filename():
    return 'SourcePawn Completions.sublime-settings'

def _get_settings():
    return sublime.load_settings(_settings_filename())

def on_settings_modified():
    load_include_dir()

def load_include_dir(register_callback = False):
    settings = _get_settings()
    if register_callback:
        settings.add_on_change('SourcePawn Completions', on_settings_modified)

    dirs = settings.get('include_directory', '.')
    include_dirs.set(dirs)
    if type(dirs) is not list:
        if not os.path.isabs(str(dirs)):
            raise RuntimeError("Invalid 'include_directory' setting (%s): directory doesn't exists" % str(dirs))

        file_observer.unschedule_all()
        file_observer.schedule(file_event_handler, dirs, True)
    else:
        for path in dirs:
            if not os.path.isabs(str(path)):
                raise RuntimeError("Invalid 'include_directory' setting (%s): directory doesn't exists" % str(path))

            file_observer.unschedule_all()
            file_observer.schedule(file_event_handler, path, True)

def sorted_nicely( l ):
    """ Sort the given iterable in the way that humans expect."""
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key[0]) ]
    return sorted(l, key = alphanum_key)

def add_to_queue_forward(view):
    sublime.set_timeout(lambda: add_to_queue(view), 0)

def add_to_queue(view):
    # The view can only be accessed from the main thread, so run the regex
    # now and process the results later
    file_name = view.file_name()
    if file_name is not None:
        to_process.put((file_name, view.substr(sublime.Region(0, view.size()))))

def add_include_to_queue(file_name):
    to_process.put((file_name, None))

def _save_user_settings():
    settings = _get_settings()
    if not settings.get('bootstrapped'):
        settings.set('bootstrapped', True)
        sublime.save_settings(_settings_filename())
        # build-system
        build_filename = 'SourcePawn.sublime-build'
        build = sublime.load_settings(build_filename)

        if not build.get('file_patterns'):
            build.set('file_patterns', ['*.sp'])
        if not build.get('quiet'):
            build.set('quiet', true)
        if not build.get('file_regex'):
            build.set('file_regex', '(.*)\\((\\d+)\\) : ()(.*$)')
        if not build.get('selector'):
            build.set('selector', 'source.sp')

        sublime.save_settings(build_filename)

class IncludeFileEventHandler(watchdog.events.FileSystemEventHandler):
    def __init__(self):
        watchdog.events.FileSystemEventHandler.__init__(self)

    def on_created(self, event):
        sublime.set_timeout(lambda: on_modified_main_thread(event.src_path), 0)

    def on_modified(self, event):
        sublime.set_timeout(lambda: on_modified_main_thread(event.src_path), 0)

    def on_deleted(self, event):
        sublime.set_timeout(lambda: on_deleted_main_thread(event.src_path), 0)

def on_modified_main_thread(file_path):
    if not is_active(file_path):
        add_include_to_queue(file_path)

def on_deleted_main_thread(file_path):
    if is_active(file_path):
        return

    node = nodes.get(file_path)
    if node is None:
        return

    node.remove_all_children_and_funcs()

def is_active(file_name):
    return sublime.active_window().active_view().file_name() == file_name

class ProcessQueueThread(watchdog.utils.DaemonThread):
    def run(self):
        while self.should_keep_running():
            (file_name, view_buffer) = to_process.get()
            if view_buffer is None:
                self.process_existing_include(file_name)
            else:
                self.process(file_name, view_buffer)

    def process(self, view_file_name, view_buffer):
        (current_node, node_added) = get_or_add_node(view_file_name)

        base_includes = set()
        includes = includes_re.findall(view_buffer)

        self.load_from_file(view_file_name, 'sourcemod', current_node, current_node, base_includes)

        for include in includes:
            if include == 'sourcemod':
                continue
            self.load_from_file(view_file_name, include, current_node, current_node, base_includes)

        for removed_node in current_node.children.difference(base_includes):
            current_node.remove_child(removed_node)

        process_buffer(view_buffer, current_node)

    def process_existing_include(self, file_name):
        current_node = nodes.get(file_name)
        if current_node is None:
            return

        base_includes = set()

        with codecs.open(file_name, 'r', 'utf-8') as f:
            print ('Processing Include File %s' % file_name)
            includes = includes_re.findall(f.read())

        self.load_from_file(file_name, 'sourcemod', current_node, current_node, base_includes)
        for include in includes:
            if include == 'sourcemod':
                continue
            self.load_from_file(file_name, include, current_node, current_node, base_includes)

        for removed_node in current_node.children.difference(base_includes):
            current_node.remove_child(removed_node)

        process_include_file(current_node)

    def load_from_file(self, view_file_name, base_file_name, parent_node, base_node, base_includes):
        (file_name, exists) = get_file_name(view_file_name, base_file_name)
        if not exists:
            print ('Include File Not Found: %s' % base_file_name)
            print ('Result: %s' % file_name)

        (node, node_added) = get_or_add_node(file_name)
        parent_node.add_child(node)

        if parent_node == base_node:
            base_includes.add(node)

        if not node_added or not exists:
            return

        with codecs.open(file_name, 'r', 'utf-8') as f:
            print ('Processing Include File %s' % file_name)
            includes = includes_re.findall(f.read())

        for include in includes:
            self.load_from_file(view_file_name, include, node, base_node, base_includes)

        process_include_file(node)

def get_file_name(view_file_name, base_file_name):
    file_name = ''
    if local_re.search(base_file_name) == None:
        dirs = include_dirs.get()
        if type(dirs) is not list:
            file_name = os.path.join(dirs, base_file_name + '.inc')
        else:
            for path in dirs:
                file_name = os.path.join(path, base_file_name + '.inc')
                if os.path.exists(file_name):
                    break
    else:
        file_name = os.path.join(os.path.dirname(view_file_name), base_file_name)

    return (file_name, os.path.exists(file_name))

def get_or_add_node(file_name):
    node = nodes.get(file_name)
    if node is None:
        node = Node(file_name)
        nodes[file_name] = node
        return (node, True)

    return (node, False)

# class SPObj:
#     def __init__(self):
#         self.name = ''
#         self.properties = dict() # name: ret
#         self.functions = dict() # name: ret

class Node:
    def __init__(self, file_name):
        self.file_name = file_name
        self.children = set()
        self.parents = set()
        self.funcs = set()
        self.classes = dict()
        self.globals = dict()

    def add_child(self, node):
        self.children.add(node)
        node.parents.add(self)

    def remove_child(self, node):
        self.children.remove(node)
        node.parents.remove(self)

        if len(node.parents) <= 0:
            nodes.pop(node.file_name)

    def remove_all_children(self):
        for child in self.children:
            self.remove_child(node)
        self.funcs.clear()
        self.classes.clear()

    def find_class(self, class_name):
        visited = set()
        if class_name == 'Handle':
            return [('Close()\t(function: void) [handles]', 'Close()')]
        if class_name == 'Handle.static':
            return []
        return self.__find_class_recur(class_name, visited)

    def __find_class_recur(self, class_name, visited):
        if self in visited:
            return
        visited.add(self)
        print('searching {} for {}'.format(self.file_name, class_name))
        if class_name in self.classes.keys():
            return self.classes[class_name]

        for child in self.children:
            c = child.__find_class_recur(class_name, visited)
            if c is not None:
                return c

        return None

class TextReader:
    def __init__(self, text):
        self.text = text.splitlines()
        self.position = -1

    def readline(self):
        self.position += 1

        if self.position < len(self.text):
            retval = self.text[self.position]
            if retval == '':
                return '\n'
            return retval
        return ''

DEPRECATED_FUNCTIONS = [
    'native Float:operator*',
    'native Float:operator/',
    'native Float:operator+',
    'native Float:operator-',
    'stock Float:operator*',
    'stock Float:operator/',
    'stock Float:operator+',
    'stock Float:operator-',
    'stock bool:operator=',
    'stock bool:operator!',
    'stock bool:operator>',
    'stock bool:operator<',
    'forward operator%('
]

loaded_files = set() # to prevent loading files more than once
docs = dict() # map function name to documentation
included_files = defaultdict(set) # map project files to included files
funcs = defaultdict(list) # map include files to functions

# Code after this point adapted from
# https://forums.alliedmods.net/showpost.php?p=1866026&postcount=19
# Credit to MCPAN (mcpan@foxmail.com)
def read_line(file):
    """read_line(File) -> string"""
    line = file.readline()
    if len(line) > 0:
        return comment_re.sub(' ', line).strip()
    return None

def process_buffer(text, node):
    text_reader = TextReader(text)
    process_lines(text_reader, node)

def process_include_file(node):
    with codecs.open(node.file_name, 'r', 'utf-8') as file:
        process_lines(file, node)

def skipMultiline(line_reader, buffer):
    while buffer.startswith('/*'):
        pos = buffer.find('*/')
        while pos == -1:
            buffer = read_line(line_reader)
            if buffer is None:
                return None
            pos = buffer.find('*/')

        idx = pos + 2
        length = len(buffer)
        if idx >= length:
            idx = length - 1
        buffer = buffer[idx].strip()
    return buffer

def process_lines(line_reader, node):
    node.funcs.clear()

    found_comment = False
    found_enum = False
    brace_level = 0

    global file
    file = os.path.basename(node.file_name).rsplit('.')[0]
    if file:
        file = ' [' + file + ']'

    semicolonRequired = False

    while True:
        buffer = read_line(line_reader)

        if buffer is None:
            break

        if not buffer or buffer.startswith('//'):
            continue

        buffer = skipMultiline(line_reader, buffer)

        if buffer is None:
            break

        if not buffer or buffer.startswith('//'):
            continue

        if brace_level == 0:
            m = enum_re.search(buffer)
            if m:
                enum_type = m.group(2)
                if not m.group(1): # if struct was found, dont do anything. Unsupported currently
                    print('Found enum: ' + m.group(0))
                    found_enum = True
                    enum_contents = ''

                    if enum_type is not None:
                        node.funcs.add((enum_type+'\t(enum)' + file, enum_type))
                else:
                    node.funcs.add((enum_type+'\t(enum struct)' + file, enum_type))

            elif buffer.startswith('#'):
                if buffer.strip().startswith('#pragma deprecated'):
                    buffer = read_line(line_reader)
                    if buffer is not None and buffer.startswith('stock '):
                        pos = buffer.find('{')
                        if pos != -1: # if line contains {, jump to it
                            buffer = buffer[pos:]
                        buffer = skip_brace_line(line_reader, buffer)
                        continue
                elif buffer.startswith('#define '):
                    buffer = get_preprocessor_define(node, buffer)
                continue

            elif buffer.startswith('public '):
                m = pubvar_re.search(buffer)
                if m:
                    val = m.group(1)
                    print('Found public const: ' + val)
                    node.funcs.add((val + '\t(const)' + file, val))
                    continue

                pos = buffer.find('{')
                # print('Skipping public')
                if pos != -1: # if line contains {, jump to it
                    buffer = buffer[pos:]
                else: # otherwise, skip to next line
                    continue

            elif buffer.startswith('struct '):
                pos = buffer.find('{')
                print('Skipping struct')
                if pos != -1: # if line contains {, jump to it
                    buffer = buffer[pos:]
                else: # otherwise, skip to next line
                    continue

            elif buffer.startswith('methodmap '):

                #  (funcname + '\t(function' + return_type + ')' + file, autocomplete)
                m = methodmap_re.search(buffer)
                if not m:
                    continue

                name = m.group(1)
                static_name = name+'.static'
                inherited = m.group(2)
                node.funcs.add((name + '\t(methodmap)' + file, name))

                if inherited is not None:
                    c = node.find_class(inherited)
                    if c is not None:
                        node.classes[name] = set(c)
                        node.classes[static_name] = set(node.find_class(inherited+'.static'))
                    else:
                        node.classes[name] = set()
                        node.classes[static_name] = set()
                else:
                    node.classes[name] = set()
                    node.classes[static_name] = set()

                in_comment = False
                searching = True
                pos = -1
                buffer = buffer[len(m.group(0)):]
                strlen = 0

                # print('new buffer: \'' + buffer + '\'')
                while pos == -1 and searching:
                    if buffer:
                        i = 0
                        strlen = len(buffer)
                        while i < strlen:
                            if in_comment:
                                if buffer[i] == '*' and i < strlen - 1 and buffer[i+1] == '/':
                                    in_comment = False
                                    i += 2
                                    continue
                            else:
                                if buffer[i] == '{':
                                    pos = i
                                    break
                                if buffer[i] == '/' and i < strlen - 1:
                                    if buffer[i+1] == '/':
                                        buffer = read_line(line_reader)
                                        break;
                                    if buffer[i+1] == '*':
                                        in_comment = True
                                        i += 2
                                        continue
                                elif buffer[i] > ' ':
                                    searching = False # found something other than comment and brace
                                    break
                            i += 1

                    if pos == -1 and searching:
                        buffer = read_line(line_reader)
                        if buffer is None:
                            return

                if pos == -1:
                    continue

                mmbody = ''
                brace_level = 1
                buffer = buffer[pos:]
                strlen = len(buffer)
                if strlen == 1:
                    buffer = read_line(line_reader)
                    while not buffer:
                        buffer = read_line(line_reader)
                    strlen = len(buffer)

                # print('found methodmap brace. new buffer: \'' + buffer + '\'' + ' len: ' + str(strlen))
                while brace_level > 0:
                    i = 0
                    while i < strlen:
                        c = buffer[i]
                        if in_comment:
                            if c == '*' and i < strlen - 1 and buffer[i+1] == '/':
                                in_comment = False
                                i += 2
                                continue
                        else:
                            if c == '/' and i < strlen - 1:
                                if buffer[i+1] == '/':
                                    break
                                if buffer[i+1] == '*':
                                    in_comment = True
                                    i += 2
                                    continue
                            else:
                                if c == '{':
                                    # brace_level += 1
                                    buffer = skip_brace_line(line_reader, buffer[i:])
                                    strlen = len(buffer)
                                    break
                                elif c == '}':
                                    brace_level -= 1
                                    buffer = buffer[i+1:]
                                    if brace_level == 0:
                                        break
                                else:
                                    mmbody += c

                        i += 1

                    if brace_level > 0:
                        buffer = read_line(line_reader)
                        strlen = len(buffer)
                        mmbody += ' '

                mmbody = re.sub(r'[ \t]{2,}', ' ', mmbody).strip()
                mmbody = re.sub(r'(\)|property[ \t]\b\w+\b[ \t]\b\w+\b);? ', r'\1\n', mmbody)
                # print('Body: ' + mmbody + '\n')

                # constructor
                m = re.search(r'public[ \t](?:native[ \t])?{}[ \t]*\((.*?)\)'.format(name), mmbody)
                if m:
                    params = m.group(1).split(',')

                    autocomplete = name + '('
                    i = 1
                    for param in params:
                        if i > 1:
                            autocomplete += ', '
                        if param:
                            autocomplete += '${%d:%s}' % (i, param.strip())
                        i += 1
                    autocomplete += ')'

                    node.funcs.add((name+'()\t(constructor)' + file, autocomplete))
                # function
                for m in re.finditer(r'public[ \t](?:native[ \t])?\b(?!native)(\w+)\b[ \t](\w+)[ \t]*\((.*?)\)', mmbody):
                    ret = m.group(1)
                    func = m.group(2)
                    params = m.group(3).split(',')

                    autocomplete = func + '('
                    i = 1
                    for param in params:
                        if i > 1:
                            autocomplete += ', '
                        if param:
                            autocomplete += '${%d:%s}' % (i, param.strip())
                        i += 1
                    autocomplete += ')'
                    node.classes[name].add((func + '()\t(function: ' + ret + ')' + file, autocomplete))
                # static function
                for m in re.finditer(r'public[ \t]static[ \t](?:native[ \t])?(\w+)\b[ \t](\w+)[ \t]*\((.*?)\)', mmbody):
                    ret = m.group(1)
                    func = m.group(2)
                    params = m.group(3).split(',')

                    autocomplete = func + '('
                    i = 1
                    for param in params:
                        if i > 1:
                            autocomplete += ', '
                        if param:
                            autocomplete += '${%d:%s}' % (i, param.strip())
                        i += 1
                    autocomplete += ')'
                    node.classes[static_name].add((func + '()\t(function: ' + ret + ')' + file, autocomplete))
                # property
                for m in re.finditer(r'property[ \t](\w+)[ \t](\w+)', mmbody):
                    ret = m.group(1)
                    prop = m.group(2)

                    node.classes[name].add((prop+'\t(property: ' + ret + ')' + file, prop))

                node.classes[name] = sorted(node.classes[name])
                node.classes[static_name] = sorted(node.classes[static_name])

            elif function_re.search(buffer):
                (buffer, found_comment, brace_level) = get_full_function_string(line_reader, node, buffer, found_comment, brace_level)

            elif re.search('^[ \t]*(?:type(?:def|set)|func(?:tag|enum))', buffer):
                pos = buffer.find('{')
                print('Skipping typedef')
                if pos != -1: # if line contains {, jump to it
                    buffer = buffer[pos:]
                else: # otherwise, skip to next line
                    continue
            # else:
                # print('End of checks: ' + buffer)

        (buffer, found_comment, brace_level) = read_string(buffer, found_comment, brace_level)

        if not buffer:
            continue

        if found_enum:
            (buffer, enum_contents, found_enum) = process_enum(node, buffer, enum_contents, found_enum)
        elif buffer.startswith('#define '):
            buffer = get_preprocessor_define(node, buffer)

# def process_methodmap(node, buffer):

def process_variable(node, buffer):
    result = ''
    consumingKeyword = True
    consumingName = False
    consumingBrackets = False

    for c in buffer:
        if consumingKeyword:
            if c == ' ':
                consumingKeyword = False
                consumingName = True
        elif consumingName:
            if c == ':':
                result = ''
            elif c == ' ' or c == '=' or c == ';':
                result = result.strip()
                if result != '':
                    node.funcs.add((result + '\t(variable)' + file, result))
                result = ''
                consumingName = False
                consumingBrackets = False
            elif c == '[':
                consumingBrackets = True
            elif not consumingBrackets:
                result += c
        elif c == ',':
            consumingName = True

    result = result.strip()
    if result != '':
        node.funcs.add((result + '\t(variable)' + file, result))

    return ''

def process_enum(node, buffer, enum_contents, found_enum):
    print('Processing enum: ' + buffer)

    pos = buffer.find('}')
    if pos != -1:
        buffer = buffer[0:pos]
        found_enum = False

    enum_contents = '%s%s' % (enum_contents, buffer)
    buffer = ''

    enum_type = ''
    m = enum_re.search(enum_contents)
    if m:
        if m.group(2):
            enum_type = ': ' + m.group(2)

    ignore = False
    if not found_enum:
        pos = enum_contents.find('{')
        enum_contents = enum_contents[pos + 1:]

        for c in enum_contents:
            if c == '=':
                ignore = True
            elif c == ':':
                buffer = ''
                continue
            elif c == ',':
                buffer = buffer.strip()
                if buffer != '':
                    node.funcs.add((buffer + '\t(enum' + enum_type + ')' + file, buffer))

                ignore = False
                buffer = ''
                continue

            if not ignore:
                buffer += c

        buffer = buffer.strip()
        if buffer != '':
            node.funcs.add((buffer + '\t(enum' + enum_type + ')' + file, buffer))

        buffer = ''

    return (buffer, enum_contents, found_enum)

def get_preprocessor_define(node, buffer):
    print('Processing define: ' + buffer)
    """get_preprocessor_define(File, string) -> string"""
    # Regex the #define. Group 1 is the name, Group 2 is the value
    define = define_re.search(buffer)
    if define:
        # The whole line is consumed, return an empty string to indicate that
        buffer = ''
        name = define.group(1)
        value = define.group(2).strip()
        node.funcs.add((name + '\t(constant: ' + value + ')' + file, name))
    return buffer

def get_full_function_string(line_reader, node, buffer, found_comment, brace_level):
    """get_full_function_string(File, string, string, bool) -> string"""
    multi_line = False
    temp = ''
    full_func_str = None
    open_paren_found = False

    while buffer is not None:
        buffer = buffer.strip()
        if not open_paren_found:
            parenpos = buffer.find('(')
            eqpos = buffer.find('=')

            if eqpos != -1 and (parenpos == -1 or eqpos < parenpos):
                return ('', found_comment, brace_level)

            if buffer.find(';') != -1 and parenpos == -1:
                return ('', found_comment, brace_level)

            if parenpos != -1:
                open_paren_found = True

        pos = buffer.rfind(')')
        if pos != -1:
            full_func_str = buffer[0:pos + 1]

            if (multi_line):
                full_func_str = '%s%s' % (temp, full_func_str)

            break

        multi_line = True
        temp = '%s%s' % (temp, buffer)

        buffer = read_line(line_reader)
        if buffer is None:
            return (buffer, found_comment, brace_level)

        (buffer, found_comment, brace_level) = read_string(buffer, found_comment, brace_level)

    if full_func_str is not None and not full_func_str in DEPRECATED_FUNCTIONS:
        process_function_string(node, full_func_str)

    return (buffer, found_comment, brace_level)

def process_function_string(node, func):
    """process_function_string(string, string, bool)"""
    if re.search(r'deprecated', func):
        return

    print('Processing Function: ' + func)

    file = os.path.basename(node.file_name).rsplit('.')[0]
    if file:
        file = ' [' + file + ']'

    func_type = ''
    return_type = ': '
    remaining = ''

    m = fullfunction_re.search(func)
    if m:
        if m.group(1):
            func_type += m.group(1) + ' '
        return_type += func_type + (m.group(2) if m.group(2) else '_')
        remaining = m.group(3)
    else:
        return

    split = remaining.split('(', 1)
    funcname = split[0].strip()
    remaining = split[1].strip()

    if remaining == ')':
        params = []
    else:
        params = remaining.strip()[:-1].split(',')

    autocomplete = funcname + '('
    i = 1
    for param in params:
        if i > 1:
            autocomplete += ', '
        if param:
            autocomplete += '${%d:%s}' % (i, param.strip())
        i += 1
    autocomplete += ')'

    node.funcs.add((funcname + '()\t(function' + return_type + ')' + file, autocomplete))

def skip_brace_line(line_reader, buffer):
    """skip_brace_line(File, string) -> string"""
    num_brace = 0
    in_comment = False

    while buffer is not None:
        print('Skipping brace: ' + buffer)
        i = 0
        pos = 0

        strlen = len(buffer)
        while i < strlen:
            c = buffer[i]
            if in_comment:
                if c == '*' and i < strlen - 1 and buffer[i+1] == '/':
                    in_comment = False
                    i += 2
                    continue
            else:
                if c == '{':
                    num_brace += 1
                elif c == '}':
                    num_brace -= 1
                elif c == '/' and i < strlen - 1:
                    if buffer[i+1] == '*':
                        in_comment = True
                        i += 2
                        continue
                    if buffer[i+1] == '/':
                        break

                if num_brace == 0:
                    print('Brace skip end.')
                    return buffer[i+1:]

            i += 1

        buffer = read_line(line_reader)
    return buffer

def read_string(buffer, found_comment, brace_level):
    """read_string(string, bool) -> (string, bool)"""
    buffer = buffer.replace('\t', ' ').strip()
    result = ''

    i = 0
    while i < len(buffer):
        if buffer[i] == '/' and i + 1 < len(buffer):
            if buffer[i + 1] == '/':
                return (result, found_comment, brace_level + result.count('{') - result.count('}'))
            elif buffer[i + 1] == '*':
                found_comment = True
                i += 1
            elif not found_comment:
                result += '/'
        elif found_comment:
            if buffer[i] == '*' and i + 1 < len(buffer) and buffer[i + 1] == '/':
                found_comment = False
                i += 1
        elif not (i > 0 and buffer[i] == ' ' and buffer[i - 1] == ' '):
            result += buffer[i]

        i += 1

    return (result, found_comment, brace_level + result.count('{') - result.count('}'))

to_process = OrderedSetQueue()
nodes = dict() # map files to nodes
file_observer = watchdog.observers.Observer()
process_thread = ProcessQueueThread()
file_event_handler = IncludeFileEventHandler()
include_dirs = Wrapper()
includes_re = re.compile(r'^[ \t]*#include[ \t]+[<"]([^>"]+)[>"]', re.MULTILINE)
local_re = re.compile(r'\.(sp|inc)$')
enum_re = re.compile(r'^[ \t]*enum\b[ \t]+(struct\b[ \t]+)?([\w_]+)?')
function_re = re.compile(r'^[ \t]*(?:(native|stock|forward)\b[ \t]+)?(?:([\w_]+)(?:[ \t]+|:))?([\w_]+[ \t]*\()')
fullfunction_re = re.compile(r'^[ \t]*(?:(native|stock|forward)\b[ \t]+)?(?:([\w_]+)(?: +|:))?([\w_]+ *\(.*?\))')
define_re = re.compile(r'#define[ \t]+([^\s]+)[\s]+(.+)')
comment_re = re.compile(r'\/\*(.*?)\*\/')
pubvar_re = re.compile(r'^[ \t]*public[ \t]+const[ \t]+\w+[ \t]+(\w+)\b\s*;')
methodmap_re = re.compile(r'^[ \t]*methodmap[ \t]+\b(\w+)\b(?:\s*<\s*(\w+)\b)?')
file = ''
# all_classes = dict(
#     {
#         'Handle': [('Close()\t(function: void) [handles]', 'Close()')],
#         'Handle.static':[]
#     }
# )