import os
import numpy as np
import pandas as pd
import sklearn as sk
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
from sklearn import metrics
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
from sklearn.metrics import plot_precision_recall_curve
from sklearn.metrics import classification_report

# import warnings filter
from warnings import simplefilter

# ignore all future warnings
simplefilter(action='ignore', category=FutureWarning)

#################################
# load data and preprocess it   #
#   CHANGE your path here       #
#################################

# use the data from the folder 'data_different_variants' - which is the updated data
source = "data_all_1"
upsample_input = 1
data = pd.read_csv(source+'.csv')

# JUST FOR TESTING
#data = data.sample(frac=0.005)

print('This Script is from', source)
print('it is upsampeld times:', upsample_input)

# IMPLEMENTING DOWNSAMPLING !!BEFORE SPLITTING!! FOR TESTING PURPOSES:
# Separate majority and minority classes
#from sklearn.utils import resample
#
#df_majority = data[data.y == 'no']
#df_minority = data[data.y == 'yes']
#
#
#df_majority_downsampled = resample(df_majority,
#                                    replace=False,  # sample with replacement
#                                    n_samples=len(df_minority),  # to match majority class
#                                    random_state=0)  # reproducible results

# Combine minority class with downsampled majority class
#df_downsampled = pd.concat([df_minority, df_majority_downsampled])
#data = df_downsampled

data_numerical = pd.get_dummies(data, drop_first=True)
data_numerical = data_numerical.drop('y_yes', axis=1)

# creating X and Y categories
X_ori = data_numerical
Y = data['y']

# Normalize the input variables
X = (X_ori - X_ori.min()) / (X_ori.max() - X_ori.min())
print(X.head(6))

# Normalize the input variables
X = (X_ori - X_ori.min()) / (X_ori.max() - X_ori.min())
print(X.head(6))

from sklearn.model_selection import train_test_split

# raised to 0.7, maybe 0.8 later
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
print(Y_train.value_counts())
print(Y_test.value_counts())

print(X_train)
print(Y_train)

# no upsampling

# add the specific hyper-parameters, that lead to the highest result, to the report df
report = pd.DataFrame(columns=['Model', "Best Params", 'Acc. Train', 'Acc. Test', 'F1-Score Test', 'ROC', 'precision', 'recall'])

#################
#   Functions   #
#################
# INTO PRESENTATION
# function for creating a plot
def create_gridsearch_plot(model_gs, param, param_label, title):
    fig, ax1 = plt.subplots()
    color = 'tab:blue'
    ax1.set_xlabel(param_label)
    ax1.set_ylabel('Mean Accuracy', color=color)
    ax1.plot(model_gs.param_grid[param], model_gs.cv_results_["mean_test_score"], color=color)
    ax1.tick_params(axis='y', labelcolor=color)

    ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis
    color = 'tab:red'
    ax2.set_ylabel('Standard Deviation', color=color)  # we already handled the x-label with ax1
    ax2.plot(model_gs.param_grid[param], model_gs.cv_results_["std_test_score"], color=color)
    ax2.tick_params(axis='y', labelcolor=color)
    fig.tight_layout()  # otherwise the right y-label is slightly clipped
    plt.title('Comparison of Accuracies and Standard Deviation ({})'.format(title))
    plt.xticks(model_gs.param_grid[param])
    plt.show()

# function to calculate cmtr, acctr, cmte, accte
def calculate_metrics(Y_train, Y_test, Y_train_pred, Y_test_pred):
    cmtr = confusion_matrix(Y_train, Y_train_pred)
    acctr = accuracy_score(Y_train, Y_train_pred)
    cmte = confusion_matrix(Y_test, Y_test_pred)
    accte = accuracy_score(Y_test, Y_test_pred)
    return {"cmtr": cmtr, "acctr": acctr, "cmte": cmte, "accte": accte}

# INTO PRESENTATION
# Visualize Confusion Matrix
from sklearn.metrics import confusion_matrix, plot_confusion_matrix
def confusion_matrix_plotter(model, X_test, Y_test):
    plot_confusion_matrix(model, X_test, Y_test, labels=['no', 'yes'],
                          cmap=plt.cm.Blues, values_format='d')
    plt.show()

# function to caluclate precision metric
def calculate_precision_recall(Y_test, Y_test_pred):
    Y_test_numb = pd.get_dummies(Y_test, drop_first=True)
    Y_test_pred_numb = pd.get_dummies(Y_test_pred, drop_first=True)
    precision = round(metrics.precision_score(Y_test_numb, Y_test_pred_numb), 4)
    recall = round(metrics.recall_score(Y_test_numb, Y_test_pred_numb), 4)
    return {"precision": precision, "recall": recall}

# calculate f1 score
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import f1_score

# function to caluclate f1 score
def calculate_f1_score(Y_test, Y_test_pred):
    lb_churn = LabelEncoder()
    # prints for verification:
    #print("y_test original: ", Y_test)
    #print("y_test_pred original: ", Y_test_pred)
    Y_test_code = lb_churn.fit_transform(Y_test)
    Y_test_pred_code = lb_churn.fit_transform(Y_test_pred)
    #print("y_test transformed before F1 score: ", Y_test_code)
    #print("y_pred transformed before F1 score: ", Y_test_pred_code)
    f1te = f1_score(Y_test_code, Y_test_pred_code)
    print(f1te)
    return f1te

# INTO PRESENTATION
# calculate ROC and AUC and plot the curve
from sklearn.metrics import roc_curve
from sklearn.metrics import auc

# functoin to calculate ROC and AUC and plot the curve
def calculate_roc_auc(model, X_test, Y_test):
    Y_probs = model.predict_proba(X_test)
    print(Y_probs[0:6, :])
    Y_test_probs = np.array(np.where(Y_test == 'yes', 1, 0))
    print(Y_test_probs[0:6])

    fpr, tpr, threshold = roc_curve(Y_test_probs, Y_probs[:, 1])
    print(fpr, tpr, threshold)

    roc_auc = auc(fpr, tpr)
    print(roc_auc)
    # Precsion-recall-curve
    Y_test_probs = np.array(np.where(Y_test == 'yes', 1, 0))
    Y_test_pred_probs = np.array(np.where(Y_test_pred == 'yes', 1, 0))
    average_precision = average_precision_score(Y_test_probs, Y_test_pred_probs)
    disp = plot_precision_recall_curve(model, X_test, Y_test)
    disp.ax_.set_title('2-class Precision-Recall curve: '
                       'AP={0:0.2f}'.format(average_precision))
    plt.show()
    return {"roc_auc": roc_auc, "fpr": fpr, "tpr": tpr}

# plot metrics
import matplotlib.pyplot as plt
# INTO PRESENTATION
# function to plot model metrics fpr, tpr, roc_auc
def plot_model_metrics(metrics_dict, title):
    plt.plot(metrics_dict["fpr"], metrics_dict["tpr"], 'b', label='AUC = %0.2f' % metrics_dict["roc_auc"])
    plt.legend(loc='lower right')
    plt.plot([0, 1], [0, 1], 'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    plt.title(title)
    plt.show()

# INTO PRESENTATION
# FOR DECISION TREE
# show feature importance
def plot_feature_importance(model, title):
    list(zip(X, model.feature_importances_))
    index = np.arange(len(model.feature_importances_))
    bar_width = 1.0
    plt.bar(index, model.feature_importances_, bar_width)
    plt.xticks(index, list(X), rotation=90)  # labels get centered
    plt.title(title)
    plt.show()

from tabulate import tabulate
# STILL NEEDS SOME REFACTORING
# function to return information about the two param variations and their outcomes
def create_tabulate_plot(model_gs, param_1, param_2, param_label_1, param_label_2, title):
    #headers = [param_label_1, param_label_2, "Mean Accuracy", "Standard Deviation"]
    #table = tabulate(pd.DataFrame(model_gs.param_grid).transpose(), headers, tablefmt="plain", floatfmt=".3f")
    #print("\n", table)
    print(pd.DataFrame({param_1: model_gs.cv_results_["param_{}".format(param_1)], param_2: model_gs.cv_results_["param_{}".format(param_2)], "mean_test_score": model_gs.cv_results_["mean_test_score"], "std_test_score": model_gs.cv_results_["std_test_score"]}))
    # not yet working:
    print("Best accuracy is: ".format(np.max(model_gs.cv_results_["mean_test_score"])))
    maxi = model_gs.best_params_
    print("Best params are: {}".format(maxi))
    #table = tabulate(train_accuracies[:, maxi[1, :]].transpose(), headers, tablefmt="plain", floatfmt=".3f")
    #print('The best Acc is: ', "\n", table)

# CUT OUT BY REFACTORING:
#from tabulate import tabulate
#headers = ["Max_Depth", "n_Estimators", "Mean Accurancie", "Standard Deviation"]
#table = tabulate(train_accuracies.transpose(), headers, tablefmt="plain", floatfmt=".3f")
#print("\n", table)
#print(train_accuracies[2].max())
#maxi = np.array(np.where(train_accuracies == train_accuracies[2].max()))
#print(maxi[0, :], maxi[1, :])
#print(train_accuracies[:, maxi[1, :]])
#table = tabulate(train_accuracies[:, maxi[1, :]].transpose(), headers, tablefmt="plain", floatfmt=".3f")
#print('The best Acc is: ', "\n", table)

#
# ######################
# #   Decision Trees   #
# ######################
#
# from sklearn.tree import DecisionTreeClassifier
#
#
# # test parameter max_depth
# # new GridSearchCV implementation
# # attention: the parameter 'n_jobs=-1' means all CPU cores will be used. This may result in an overflow of the RAM
# # do not forget to change parameter 'cv'
# model_gs_param_grid = {'max_depth': [2, 3, 4, 5, 6]}
# model_gs = sk.model_selection.GridSearchCV(estimator=DecisionTreeClassifier(criterion='entropy', random_state=0, class_weight='balanced'), param_grid=model_gs_param_grid,
#                                            scoring='accuracy', cv=10, n_jobs=-1)
# model_gs.fit(X_train, Y_train)
# print(model_gs.cv_results_)
#
#
# # creating a Plot
# create_gridsearch_plot(model_gs, "max_depth", "max depth of the tree", "Decision Tree")
#
#
# # finding the maximum to test it
# etmodel = model_gs.best_estimator_
# etmodel.fit(X_train, Y_train)
# Y_train_pred = etmodel.predict(X_train)
# Y_test_pred = etmodel.predict(X_test)
#
#
# # calculate cmtr, acctr, cmte, accte
# metrics_dict = calculate_metrics(Y_train, Y_test, Y_train_pred, Y_test_pred)
#
#
# # Visualize Confusion Matrix
# confusion_matrix_plotter(etmodel, X_test, Y_test)
#
#
# # plot classification report
# print('Classification Report \n', classification_report(Y_test, Y_test_pred))
#
#
# # calculate precision & recall metrics
# metrics_dict.update(calculate_precision_recall(Y_test, Y_test_pred))
#
#
# # calculate f1 score
# metrics_dict["f1te"] = calculate_f1_score(Y_test, Y_test_pred)
#
#
# # calculate ROC and AUC and plot the curve
# roc_auc_results = calculate_roc_auc(etmodel, X_test, Y_test)
# metrics_dict["roc_auc"] = roc_auc_results["roc_auc"]
# metrics_dict["fpr"] = roc_auc_results["fpr"]
# metrics_dict["tpr"] = roc_auc_results["tpr"]
# plot_model_metrics(metrics_dict, 'ROC Curve of Decision Tree')
#
#
# # plot feature importance
# plot_feature_importance(etmodel, 'Decision Tree - Feature Importance')
#
# # add metrics to the report
# report.loc[len(report)] = ['Decision Tree', model_gs.best_params_, metrics_dict["acctr"], metrics_dict["accte"], metrics_dict["f1te"], metrics_dict["roc_auc"], metrics_dict["precision"], metrics_dict["recall"]]
#
#
# # testing
# print(report)
#
# results = pd.DataFrame(model_gs.cv_results_)
# results.to_csv('Testing/'+source+'_decision_tree_report.csv', index = False)
#
#
#
# # #=============================================================================
# # #show tree using graphviz
# # import graphviz
# # dot_data = sk.tree.export_graphviz(etmodel, out_file=None,
# #                           feature_names=list(X),
# #                           filled=True, rounded=True,
# #                           special_characters=True)
# # graph = graphviz.Source(dot_data)
# # graph.format = 'png'
# # graph.render("Churn_entropy")
# # #=============================================================================
#
# #################
# #     Gini      #
# #################
#
#
# # Build Gini model where the max_depth is tested
# # new GridSearchCV implementation
# # attention: the parameter 'n_jobs=-1' means all CPU cores will be used. This may result in an overflow of the RAM
# # do not forget to change parameter 'cv'
# model_gs_param_grid = {"max_depth": [2,3,4,5,6,7]}
# model_gs = sk.model_selection.GridSearchCV(estimator=DecisionTreeClassifier(random_state=0, class_weight='balanced'), param_grid=model_gs_param_grid,
#                                            scoring='accuracy', cv=10, n_jobs=-1)
# model_gs.fit(X_train, Y_train)
# print(model_gs.cv_results_)
#
# results = pd.DataFrame(model_gs.cv_results_)
# results.to_csv('Testing/'+source+'_gini_report.csv', index = False)
#
# # creating a Plot
# create_gridsearch_plot(model_gs, "max_depth", "max depth of the tree", "Gini Decision Tree")
#
#
# # finding the maximum to test it
# gtmodel = model_gs.best_estimator_
# gtmodel.fit(X_train, Y_train)
# Y_train_pred = gtmodel.predict(X_train)
# Y_test_pred = gtmodel.predict(X_test)
#
# # calculate cmtr, acctr, cmte, accte
# metrics_dict = calculate_metrics(Y_train, Y_test, Y_train_pred, Y_test_pred)
#
#
# # Visualize Confusion Matrix
# confusion_matrix_plotter(gtmodel, X_test, Y_test)
#
#
# # print classification report
# print('Classification Report \n', classification_report(Y_test, Y_test_pred))
#
#
# # calculate precision & recall metrics
# metrics_dict.update(calculate_precision_recall(Y_test, Y_test_pred))
#
#
# # calculate f1 score
# metrics_dict["f1te"] = calculate_f1_score(Y_test, Y_test_pred)
#
#
# # calculate ROC and AUC and plot the curve
# roc_auc_results = calculate_roc_auc(gtmodel, X_test, Y_test)
# metrics_dict["roc_auc"] = roc_auc_results["roc_auc"]
# metrics_dict["fpr"] = roc_auc_results["fpr"]
# metrics_dict["tpr"] = roc_auc_results["tpr"]
# plot_model_metrics(metrics_dict, 'ROC Curve of Gini Decision Tree')
#
#
# # plot feature importance
# plot_feature_importance(gtmodel, 'Gini Decision Tree - Feature Importance')
#
# # add metrics to the report
# report.loc[len(report)] = ['Gini', model_gs.best_params_, metrics_dict["acctr"], metrics_dict["accte"], metrics_dict["f1te"], metrics_dict["roc_auc"], metrics_dict["precision"], metrics_dict["recall"]]
#
#
# # testing
# print(report)
#
#
# results = pd.DataFrame(model_gs.cv_results_)
# results.to_csv('Testing/'+source+'_gini_report.csv', index = False)
#

#################
# Random Forest #
#################

from sklearn.ensemble import RandomForestClassifier

# Build RandomForest model where the max_depth and the n_estimators is tested
# new GridSearchCV implementation
# attention: the parameter 'n_jobs=-1' means all CPU cores will be used. This may result in an overflow of the RAM
# do not forget to change parameter 'cv'
#model_gs_param_grid = {'max_depth': [12, 13, 14, 15, 16, 17, 18], 'n_estimators': [200, 400, 500, 600]}
model_gs_param_grid = {'max_depth': [6, 7, 8, 9, 10, 11, 12, 13]}
model_gs = sk.model_selection.GridSearchCV(estimator=RandomForestClassifier(random_state=0, class_weight='balanced', n_estimators=200), param_grid=model_gs_param_grid,
                                           scoring='accuracy', cv=10, n_jobs=-1)
model_gs.fit(X_train, Y_train)
print('Random-Forest-Results-Training: /n:', model_gs.cv_results_)

# # print GridSearchCV results
# create_tabulate_plot(model_gs, "max_depth", "n_estimators", "max depth", "number of trees", "Random Forest")

# creating a Plot
create_gridsearch_plot(model_gs, "max_depth", "max depth of the tree", "Random Forest")

# finding the maximum to test it
rfmodel = model_gs.best_estimator_
rfmodel.fit(X_train, Y_train)
Y_train_pred = rfmodel.predict(X_train)
Y_test_pred = rfmodel.predict(X_test)

# calculate cmtr, acctr, cmte, accte
metrics_dict = calculate_metrics(Y_train, Y_test, Y_train_pred, Y_test_pred)

# Visualize Confusion Matrix
confusion_matrix_plotter(rfmodel, X_test, Y_test)

# print classification report
print('Classification Report \n', classification_report(Y_test, Y_test_pred))

# calculate precision & recall metrics
metrics_dict.update(calculate_precision_recall(Y_test, Y_test_pred))

# calculate f1 score
metrics_dict["f1te"] = calculate_f1_score(Y_test, Y_test_pred)

# calculate ROC and AUC and plot the curve
roc_auc_results = calculate_roc_auc(rfmodel, X_test, Y_test)
metrics_dict["roc_auc"] = roc_auc_results["roc_auc"]
metrics_dict["fpr"] = roc_auc_results["fpr"]
metrics_dict["tpr"] = roc_auc_results["tpr"]
plot_model_metrics(metrics_dict, 'ROC Curve of Random Forest')

# show feature importance
plot_feature_importance(rfmodel, 'Random Forest - Feature Importance')

results = pd.DataFrame(model_gs.cv_results_)
results.to_csv('Testing/'+source+'_random_forest_report.csv', index = False)

# add metrics to report
report.loc[len(report)] = ['Random-Forest', model_gs.best_params_, metrics_dict["acctr"], metrics_dict["accte"], metrics_dict["f1te"], metrics_dict["roc_auc"], metrics_dict["precision"], metrics_dict["recall"]]

# View a list of the features and their importance scores
print(list(zip(X_train, rfmodel.feature_importances_)))

#################################
# NEEDS TO BE THE LAST MODEL!!  #
#################################
# LightGBM  #
#############

# USES THE ORIGINAL DATASET WITHOUT DURATION
source = "BankMarketing"

data = pd.read_csv(source+'.csv')

data = data.drop(columns='duration')
data[data.select_dtypes('object').columns.tolist()] = data[data.select_dtypes('object').columns.tolist()].astype('category')

# JUST FOR TESTING
#data = data.copy().iloc[:1000,:]

# creating X and Y categories
X = data.copy().drop(columns='y')
Y = data['y']

from sklearn.model_selection import train_test_split

# raised to 0.8
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

import lightgbm as lgb

#print("object columns: \n", X_train.select_dtypes('object').columns.tolist())
#print("category columns:\n", X_train.select_dtypes('category').columns.tolist())
#print("columns:\n", X_train.columns)
#not used yet
cat_col = X_train.select_dtypes('object').columns.tolist() + X_train.select_dtypes('category').columns.tolist()

#X_train[cat_col] = X_train[cat_col].astype('category')
#X_train[cat_col] = X_train[cat_col].astype('category')

d_train = lgb.Dataset(X_train, label=Y_train)
d_valid = lgb.Dataset(X_test, label=Y_test)
#d_train = lgb.Dataset(X_train, label=Y_train, feature_name=X_train.columns.tolist(), categorical_feature=cat_col)

print(X_train.columns.tolist())

model_gs_param_grid = {"max_depth": [3,4,5,6,7,8,10,12,15,17,20], "learning_rate": [0.005, 0.01, 0.05, 0.1, 0.2], "num_leaves": [7, 15, 31, 63, 127, 255], "num_iteration": [300, 500, 700]}
# parameter for lgb.LGBMClassifier(feature_name=X_train.columns.tolist())
model_gs = sk.model_selection.GridSearchCV(estimator=lgb.LGBMClassifier(random_state=0, objective='binary', class_weight='balanced'), param_grid=model_gs_param_grid,
                                           scoring='accuracy', cv=10, n_jobs=-1)
model_gs.fit(X_train, Y_train)

# finding the maximum to test it
lgbmodel = model_gs.best_estimator_
lgbmodel.fit(X_train, Y_train)
Y_train_pred = lgbmodel.predict(X_train)
Y_test_pred = lgbmodel.predict(X_test)

# calculate cmtr, acctr, cmte, accte
metrics_dict = calculate_metrics(Y_train, Y_test, Y_train_pred, Y_test_pred)

# Visualize Confusion Matrix
confusion_matrix_plotter(lgbmodel, X_test, Y_test)

# print classification report
print('Classification Report \n', classification_report(Y_test, Y_test_pred))

# calculate precision & recall metrics
metrics_dict.update(calculate_precision_recall(Y_test, Y_test_pred))

# calculate f1 score
metrics_dict["f1te"] = calculate_f1_score(Y_test, Y_test_pred)

# calculate ROC and AUC and plot the curve
roc_auc_results = calculate_roc_auc(lgbmodel, X_test, Y_test)
metrics_dict["roc_auc"] = roc_auc_results["roc_auc"]
metrics_dict["fpr"] = roc_auc_results["fpr"]
metrics_dict["tpr"] = roc_auc_results["tpr"]
plot_model_metrics(metrics_dict, 'ROC Curve of LightGBM')

# show feature importance
plot_feature_importance(lgbmodel, 'LightGBM - Feature Importance')

report.loc[len(report)] = ['LightGBM Classifier', model_gs.best_params_, metrics_dict["acctr"], metrics_dict["accte"], metrics_dict["f1te"], metrics_dict["roc_auc"], metrics_dict["precision"], metrics_dict["recall"]]

results = pd.DataFrame(model_gs.cv_results_)
results.to_csv('Testing/'+source+'_lightGBM_report.csv', index = False)

#############################
# PRINT THE FINAL REPORT    #
#############################

report.to_csv('Testing/report_all'+source+'.csv', index=False)

print("stop")


import urllib
from datetime import datetime
from urllib.parse import urlunparse, urlencode

import requests
from django.conf import settings

from apps.authorization.models import HabrUserProfile, HabrUser

def save_user_profile(backend, user, response, *args, **kwargs):
    if backend.name == 'vk-oauth2':
        api_url = urlunparse(('https',
                              'api.vk.com',
                              '/method/users.get',
                              None,
                              urlencode(dict(fields=','.join(('first_name', 'last_name', 'career', 'email',
                                                              'sex', 'bdate', 'country', 'city', 'photo_max_orig')),
                                             access_token=response['access_token'],
                                             v='5.92')),
                              None
                              ))
        response = requests.get(api_url)
        if response.status_code != 200:
            return
        data = response.json()['response'][0]
        user.is_confirmed = True
        if data.get('first_name'):
            user.habruserprofile.first_name = data["first_name"]
        if data.get('last_name'):
            user.habruserprofile.last_name = data["last_name"]
        if data.get('career'):
            user.habruserprofile.place_of_work = data['career'][-1]['company']
            user.habruserprofile.specialization = data['career'][-1]['position']
        if data.get('sex'):
            user.habruserprofile.gender = HabrUserProfile.MALE if data['sex'] == 2 else HabrUserProfile.FEMALE
        if data.get('bdate'):
            user.habruserprofile.birth_date = datetime.strptime(data['bdate'], '%d.%m.%Y').date()
        if data.get('country'):
            user.habruserprofile.country = data['country']['title']
        if data.get('city'):
            user.habruserprofile.city = data['city']['title']
        if data.get('photo_max_orig'):
            urllib.request.urlretrieve(data['photo_max_orig'], f'{settings.MEDIA_ROOT}/avatars/{user.pk}.jpg')
            user.habruserprofile.avatar = f'avatars/{user.pk}.jpg'
        user.save()
    elif backend.name == 'google-oauth2':
        user.is_confirmed = True
        if response.get('given_name'):
            user.habruserprofile.first_name = response['given_name']
        if response.get('family_name'):
            user.habruserprofile.last_name = response['family_name']
        if response.get('picture'):
            urllib.request.urlretrieve(response['picture'], f'{settings.MEDIA_ROOT}/avatars/{user.pk}.jpg')
            user.habruserprofile.avatar = f'avatars/{user.pk}.jpg'
        user.save()
    elif backend.name == 'github':
        user.is_confirmed = True
        if response.get('name'):
            user.habruserprofile.first_name = response['name']
        if response.get('avatar_url'):
            urllib.request.urlretrieve(response['avatar_url'], f'{settings.MEDIA_ROOT}/avatars/{user.pk}.jpg')
            user.habruserprofile.avatar = f'avatars/{user.pk}.jpg'
        user.save()
    else:
        return


# -*- coding: utf-8 -*-

from django import forms
from django.utils import timezone

from irs_common.datasync import DataSync
from utils.forms import ImprovedForm
from clients.models import Client
# from .models import

class ClientFormMixin(object):
    prefix = ''
    subform_classname = ''
    mandatory_client = True

    def __init__(self, *args, **kwargs):
        client_id = kwargs.pop('client_id')
        self.client = None
        if self.mandatory_client or client_id:
            self.client = Client.objects.get(pk=client_id)
        self._object = kwargs.pop('object', None)
        super().__init__(*args, **kwargs)
        self.reinit_widgets()

    def add_subform_checkbox(self, fieldname, subform_name, reltype='1x1', count=0, max_count=0):
        if reltype == '1x1':
            self.set_widget_class(fieldname, "form1x1switcher switcher-%s" % subform_name)
        else:
            self.set_widget_class(fieldname, "form1xnswitcher switcher-%s" % subform_name)
            self.fields[fieldname].widget.attrs['data-subform-count'] = str(count)
            self.fields[fieldname].widget.attrs['data-subform-maxcount'] = str(max_count)
        self.fields[fieldname].widget.attrs['data-subform-name'] = subform_name

class OICCalculatorForm(ClientFormMixin, ImprovedForm, forms.ModelForm):
    pp_debt_total = forms.IntegerField(initial=15000)
    pp_monthly_income = forms.IntegerField(initial=200)
    pp_assessed_date = forms.DateField(initial=timezone.datetime(2009, 2, 5))
    pp_offer_amount = forms.IntegerField(initial=0)

    mandatory_client = False

    class Meta:
        model = Client  # actually not needed, but it would be useful for mixins
        exclude = ()

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        if self.client:
            dsync = DataSync(self.client)
            self.fields['pp_debt_total'].initial = dsync.get('debt_total', 0) or 15000
            income = dsync.get('net_difference', lambda: 0)()
            if income == 0:
                income = dsync.get('monthly_income', 0)
            self.fields['pp_monthly_income'].initial = income or 200
            self.fields['pp_assessed_date'].initial = dsync.get('assessed_date') or timezone.datetime(2009, 2, 5)
            self.fields['pp_offer_amount'].initial = dsync.get('offer_amount', 0)


from requests import get
from functools import reduce
from data.text_utils import remove_unwanted, remove_special_characters

def get_subreddit_top_posts(subreddit):
    """
    inputs:
        subreddit - a subreddit name as a string
    returns:
        post_urls - a list of the urls of top posts in subreddit
    """
    base_url = "https://www.reddit.com/r/"
    top = "/top.json"
    user_agent = {'User-agent': 'degenbot 1.0'}
    resp = get(base_url + subreddit + top, headers = user_agent)
    if resp.status_code != 200:
        print("ERROR: Could not complete Reddit request. ERROR CODE: " + str(resp.status_code))
        return
    posts = resp.json().get('data',{}).get('children')
    if not posts:
        print("ERROR: No valid posts returned from Reddit request.")
        return
    # Map data to text of post
    post_urls = list(map(lambda post: post.get('data',{}).get('url'), posts))
    # Remove posts with no text
    post_urls = list(filter(lambda url: url, post_urls))
    return post_urls

def get_comment_text(comment):
    """
    inputs:
        comment - a dict that expects comment content
                  as a string in comment['data']['body']
    returns:
        valid_comment_text - the comment contend without undesired characters
                             as a list of words
    """
    if not comment:
        return []
    comment_text = comment.get("data",{}).get("body")
    if not comment_text:
        return []
    comment_text = comment_text.lower()
    comment_text = remove_unwanted(comment_text)
    comment_text_list = comment_text.split(".")
    comment_text_list = list(map(lambda text: text.split(" "), comment_text_list))
    valid_comment_text = list(filter(lambda comment: len(comment)>5, comment_text_list))
    return valid_comment_text

def get_post_and_comment_text(post_url):
    user_agent = {'User-agent': 'degenbot 1.0'}
    resp = get(post_url + "comments.json", headers = user_agent)
    if resp.status_code != 200:
        print("ERROR: Could not find comments for post "+ post_url +". ERROR CODE: " + str(resp.status_code))
        return
    try:
        post = resp.json()[0]
        comments =  resp.json()[1]
    except:
        print("ERROR: Invalid response type for "+ post_url +".")
        return
    comments = comments.get("data",{}).get("children")
    comment_texts = list(map(get_comment_text,comments))
    post_text = post.get("data",{}).get("selftext")
    if post_text:
        post_text = post_text.lower()
        post_text = remove_unwanted(post_text)
        post_text = post_text.split(".")
        post_text = list(map(lambda text: text.split(" "), post_text))
    else:
        post_text = []
    content_text = post_text + comment_texts
    valid_content_text = list(filter(lambda text: text, content_text))
    if not valid_content_text:
        return
    valid_content_text = list(reduce(lambda s1, s2: s1+s2 , valid_content_text))
    valid_content_text = list(map(remove_special_characters, valid_content_text))
    return valid_content_text

def get_subreddit_content(subreddit):
    post_urls = get_subreddit_top_posts(subreddit)
    content_texts = list(map(get_post_and_comment_text, post_urls))
    content_texts = list(filter(lambda text: text, content_texts))
    if not content_texts:
        print("ERROR: No usable content from r/" + subreddit)
        return []
    content = list(reduce(lambda s1, s2: s1+s2, content_texts))
    return content

if __name__ == "__main__":
    content = get_subreddit_content("wallstreetbets")
    print(content[0])


#!/usr/bin/env python
# Name: Puja Chandrikasingh
# Student number: 11059842
"""
This script converts data in a csv file to a json file.
"""

import csv
import json

def convertToJson(file):
    """
    This function converts the data in a csv file of the KNMI to a json file.
    """

    with open(file, 'r') as csv_file:
        csv_reader = csv.DictReader(csv_file, delimiter= ",")
        title = csv_reader.fieldnames

        # make title layout nice (first letter upercase, rest lowercase)
        for i in range(len(title)):
            title[i] = title[i].strip().lower().capitalize()

        data_for_json = []

        for row in csv_reader:
            # first find the right year and extract then the data
            if row[title[1]] == '1997':
                for i in range(2, len(title) - 1):
                    data_for_json.extend([{"Date":title[i].strip(),
                                           "Temp" : row[title[i]].strip()}])

        # save the data as json
        with open("data.json", "w") as j:
            j.write(json.dumps(data_for_json, indent = 4))

if __name__ == "__main__":
    # get the csv file and call the convert function
    convertToJson("D:/Econometrie/Jaar 3/Semester 2/Data Processing/data2.csv")


# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.

import os
import pathlib
import sys

import determined_ai_sphinx_theme

sys.path.append(os.path.abspath(os.path.dirname(__file__)))

# -- Project information -----------------------------------------------------

project = "Determined"
html_title = "Determined AI Documentation"
copyright = "2020, Determined AI"
author = "hello@determined.ai"

# The version info for the project you"re documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = pathlib.Path(__file__).parents[1].joinpath("VERSION").read_text()

# The full version, including alpha/beta/rc tags.
release = version

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx_ext_downloads",
    "sphinx.ext.autodoc",
    "sphinx.ext.extlinks",
    "sphinx.ext.intersphinx",
    "sphinx.ext.mathjax",
    "sphinx.ext.napoleon",
    "sphinxarg.ext",
    "sphinx_gallery.gen_gallery",
    "sphinx_copybutton",
    "sphinx_sitemap",
]

autosummary_generate = True
autoclass_content = "class"

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also effect to html_static_path and html_extra_path
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "examples", "requirements.txt"]

# The suffix of source filenames.
source_suffix = {".rst": "restructuredtext", ".txt": "restructuredtext"}

highlight_language = "none"

# Base URL for sitemap.
html_baseurl = "https://docs.determined.ai"

# The URLs generated by sphinx-sitemap include both the version number and the
# language by default. We don't use language in the published URL, and we also
# want to encourage the latest version of the docs to be indexed, so only
# include that variant in the sitemap.
sitemap_url_scheme = "latest/{link}"

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named 'default.css' will overwrite the builtin 'default.css'.
# html_static_path = ["_static"]

# Our custom sphinx extension uses this value to decide where to look for
# downloadable files.
dai_downloads_root = os.path.join("site", "downloads")

# -- HTML theme settings ------------------------------------------------

html_show_sourcelink = False
html_show_sphinx = False
html_last_updated_fmt = None
html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]}

html_theme_path = [determined_ai_sphinx_theme.get_html_theme_path()]
html_theme = "determined_ai_sphinx_theme"
html_logo = "assets/images/logo.png"
html_favicon = "assets/images/favicon.ico"

html_theme_options = {
    "analytics_id": "UA-110089850-1",
    "kickfire_id": "258260",
    "collapse_navigation": False,
    "display_version": True,
    "logo_only": False,
}

language = "en"

todo_include_todos = True

html_use_index = True
html_domain_indices = True

# -- Sphinx Gallery settings -------------------------------------------

class ExplicitOrder:
    """
    sphinx_gallery.sorting.ExplicitOrder doesn't work with
    within_subsection_order. Define a custom class for ordering examples within
    subsections with a static ordering.
    """

    ORDERING = {
        "native-tf-keras": [
            "tf_keras_native.py",
            "tf_keras_native_hparam_search.py",
            "tf_keras_native_dtrain.py",
        ]
    }

    def __init__(self, src_dir):
        self.gallery = pathlib.Path(src_dir).name
        if self.gallery not in ExplicitOrder.ORDERING:
            raise Exception("Ordering for gallery {} not found".format(self.gallery))

        self.ordering = ExplicitOrder.ORDERING[pathlib.Path(src_dir).name]

    def __call__(self, item):
        if item in self.ordering:
            return self.ordering.index(item)
        else:
            raise Exception(
                "Item '{}' not found in ordering for gallery {}".format(item, self.gallery)
            )

sphinx_gallery_conf = {
    "examples_dirs": "../examples/tutorials/native-tf-keras",
    "gallery_dirs": "tutorials/native-tf-keras",
    # Subsections are sorted by number of code lines per example. Override this
    # to sort via the explicit ordering.
    # "within_subsection_order": CustomOrdering,
    "within_subsection_order": ExplicitOrder,
    "download_all_examples": True,
    "plot_gallery": False,
    "min_reported_time": float("inf"),
}


# coding: utf-8
import argparse
import asyncio

import urwid

from rolling.client.http.client import HttpClient
from rolling.gui.controller import Controller
from rolling.gui.image.widget import ImageWidget
from rolling.kernel import Kernel

def run(args: argparse.Namespace) -> None:
    def exit_(*args, **kwargs):
        raise urwid.ExitMainLoop()

    image_widget = ImageWidget(args.image, callback=exit_)
    client = HttpClient(server_address="http://127.0.0.1")

    loop = asyncio.get_event_loop()
    kernel = Kernel(loop=loop)
    controller = Controller(client=client, kernel=kernel, root_menu_mode="exit_only")
    controller.view.main_content_container.original_widget = image_widget

    controller.main()

def main() -> None:
    parser = argparse.ArgumentParser(description="View image in 256 term colors")
    parser.add_argument("image", type=str, help="image path")
    args = parser.parse_args()
    run(args)

if __name__ == "__main__":
    main()


# -*- encoding: utf-8 -*-
# from django.db.models import Q
# from django.shortcuts import render_to_response, HttpResponse
# from django.template import RequestContext
# from forms import CorrespondenciaForm, CorrespondenciaArForm
# from django.http import HttpResponseRedirect
# from adscripciones.models import Adscripcion
# from contratos.models import  Datos_persona, Codigo
# from correspondencias.models import Correspondencia, CorrespondenciaAr
# from django.core.context_processors import csrf
# from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
# from openpyxl import Workbook
# from openpyxl.style import Color, Fill
# from openpyxl.drawing import Image
# from reportlab.pdfgen import canvas
# from io import BytesIO
from django.shortcuts import render
from django.views.generic import TemplateView
from django.views.generic.list import ListView
from django.contrib.admin.models import LogEntry

class dashboardView(TemplateView):
	template_name = 'dashboard.html'

class logView(ListView):
	queryset = LogEntry.objects.all()

	# model = LogEntry
	#
	# def get_context_data(self, **kwargs):
	# 	context = super(logView, self).get_context_data(**kwargs)
	# 	return context

# #ROAV --------------- View's to make a search's ---------------

# #ROAV ----- View to search into Archivo. -----
# def search_archivo(request):
# 	query = request.POST.get('buscar',)
# 	if query:
# 		qset = (
# 			Q(no_control__icontains=query) |
# 			Q(expide__icontains=query) |
# 			Q(firma__icontains=query)
# 		)
# 		registros = CorrespondenciaAr.objects.filter(qset)
# 	else:
# 		registros = CorrespondenciaAr.objects.all()

# 	correspondencia_list = registros
# 	paginator = Paginator(correspondencia_list, 25)

# 	page = request.GET.get('page')
# 	try:
# 		results = paginator.page(page)
# 	except PageNotAnInteger:
# 		results = paginator.page(1)
# 	except EmptyPage:
# 		results = paginator.page(paginator.num_pages)

# 	return render_to_response("informacion_del_archivo.html", {
# 						        "archivo": results,
# 						        "query": query },
# 						        context_instance=RequestContext(request)
# 						     )

# #ROAV --------------- View's to create information ---------------

# #ROAV ----- View to create a new register into Archivo. -----
# def crear_archivo(request):
# 	if request.POST:
# 		form = CorrespondenciaArForm(request.POST)
# 		if form.is_valid():
# 			form.save()

# 			return HttpResponseRedirect('/')

# 	else:
# 		form = CorrespondenciaArForm()

# 	args = {}
# 	args.update(csrf(request))

# 	args['form'] = form

# 	return render_to_response('crear_correspondenciaar.html',
# 								args,
# 								context_instance=RequestContext(request)
# 								)

# #ROAV ----- View to create a new register into Correspondencia RH. -----
# def crear_correspondencia(request):
# 	if request.POST:
# 		form = CorrespondenciaForm(request.POST)
# 		if form.is_valid():
# 			form.save()

# 			return HttpResponseRedirect('/')

# 	else:
# 		form = CorrespondenciaForm()

# 	args = {}
# 	args.update(csrf(request))

# 	args['form'] = form

# 	return render_to_response('crear_correspondencia.html',
# 								args,
# 								context_instance=RequestContext(request)
# 								)
# def codigos(request):
# 	if request.POST:
# 		form = CodigoForm(request.POST)
# 		if form.is_valid():
# 			form.save()

# 			return HttpResponseRedirect('/')

# 	else:
# 		form = CodigoForm()

# 	args = {}
# 	args.update(csrf(request))

# 	args['form'] = form

# 	return render_to_response('crear_codigo.html',
# 								args,
# 								context_instance=RequestContext(request)
# 								)

#ROAV --------------- View's to show information ---------------

#ROAV ----- View to show Plantilla. -----
# def informacion_del_personal(request):
# 	ads = request.session['gAds']

# 	query = request.POST.get('buscar',)

# 	if request.user.is_superuser:
# 		adscripciones =  Adscripcion.objects.raw('SELECT id_ads, nombre FROM sit_adscripcion')
# 		allPersonal = Datos_persona.objects.raw('SELECT id_persona, nombre FROM sit_datos_persona')

# 		if query:
# 			if len(query) == 4:
# 				q = int(query)
# 				qset = Q(fecha_ingreso__year=q)
# 			else:
# 				q = str(query)
# 				qset = (
# 					Q(nombre__icontains=q) |
# 					Q(rfc__icontains=q) |
# 					Q(adscripcion__nombre__icontains=q) |
# 					Q(contrato__tipo__icontains=q)
# 					)
# 			results = Datos_persona.objects.filter(qset).order_by('nombre')
# 			encontrados = Datos_persona.objects.filter(qset).count()
# 			query = query
# 		else:
# 			results = Datos_persona.objects.all().order_by('nombre')
# 			encontrados = ""
# 			query = ""

# 		noPersonal = Datos_persona.objects.all().count()
# 		contratos = Datos_persona.objects.filter(contrato__in = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25]).count()
# 		regularizados = Datos_persona.objects.filter(contrato = 28).count()
# 		homologados = Datos_persona.objects.filter(contrato = 29).count()
# 		bases = Datos_persona.objects.filter(contrato = 27).count()
# 	else:
# 		adscripciones =  ""
# 		allPersonal = Datos_persona.objects.raw('SELECT id_persona, nombre FROM sit_datos_persona WHERE adscripcion_id = %s', [ads])
# 		if query:
# 			if len(query) == 4:
# 				q = int(query)
# 				qset = Q(fecha_ingreso__year=q)
# 			else:
# 				qset = (
# 					Q(nombre__icontains=query) |
# 					Q(rfc__icontains=query) |
# 					Q(adscripcion__nombre__icontains=query) |
# 					Q(contrato__tipo__icontains=query)
# 					) & Q(adscripcion = ads)
# 		else:
# 			qset = Q(adscripcion = ads)

# 		encontrados = Datos_persona.objects.filter(qset).count()
# 		if encontrados >= 1:
# 			query = query
# 		else:
# 			query = ""
# 		noPersonal = Datos_persona.objects.filter(adscripcion = ads).count()
# 		results =  Datos_persona.objects.filter(qset).order_by('nombre')
# 		contratos = Datos_persona.objects.filter(Q(contrato__in = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25]) & Q(adscripcion=ads)).count()
# 		regularizados = Datos_persona.objects.filter(Q(contrato = 28) & Q(adscripcion = ads)).count()
# 		homologados = Datos_persona.objects.filter(Q(contrato = 29) & Q(adscripcion = ads)).count()
# 		bases = Datos_persona.objects.filter(Q(contrato = 27) & Q(adscripcion = ads)).count()

# 	personal_list = results
# 	paginator = Paginator(personal_list, 40)

# 	page = request.GET.get('page')
# 	try:
# 		personal = paginator.page(page)
# 	except PageNotAnInteger:
# 		personal = paginator.page(1)
# 	except EmptyPage:
# 		personal = paginator.page(paginator.num_pages)
# 	return render_to_response('informacion_del_personal.html',
# 								{'personal' : personal,
# 								 'adscripciones' : adscripciones,
# 								 'allPersonal' : allPersonal,
# 								 'encontrados' : encontrados,
# 								 'personalContrato' : contratos,
# 								 'personalRegularizado' : regularizados,
# 								 'personalHomologado' : homologados,
# 								 'personalBase' : bases,
# 								 'personalTotal' : noPersonal,
# 								 'query' : query,
# 								},
# 							 	context_instance=RequestContext(request)
# 							 )

# #ROAV ----- View to show an any person. -----
# def informacion_de_la_persona(request):
# 	person_id = request.GET.get('id')

# 	return render_to_response('informacion_de_la_persona.html',
# 								{'personal' : Datos_persona.objects.get(id_persona = person_id),},
# 							 	context_instance=RequestContext(request)
# 							 )

# #ROAV ----- View to show Archivo. -----
# def informacion_del_archivo(request):
# 	query = request.POST.get('buscar',)
# 	if query:
# 		qset = (
# 			Q(no_control__icontains=query) |
# 			Q(no_documento__icontains=query) |
# 			Q(adscripcion__icontains=query) |
# 			Q(firma__icontains=query)
# 			)
# 		results = CorrespondenciaAr.objects.filter(qset).order_by('no_control')
# 		noCorres = CorrespondenciaAr.objects.filter(qset).count()
# 	else:
# 		results = CorrespondenciaAr.objects.all().order_by('no_control')
# 		noCorres = CorrespondenciaAr.objects.all().count()

# 	archivos_list = results
# 	paginator = Paginator(archivos_list, 26)

# 	page = request.GET.get('page')
# 	try:
# 		archivos = paginator.page(page)
# 	except PageNotAnInteger:
# 		archivos = paginator.page(1)
# 	except EmptyPage:
# 		archivos = paginator.page(paginator.num_pages)

# 	return render_to_response('informacion_del_archivo.html',
# 								{'archivo' : archivos,
# 								 'encontrados' : noCorres},
# 								context_instance = RequestContext(request)
# 							)

# def detalle_del_archivo(request):
# 	cor_id = request.GET.get('id_cor')

# 	return render_to_response('detalle_del_archivo.html',
# 								{'archivo' : CorrespondenciaAr.objects.get(id_cor = cor_id),},
# 							 	context_instance=RequestContext(request)
# 							 )

# #ROAV ----- View to show Correspondencia RH. -----
# def informacion_de_la_correspondencia(request):
# 	correspondencia_list = Correspondencia.objects.all()
# 	paginator = Paginator(correspondencia_list, 25)

# 	page = request.GET.get('page')
# 	try:
# 		correspondencia = paginator.page(page)
# 	except PageNotAnInteger:
# 		correspondencia = paginator.page(1)
# 	except EmptyPage:
# 		correspondencia = paginator.page(paginator.num_pages)

# 	return render_to_response('informacion_de_la_correspondencia.html',
# 								{'correspondencia' : correspondencia},
# 								context_instance = RequestContext(request)
# 							)

# #ROAV ----- View to show Centros. -----
# def informacion_de_los_centros(request):
# 	query = request.POST.get('buscar')
# 	if query:
# 		qset = (
# 			Q(nombre__icontains=query) |
# 			Q(clave__icontains=query) |
# 			Q(tipo_centro__icontains=query) |
# 			Q(municipio__nombre__icontains=query)
# 			)
# 		results = Adscripcion.objects.filter(qset).order_by('nombre')
# 		noCentros = Adscripcion.objects.filter(qset).count()
# 	else:
# 		results = Adscripcion.objects.all().order_by('nombre')
# 		noCentros = ""
# 		query = ""

# 	centro_list = results
# 	paginator = Paginator(centro_list, 25)

# 	page = request.GET.get('page')
# 	try:
# 		centro = paginator.page(page)
# 	except PageNotAnInteger:
# 		centro = paginator.page(1)
# 	except EmptyPage:
# 		centro = paginator.page(paginator.num_pages)

# 	return render_to_response('informacion_de_los_centros.html',
# 								{'centros' : centro,
# 								 'encontrados' : noCentros,
# 								 'query' : query,
# 								},
# 								context_instance = RequestContext(request)
# 							)

# def informacion_del_centro(request):
# 	centro_id = request.GET.get('id')

# 	return render_to_response('informacion_del_centro.html',
# 								{'centro' : Adscripcion.objects.get(id_ads = centro_id),},
# 							 	context_instance=RequestContext(request)
# 							 )

# #ROAV ----- View to show Minutario. -----
# def informacion_del_minutario(request):
# 	return render_to_response('informacion_del_personal.html',
# 								{'info' : Datos_persona.objects.all() },
# 							 	context_instance=RequestContext(request)
# # 							 )
# # def historial(request):
# # 	return render_to_response('historial.html',
# # 								{'historial' : LogEntry.objects.all()},
# # 							 	context_instance=RequestContext(request)
# 							 )

#ROAV --------------- View's to do export to excel. ---------------

#ROAV ----- Export plantilla to excel. -----
def exportar_plantilla(request):
	ads = request.session['gAds']
	query = request.GET.get('buscar')

	if request.user.is_superuser:
		if query:
			qset = (
				Q(nombre__icontains=query) |
				Q(rfc__icontains=query) |
				Q(adscripcion__nombre__icontains=query) |
				Q(contrato__tipo__icontains=query)
			)
			registros = Datos_persona.objects.filter(qset).order_by('nombre')
		else:
			registros = Datos_persona.objects.all().order_by('nombre')
	else:
		if query:
			qset = (
				Q(nombre__icontains=query) |
				Q(rfc__icontains=query) |
				Q(adscripcion__nombre__icontains=query) |
				Q(contrato__tipo__icontains=query)
				) & Q(adscripcion = ads)
			registros = Datos_persona.objects.filter(qset).order_by('nombre')
		else:
			registros = Datos_persona.objects.filter(adscripcion = ads).order_by('nombre')

	libro = Workbook()
	hoja = libro.get_active_sheet()
	hoja.title = "Personal"
	imagen = Image("sit/static/imgs/2808-small.png")
	imagen.drawing.left = 150 # distancia al borde izquierdo
	imagen.drawing.top = 0 # distancia al borde superior
	hoja.add_image(imagen)
	hoja.merge_cells("A2:J2")
	c = hoja["A2"]
	hoja["A2"] = "Subdirección de Recursos Humanos"
	c.style.font.size = 20
	c.style.font.name = "Calibri"
	c.style.font.bold = True
	c.style.alignment.horizontal = "center"

  # Ahora, se obtiene las celdas en la cuál se colocará el nombre
  # del campo. como son 8 campos, se necesita 8 celdas
	rango_celdas = hoja.range("B5:W5")

  # se crea una tupla con los nombres de los campos
	nombre_campos = ("NOMBRE","RFC","CURP","SEXO","JEFE INMEDIATO","CODIGO","CONTRATO","ADSCRIPCION","DEPARTAMENTO","FUNCION REAL","FECHA DE INGRESO","GRADO MAX ESTUDIOS",
  				"ESPECIALIDAD","SUELDO NETO","FECHA NACIMIENTO","ESTADO CIVIL","EMAIL","TEL CASA","TEL CEL","DOMICILIO","DIAS LABORALES","HORARIO",)
  # ahora, se asigna cada nombre de campo a cada celda
	cls = [0,4,7,9,19]
	for campo in rango_celdas:
		indice = 0  # se crea un contador para acceder a la tupla
		for celda in campo:
			celda.value = nombre_campos[indice]
			if indice in cls:
				hoja.column_dimensions[celda.column].width = 50
			elif indice == 3:
				hoja.column_dimensions[celda.column].width = 15
			elif indice == 16:
				hoja.column_dimensions[celda.column].width = 35
			else:
				hoja.column_dimensions[celda.column].width = 25

			celda.style.font.size = 14
			celda.style.font.name = "Calibri"
			celda.style.font.bold = True
			celda.style.alignment.wrap_text = True
			celda.style.font.color.index = Color.WHITE
			celda.style.fill.fill_type = Fill.FILL_SOLID
			celda.style.fill.start_color.index = Color.BLUE
			indice += 1
  # ya se tiene los nombres de los campos
  # ahora se obtiene el rango de celdas en donde irán los datos
	longitud = 5 + len(registros)
	celdas_datos = hoja.range("B6:W{0}".format(longitud))
  # # ahora vamos a dar los valores a las celdas con los datos
	fila = 0
  #hoja.row_dimenions[celda.row].height = 50
	for registro in registros:
		codigo = '%s %s %s %s %s %s' % (registro.psp, registro.ur, registro.ptda, registro.puesto, registro.hrs, registro.npto)
		contrato = '%s' % (registro.contrato)
		adscripcion = '%s' % (registro.adscripcion)
		departamento = '%s' % (registro.departamento)
		diasLabs = '%s' % (registro.dias_labs)
		horario = '%s' % (registro.horario)
		datos = (registro.nombre,registro.rfc,registro.curp,registro.sexo,registro.jefe_inmeadiato,codigo,contrato,adscripcion,departamento,registro.funcion_real,registro.fecha_ingreso,registro.gme,
			registro.especialidad,registro.sueldo_neto,registro.fecha_nac,registro.estado_civil,registro.email,registro.tel_casa,registro.tel_cel,registro.domicilio,
			diasLabs,horario,)
		indice = 0
		for celda in celdas_datos[fila]:
			celda.value = datos[indice]
			indice += 1
		fila += 1

   # se crea un objeto httpresponse y se pasa como parámetro el mimetype
   # diciendo que es excel
	response = HttpResponse(mimetype="application/ms-excel")  # HttpResponse viene del modulo django.http
	nombre_archivo = "Plantilla.xlsx"
	contenido = "attachment; filename={0}".format(nombre_archivo)
	response["Content-Disposition"] = contenido
	libro.save(response)
	return response

#ROAV ----- Export centros to excel. -----
def exportar_centros(request):
	# primero, hacemos la consulta de los usuarios
    #registros = Datos_persona.objects.all()
    query = request.GET.get('buscar')
    if query:
    	qset = (
    		Q(nombre__icontains=query) |
    		Q(clave__icontains=query) |
    		Q(tipo_centro__icontains=query)
    		)
    	registros = Adscripcion.objects.filter(qset).order_by('nombre')
    else:
    	registros = Adscripcion.objects.all().order_by('nombre')

    libro = Workbook()
    hoja = libro.get_active_sheet()
    hoja.title = "Centros"
    imagen = Image("sit/static/imgs/2808-small.png")
    imagen.drawing.left = 150 # distancia al borde izquierdo
    imagen.drawing.top = 0 # distancia al borde superior
    hoja.add_image(imagen)
    hoja.merge_cells("A2:J2")
    c = hoja["A2"]
    hoja["A2"] = "Subdirección de Recursos Humanos"
    c.style.font.size = 20
    c.style.font.name = "Calibri"
    c.style.font.bold = True
    c.style.alignment.horizontal = "center"

    # Ahora, se obtiene las celdas en la cuál se colocará el nombre
    # del campo. como son 8 campos, se necesita 8 celdas
    rango_celdas = hoja.range("B5:J5")
    # se crea una tupla con los nombres de los campos
    nombre_campos = ("CLAVE","TIPO CENTRO","NOMBRE","DOMICILIO","DIRECTOR","ADMINISTRADOR","JEFE DE RH","TELEFONO","CLUE",)
    # ahora, se asigna cada nombre de campo a cada celda
    for campo in rango_celdas:
        indice = 0  # se crea un contador para acceder a la tupla
        for celda in campo:
            celda.value = nombre_campos[indice]
            if indice == 3:
            	hoja.column_dimensions[celda.column].width = 15
            elif indice == 16:
            	hoja.column_dimensions[celda.column].width = 35
            else:
            	hoja.column_dimensions[celda.column].width = 25
            celda.style.font.size = 14
            celda.style.font.name = "Calibri"
            celda.style.font.bold = True
            celda.style.alignment.horizontal = "center"
            celda.style.alignment.wrap_text = True
            celda.style.font.color.index = Color.WHITE
            celda.style.fill.fill_type = Fill.FILL_SOLID
            celda.style.fill.start_color.index = Color.BLUE
            indice += 1
    # ya se tiene los nombres de los campos
    # ahora se obtiene el rango de celdas en donde irán los datos
    longitud = 5 + len(registros)
    celdas_datos = hoja.range("B6:J{0}".format(longitud))
    # # ahora vamos a dar los valores a las celdas con los datos
    fila = 0
    #hoja.row_dimenions[celda.row].height = 50
    for registro in registros:
    	datos = (registro.clave,registro.tipo_centro,registro.nombre,registro.domicilio,registro.director,registro.administrador,registro.jefe_rh,registro.telefono,registro.clue,)
    	indice = 0
    	for celda in celdas_datos[fila]:
        	celda.value = datos[indice]
        	indice += 1
    	fila += 1

     # se crea un objeto httpresponse y se pasa como parámetro el mimetype
     # diciendo que es excel
    response = HttpResponse(mimetype="application/ms-excel")  # HttpResponse viene del modulo django.http
    nombre_archivo = "Centros.xlsx"
    contenido = "attachment; filename={0}".format(nombre_archivo)
    response["Content-Disposition"] = contenido
    libro.save(response)
    return response

# def some_view(request):
#     # Create the HttpResponse object with the appropriate PDF headers.
#     response = HttpResponse(content_type='application/pdf')
#     response['Content-Disposition'] = 'attachment; filename="somefilename.pdf"'

#     buffer = BytesIO()

#     # Create the PDF object, using the BytesIO object as its "file."
#     p = canvas.Canvas(buffer)

#     # Draw things on the PDF. Here's where the PDF generation happens.
#     # See the ReportLab documentation for the full list of functionality.
#     p.drawString(100, 100, "Hello world.")
#     #p.drawInlineImage(self, image, x,y, width=None,height=None)

#     # Close the PDF object cleanly.
#     p.showPage()
#     p.save()

#     # Get the value of the BytesIO buffer and write it to the response.
#     pdf = buffer.getvalue()
#     buffer.close()
#     response.write(pdf)
#     return response


from django.shortcuts import render
from django import forms
from django.forms import fields, widgets
from app01 import models

class DetailedForm(forms.Form):
    """
    required：是否必填
    error_messages：錯誤提示(要透過{{ obj.errors.user.0 }}調用)
    widget：訂製html插件(重要)
    label：訂製標籤名(要透過{{ obj.user.label }}調用)
    initial：默認值
    validators：自訂製驗證規則
    disabled：是否禁止編輯
    """
    user = fields.CharField(
        required=True,
        max_length=12,  # 最大長度
        min_length=3,  # 最小長度
        error_messages={},
        widget=widgets.TextInput(attrs={'n': 123}),  # 參數為自定義屬性
        label="用戶名",
        initial='請輸入用戶名',
        disabled=True,
    )
    age = fields.IntegerField(
        label="年齡",
        max_value=1000,  # 最大值
        min_value=0,  # 最小值
        error_messages={
            'max_value': '太大了',
        }
    )
    email = fields.EmailField(
        label="信箱",
    )
    smath = fields.DecimalField()  # 小數
    file = fields.FileField()
    city = fields.ChoiceField(
        choices=[(1, "上海"), (2, "北京"), (3, "南京")],  # (value, 顯示文字)
        initial=3,  # 默認value=3
    )
    hobby = fields.MultipleChoiceField(
        choices=[(1, "籃球"), (2, "電動"), (3, "小說")],  # (value, 顯示文字)
        initial=[1, 2],
    )
    # date = fields.DateField()  # 格式：2015-09-01
    # datetime = fields.DateTimeField()  # 格式：2015-09-01 11:12
    radio = fields.ChoiceField(
        choices=[(1, 'man'), (2, 'woman')],
        widget=widgets.RadioSelect(),
    )
    check = fields.MultipleChoiceField(
        choices=[(1, 'man'), (2, 'woman')],
        widget=widgets.CheckboxSelectMultiple(),
    )
    from django.core.validators import RegexValidator
    my = fields.CharField(
        # 自定義正則一
        # validators可設置多個RegexValidator
        # RegexValidator(正則, 錯誤信息)
        validators=[RegexValidator(r"^[0-9]+$", "請輸入數字"), RegexValidator(r"^87[0-9]+$", "請以87開頭")],
    )
    # 自定義正則二
    # fields.RegexField(正則)，只可設置單個正則
    my2 = fields.RegexField(
        r"^[0-9]+$",
        error_messages={
            "invalid": "請輸入數字",
        }
    )

def detailed_form(request):
    if request.method == "GET":
        obj = DetailedForm()
        return render(request, 'detailed_form.html', locals())
    else:
        # 文件存在request.FILES
        obj = DetailedForm(request.POST, request.FILES)
        obj.is_valid()
        print(obj.cleaned_data)
        return render(request, 'detailed_form.html', locals())

class DynamicForm(forms.Form):
    price = fields.IntegerField()
    # 因為是靜態字段，所以在程序啟動時就會給一塊內存去加載，除非重啟，否則不會動態更改
    user_id = fields.IntegerField(
        # widget=widgets.Select(choices=[(0, 'jamie'), (1, 'joker'), (2, 'jj'), ])
        # widget=widgets.Select(choices=models.UserInfo.objects.values_list('id', 'username'))
        widget=widgets.Select()  # 因為init會加載，所以不用再這邊查models
    )

    def __init__(self, *args, **kwargs):
        # super一定要寫在上面，用來拷貝所有的靜態字段複製給self.fields
        super(DynamicForm, self).__init__(*args, **kwargs)
        # 每次調用obj時就會執行一次init，所以在此更新choices，實現動態更新
        self.fields['user_id'].widget.choices = models.UserInfo.objects.values_list('id', 'username')

def dynamic_form(request):
    obj = DynamicForm()
    return render(request, 'dynamic_form.html', locals())


def main():
    from mkinit import static_mkinit
    # from os.path import exists, join, isdir
    import textwrap
    import os
    import six
    import argparse
    description = textwrap.dedent(
        '''
        discover and run doctests within a python package
        ''').strip('\n')

    parser = argparse.ArgumentParser(prog='python -m mkinit', description=description)
    parser.add_argument('modname_or_path', nargs='?', help='module or path to generate __init__.py for', default='.')
    parser.add_argument('--dry', action='store_true', default=False)

    parser.add_argument('--noattrs', action='store_true', default=False,
                        help='Do not generate attribute from imports')
    parser.add_argument('--nomods', action='store_true', default=False,
                        help='Do not generate modules imports')
    parser.add_argument('--noall', action='store_true', default=False,
                        help='Do not generate an __all__ variable')

    parser.add_argument('--relative', action='store_true', default=False,
                        help='Use relative . imports instead of <modname>')

    parser.add_argument('--ignore_all', action='store_true', default=False,
                        help='Ignores __all__ variables when parsing')

    args, unknown = parser.parse_known_args()
    ns = args.__dict__.copy()

    # modpath_or_name = sys.argv[1]
    print('ns = {!r}'.format(ns))
    modname_or_path = ns['modname_or_path']

    def touch(fpath):
        if six.PY2:  # nocover
            with open(fpath, 'a'):
                os.utime(fpath, None)
        else:
            flags = os.O_CREAT | os.O_APPEND
            with os.fdopen(os.open(fpath, flags=flags, mode=0o666)) as f:
                os.utime(f.fileno() if os.utime in os.supports_fd else fpath)

    # Hack in the ability to handle the case where __init__ does not exist yet
    # if isdir(modname_or_path) and not exists(join(modname_or_path, '__init__.py')):
    #     touch(join(modname_or_path, '__init__.py'))

    use_all = not ns['ignore_all']

    # Formatting options
    options = {
        'with_attrs': not ns['noattrs'],
        'with_mods': not ns['nomods'],
        'with_all': not ns['noall'],
        'relative': ns['relative'],
    }
    static_mkinit.autogen_init(modname_or_path, use_all=use_all,
                               options=options, dry=ns['dry'])

if __name__ == '__main__':
    main()


import torch
from torch import nn
from torch.autograd import Variable

from prettytable import PrettyTable
from tqdm import tqdm
import numpy as np

from datasets import BurstDataset, ShuffledBatchSequentialSampler, FakeBurstDataset

import sys
import os
import random

import losswise

def flip(x, dim):
    xsize = x.size()
    dim = x.dim() + dim if dim < 0 else dim
    x = x.contiguous().view(-1, *xsize[dim:])
    x = x.view(x.size(0), x.size(1), -1)[:, getattr(torch.arange(x.size(1)-1,
                      -1, -1), ('cpu','cuda')[x.is_cuda])().long(), :]
    return x.view(xsize)

class TeacherForcingProb:
    def __init__(self, slope, start_epoch_num):
        self.slope = slope
        self.tf_prob = 1
        for i in range(1, start_epoch_num):
            # if resuming training rather than starting at beginning,
            # fast-forward probability to proper place
            self.epoch_seen()

    def epoch_seen(self):
        if self.slope is None:
            return
        self.tf_prob = max(0, self.tf_prob - self.slope)

    def should_teacher_force(self):
        if self.slope is None:
            return False
        rand_num = random.uniform(0, 1)
        if rand_num < self.tf_prob:
            return True
        return False

def run_batch(masks, bursts, encoder, decoder, teacher_forcing, train_reversed, is_training):
    burst_lens = masks.sum(dim=1)
    max_seq_length = int(burst_lens.max().data.cpu().numpy()[0])

    # trim bursts and masks to be padded only up to the max_seq_length
    bursts_trimmed = bursts.narrow(1, 0, max_seq_length)
    masks_trimmed = masks.narrow(1, 0, max_seq_length)

    # print burst_lens.min().data.numpy()[0], burst_lens.max().data.numpy()[0]
    enc_out, hidden, cell = encoder(bursts_trimmed)

    if train_reversed:
        target_bursts = flip(bursts_trimmed, 1)   # still batch_size x output_seq_length, but reversed in seq_length dimension
    else:
        target_bursts = bursts_trimmed

    # output is size batch_size x output_seq_length x input_size
    if is_training and teacher_forcing.should_teacher_force():
        # to do teacher forcing, pass in something for target_output
        output = decoder(hidden, cell, max_seq_length, target_output=target_bursts.permute(1,0).unsqueeze(2))
    else:
        output = decoder(hidden, cell, max_seq_length)

    loss_fn = torch.nn.MSELoss(reduce=False)
    mses = loss_fn(output.squeeze(), target_bursts)
    # avg_mses is size [batch_size], giving mse for each elt in batch
    avg_mses = torch.sum(mses * masks_trimmed, 1) / burst_lens
    loss = avg_mses.mean()
    return loss

def summarize_params(model):
    params = []
    for param in model.parameters():
        params.append(param.sum().data.cpu().numpy()[0])
    return params

def run_epoch(dataset, batch_size, is_training, encoder, decoder, encoder_optimizer, decoder_optimizer,
              teacher_forcing, train_reversed, batch_by_len, save_path):
    if is_training:
        print('encoder params at epoch start', summarize_params(encoder))
        print('decoder params at epoch start', summarize_params(decoder))
    sys.stdout.flush()
    batch_sampler = ShuffledBatchSequentialSampler(dataset, batch_size=batch_size, drop_last=False)
    if batch_by_len:
        data_loader = torch.utils.data.DataLoader(dataset, batch_sampler=batch_sampler)
    else:
        data_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, drop_last=True)
    losses = []
    if is_training:
        encoder.train()
        decoder.train()
    else:
        encoder.eval()
        decoder.eval()
    for batch in tqdm(data_loader):
        if is_training:
            encoder_optimizer.zero_grad()
            decoder_optimizer.zero_grad()
        masks = Variable(batch['mask']) # [batch_size x pad_length]
        bursts = Variable(batch['burst'])   # [batch_size x pad_length]
        if torch.cuda.is_available():
            masks = masks.cuda()
            bursts = bursts.cuda()
        loss = run_batch(masks, bursts, encoder, decoder, teacher_forcing, train_reversed, is_training)

        if is_training:
            loss.backward()
            encoder_optimizer.step()
            decoder_optimizer.step()
        losses.append(loss.cpu().data[0])
    if is_training and save_path is not None:
        # save the model
        enc_save_path = save_path + '_enc.pkl'
        dec_save_path = save_path + '_dec.pkl'
        torch.save(encoder.state_dict(), enc_save_path)
        torch.save(decoder.state_dict(), dec_save_path)
    if is_training:
        teacher_forcing.epoch_seen()
    avg_loss = np.mean(losses)
    return avg_loss

def train_model(train_data, dev_data, test_data, encoder, decoder, save_dir=None, batch_size=50,
                num_epochs=50, start_epoch_num=1, lr=1e-3, weight_decay=1e-1, teacher_forcing_slope=None, train_reversed=False, batch_by_len=True,
               losswise_graph=None, params_dict={}):
    # Note: params dict is used only for printing
    if (save_dir is not None) and (not os.path.exists(save_dir)):
        os.makedirs(save_dir)
    print("start train_model")
    print("****************************************")
    print(params_dict)
    print("Batch size: {}, num_epochs: {}, lr: {}, weight_decay: {}".format(batch_size, num_epochs, lr, weight_decay))
    print("Encoder", encoder)
    print("Decoder", decoder)
    print("*****************************************")

    # might need to do parameters = filter(lambda p: p.requires_grad, model.parameters())
    encoder_optimizer = torch.optim.Adam(encoder.parameters(), lr=lr, weight_decay=weight_decay)
    decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=lr, weight_decay=weight_decay)
    teacher_forcing = TeacherForcingProb(teacher_forcing_slope, start_epoch_num)
    result_table = PrettyTable(["Epoch", "train loss", "dev loss", "test loss"])
    for epoch in range(start_epoch_num, num_epochs + start_epoch_num):
        print("epoch", epoch)
        if save_dir is None:
            save_path = None
        else:
            save_path = os.path.join(save_dir, 'epoch{}'.format(epoch))
        sys.stdout.flush()
        train_loss = run_epoch(train_data, batch_size, True, encoder, decoder, encoder_optimizer, decoder_optimizer, teacher_forcing, train_reversed, batch_by_len, save_path)
        dev_loss = run_epoch(dev_data, batch_size, False, encoder, decoder, encoder_optimizer, decoder_optimizer, teacher_forcing, train_reversed, batch_by_len, save_path)
        test_loss = run_epoch(test_data, batch_size, False, encoder, decoder, encoder_optimizer, decoder_optimizer, teacher_forcing, train_reversed, batch_by_len, save_path)
        result_table.add_row(
                            [ epoch ] +
                            [ "%.7f" % x for x in [train_loss, dev_loss, test_loss]
                                         ])
        print("{}".format(result_table))
        if losswise_graph is not None:
            losswise_graph.append(epoch, {'train_loss': train_loss, 'dev_loss': dev_loss, 'test_loss': test_loss})
        sys.stdout.flush()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 20 16:42:12 2019

@author: mitja
"""

import numpy as np
import matplotlib.pyplot as plt

input_data=np.loadtxt('gaia_tri_03_results.csv',dtype='str',skiprows=1,delimiter=',',usecols=(0,3,8,9,10,11,12,13,14,15,16,17,19))

name,sourceid,m1,m2,m3,m4,m5,m6,m7,m8,m9,ot,publish_name=input_data.T

unique_sourceid=np.unique(sourceid)

m1=m1.astype(float)
m2=m2.astype(float)
m3=m3.astype(float)
m4=m4.astype(float)
m5=m5.astype(float)
m6=m6.astype(float)
m7=m7.astype(float)
m8=m8.astype(float)
m9=m9.astype(float)

ot=ot.astype(float)

mag=np.zeros(len(input_data))

for i in range(len(input_data)):
    mag[i]=np.nanmedian([m1[i],m2[i],m3[i],m4[i],m5[i],m6[i],m7[i],m8[i],m9[i]])

idx=[]
for j in range(len(unique_sourceid)):
    idx.append(np.where(sourceid==unique_sourceid[j])[0])

for i in range(len(unique_sourceid)):
    plt.figure()
    plt.plot(ot[np.min(np.where(sourceid==unique_sourceid[i])[0]):np.max(np.where(sourceid==unique_sourceid[i])[0])+1]-2455000,mag[np.min(np.where(sourceid==unique_sourceid[i])[0]):np.max(np.where(sourceid==unique_sourceid[i])[0])+1],'b.')
    plt.gca().invert_yaxis()
    plt.xlabel('time[JD-2455000]')
    plt.ylabel('mag')
    plt.title('Publish name: %s'%publish_name[np.where(sourceid==unique_sourceid[i])[0][0]])
    plt.show()


import os, time, threading, logging, json, requests
print("=========================================================")
print(" _              ___ _ _                                  ")
print("(_)            / __) (_)                                 ")
print(" _ ____   ____| |__| |_ ____     ____   ____ ____   ____ ")
print("| |    \ / _  |  __) | |  _ \   |    \ / _  )    \ / _  )")
print("| | | | ( ( | | |  | | | | | |  | | | ( (/ /| | | ( (/ / ")
print("|_|_|_|_|\_|| |_|  |_|_| ||_/   |_|_|_|\____)_|_|_|\____)")
print("        (_____|        |_|                               ")
print("=========================================================")
try:
    how_much = int(input("[Default 100]How Much :"))
except ValueError:
    how_much = 100

api_url_imgflip = 'https://api.imgflip.com/get_memes'
datas = ''
format = "%(asctime)s: %(message)s"
logging.basicConfig(format=format, level=logging.INFO,datefmt="%H:%M:%S")

logging.info("[Information] Check if data.json is exists")
if os.path.exists('data.json'):
    logging.info("[Information] Found data.json")
    with open('data.json','r') as load:
        datas = json.load(load)
    logging.info("[Information] Data Loaded")
else:
    try:
        r = requests.get(api_url_imgflip,timeout=30)
        logging.info("[Information] Awaiting Information From API Endpoint")
        datas = json.loads(r.content.decode('utf-8'))
        with open('data.json','w', encoding='utf-8') as save:
            json.dump(datas,save,ensure_ascii=False,indent=4)
        logging.info("[Information] Data Loaded")
    except Exception:
        logging.error("[ERROR] Error Parsing Data")

hasil = datas['success']

i = 1

try:
    if hasil == True:
        logging.info("[Information] Getting information from API endpoint")
        for hasil in datas['data']['memes']:
            ID = hasil['id']
            name = hasil['name']
            url = hasil['url']
            width = hasil['width']
            height = hasil['height']
            box_count = hasil['box_count']
            #filename = '{}.jpg'.format(ID)
            filename = '{0} {1} {2}x{3}.jpg'.format(i,name,width,height)
            #logging.info("[Information] Get Image Url from API")
            Picture_request = requests.get(url)
            if Picture_request.status_code == 200:
                #logging.info("[Information] Image Loaded")
                with open(filename,'wb') as savePic:
                    savePic.write(Picture_request.content)
                    logging.warn("[Information] {} Image Downloaded".format(i))
                    i += 1
                    if i-1 >= how_much:
                        break
            else:
                logging.error("[ERROR] Request Failed")
                print("Exiting...")
                time.sleep(1)
                exit()
    else:
        logging.error("[ERROR] Cannot Parse Data from Api")
        #print(datas['success'])
        print("Some hopefully-useful statement about why it failed : ")
        print(datas['error_message'])
except Exception as e:
    print(e)


# (The MIT License)
#
# Copyright (c) 2016 Kura
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the 'Software'), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""Provides control functionality, including socket wrappers."""

import grp
import logging
import os
import pwd
import socket
try:
    import ssl
except ImportError:
    ssl = None

from .config import Config
from .exceptions import BlackholeRuntimeException

__all__ = ('pid_permissions', 'server', 'setgid', 'setuid')

logger = logging.getLogger('blackhole.control')
ciphers = ['ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384',
           'ECDHE-ECDSA-CHACHA20-POLY1305', 'ECDHE-RSA-CHACHA20-POLY1305',
           'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256',
           'ECDHE-ECDSA-AES256-SHA384', 'ECDHE-RSA-AES256-SHA384',
           'ECDHE-ECDSA-AES128-SHA256', 'ECDHE-RSA-AES128-SHA256']

def _context(use_tls=False):
    """
    Create a TLS context using the certificate, key and dhparams file.

    :param use_tls: Whether to create a TLS context or not.
    :type use_tls: :any:`bool`
    :returns: A TLS context or none.
    :rtype: :any:`ssl.SSLContext` or :any:`None`.

    .. note::

       Created with:

       - :any:`ssl.OP_NO_SSLv2`
       - :any:`ssl.OP_NO_SSLv3`
       - :any:`ssl.OP_NO_COMPRESSION`
       - :any:`ssl.OP_CIPHER_SERVER_PREFERENCE`

       Also responsible for loading Diffie Hellman ephemeral parameters if
       they're provided -- :any:`ssl.SSLContext.load_dh_params`

       If the ``-ls`` or ``--less-secure`` option is provided,
       :any:`ssl.OP_SINGLE_DH_USE` and :any:`ssl.OP_SINGLE_ECDH_USE` will be
       omitted from the context. --
       https://blackhole.io/command-line-options.html#command-line-options --
       added in :ref:`2.0.13`
    """
    if use_tls is False:
        return None
    config = Config()
    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
    ctx.load_cert_chain(config.tls_cert, config.tls_key)
    ctx.options |= ssl.OP_NO_SSLv2
    ctx.options |= ssl.OP_NO_SSLv3
    ctx.options |= ssl.OP_NO_COMPRESSION
    ctx.options |= ssl.OP_CIPHER_SERVER_PREFERENCE
    if not config.args.less_secure:
        ctx.options |= ssl.OP_SINGLE_DH_USE
        ctx.options |= ssl.OP_SINGLE_ECDH_USE
    ctx.set_ciphers(':'.join(ciphers))
    if config.tls_dhparams:
        ctx.load_dh_params(config.tls_dhparams)
    return ctx

def _socket(addr, port, family):
    """
    Create a socket, bind and listen.

    :param addr: The address to use.
    :type addr: :any:`str`
    :param port: The port to use.
    :type port: :any:`int`
    :param family: The type of socket to use.
    :type family: :any:`socket.AF_INET` or :any:`socket.AF_INET6`.
    :returns: A bound socket.
    :rtype: :any:`socket.socket`
    :raises: :any:`blackhole.exceptions.BlackholeRuntimeException`
    """
    sock = socket.socket(family, socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    try:
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
    except (AttributeError, OSError):
        pass
    if family == socket.AF_INET6:
        sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
    try:
        sock.bind((addr, port))
    except OSError:
        msg = 'Cannot bind to {}:{}.'.format(addr, port)
        logger.fatal(msg)
        sock.close()
        raise BlackholeRuntimeException(msg)
    os.set_inheritable(sock.fileno(), True)
    sock.listen(1024)
    sock.setblocking(False)
    return sock

def server(addr, port, family, flags={}, use_tls=False):
    """
    A socket and possibly a TLS context.

    Create an instance of :any:`socket.socket`, bind it and return a dictionary
    containing the socket object and a TLS context if configured.

    :param addr: The address to use.
    :type addr: :any:`str`
    :param port: The port to use.
    :type port: :any:`int`
    :param family: The type of socket to use.
    :type family: :any:`socket.AF_INET` or :any:`socket.AF_INET6`.
    :param flags: Flags to use.
    :type flags: :any:`dict`. Default: {}
    :param use_tls: Whether to create a TLS context or not.
    :type use_tls: :any:`bool`
    :returns: A bound socket, a TLS context if configured and any configured
              flags.
    :rtype: :any:`dict`
    """
    sock = _socket(addr, port, family)
    ctx = _context(use_tls=use_tls)
    return {'sock': sock, 'ssl': ctx, 'flags': flags}

def pid_permissions():
    """
    Change the pid file ownership.

    Called before :any:`blackhole.control.setgid` and
    :any:`blackhole.control.setuid` are called to stop
    :any:`blackhole.daemon.Daemon` losing permissions to modify the pid file.

    :raises: :any:`SystemExit` -- :any:`os.EX_USAGE`
    """
    config = Config()
    try:
        user = pwd.getpwnam(config.user)
        group = grp.getgrnam(config.group)
        os.chown(config.pidfile, user.pw_uid, group.gr_gid)
    except (KeyError, PermissionError):
        logger.error('Unable to change pidfile ownership permissions.')
        raise SystemExit(os.EX_USAGE)

def setgid():
    """
    Change group.

    Change to a less privileged group. Unless you're using it incorrectly --
    in which case, don't use it.
    :raises: :any:`SystemExit` -- :any:`os.EX_USAGE` or :any:`os.EX_NOPERM`

    .. note::

       MUST be called BEFORE setuid, not after.
    """
    config = Config()
    try:
        gid = grp.getgrnam(config.group).gr_gid
        os.setgid(gid)
    except KeyError:
        logger.error('Group \'%s\' does not exist.', config.group)
        raise SystemExit(os.EX_USAGE)
    except PermissionError:
        logger.error('You do not have permission to switch to group \'%s\'.',
                     config.group)
        raise SystemExit(os.EX_NOPERM)

def setuid():
    """
    Change user.

    Change to a less privileged user.Unless you're using it incorrectly --
    inwhich case, don't use it.
    :raises: :any:`SystemExit` -- :any:`os.EX_USAGE` or :any:`os.EX_NOPERM`

    .. note::

       MUST be called AFTER setgid, not before.
    """
    config = Config()
    try:
        uid = pwd.getpwnam(config.user).pw_uid
        os.setuid(uid)
    except KeyError:
        logger.error('User \'%s\' does not exist.', config.user)
        raise SystemExit(os.EX_USAGE)
    except PermissionError:
        logger.error('You do not have permission to switch to user \'%s\'.',
                     config.user)
        raise SystemExit(os.EX_NOPERM)


import csv
import yaml

cutflows={}
with open("nevents.list") as f:
  reader = csv.reader(f, delimiter=":")
  c = list(reader)
f.close()

for cc in c:
  cutflows[cc[0]] = cc[1]

def write_new_weights(sample):
  filename = sample + '.json'
  weights = yaml.load(file(filename))
  for k in weights.keys():
    tag = weights[k].keys()[0]
    try:
      weights[k][tag]['num events'] = cutflows[k]
    except:
      weights[k][tag]['num events'] = 0

  with open(sample+'.yml','w+') as outfile:
    outfile.write(yaml.dump(weights,default_flow_style=False))
  outfile.close()

write_new_weights('ttbar')


# Importing Required libraries
import pandas as pd
import numpy as np
import pickle
import cv2
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings("ignore")
from IPython.core.debugger import set_trace

def random_flip(image, steering_angle):
    """
    Function to Flip the images Horizontally randomly
    Flipped images will be assigned with the Negative steering angle mentioning the network in which direction curve associated with.

    """
    if np.random.rand() < 0.5:
        image = cv2.flip(image,1)
        steering_angle = -steering_angle
    return image, steering_angle

def random_shadow(image):
    """
    Function to add shadow in images randomly at random places, Random shadows meant to make the Convolution model learn
    Lanes and lane curvature patterns effectively in dissimilar places.
    """
    if np.random.rand() < 0.5:
        # (x1, y1) and (x2, y2) forms a line
        # xm, ym gives all the locations of the image
        x1, y1 = image.shape[1] * np.random.rand(), 0
        x2, y2 = image.shape[1] * np.random.rand(), image.shape[0]
        xm, ym = np.mgrid[0:image.shape[0], 0:image.shape[1]]

        mask = np.zeros_like(image[:, :, 1])
        mask[(ym - y1) * (x2 - x1) - (y2 - y1) * (xm - x1) > 0] = 1

        # choose which side should have shadow and adjust saturation
        cond = mask == np.random.randint(2)
        s_ratio = np.random.uniform(low=0.2, high=0.5)

        # adjust Saturation in HLS(Hue, Light, Saturation)
        hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
        hls[:, :, 1][cond] = hls[:, :, 1][cond] * s_ratio
        return cv2.cvtColor(hls, cv2.COLOR_HLS2RGB)
    else:
        return image

def random_brightness(image):
    """
    Function to add random brightness to random images. Random brightness will allow the model to learn and coverge on lane lines
    and lane curvatures even in low light conditions.

    """
    if np.random.rand() < 0.5:
        # HSV (Hue, Saturation, Value) is also called HSB ('B' for Brightness).
        image = np.array(image, dtype=np.uint8)
        hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
        ratio = 1.0 + 0.4 * (np.random.rand() - 0.5)
        hsv[:,:,2] =  hsv[:,:,2] * ratio
        return cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
    else:
        return image

def Image_Preprocessing(Drivinglog):
    '''
        Function to read each image in a directory and making them as a generalized array to feed Neural Network.
        Preprocessing of images included with 5 steps
            1. Reading all training images and the driving log csv to create Train features and predicting variable data.
            2. Translating the images with the use Random flipping and assigning a steering angle negatively for each flipped image.
            3. Randomly adding shadow to each image
            4. Concatenating all augmented data
            5. Splitting the data to train, test and valid randomly

    '''
    #1. Reading all training images and the driving log csv to create Train features and predicting variable data.
    CenterImages = np.concatenate([cv2.cvtColor(cv2.imread(load_dir +file), cv2.COLOR_BGR2RGB)[np.newaxis] for file in Drivinglog[Drivinglog.columns[0]].tolist()])
#     a = Drivinglog[Drivinglog.columns[3]].astype(float)
  #   y_data = (a + ((((~a.between(-0.05,0.01))  & (a.gt(0.01))).astype(int) * 0.05) + (((~a.between(-0.05,0.01))  & (a.lt(-0.01))).astype(int) * -0.05))).values[:,None]
    y_data = Drivinglog[Drivinglog.columns[3]].astype(float).values[:,None]

    #FlipedCenterImages = np.concatenate([cv2.flip(image,1)[np.newaxis] for image in CenterImages])
    print('Translating the Image data  ... ')
    # 2. Translating the images with the use Random flipping and assigning a steering angle negatively for each flipped image.
    val = [random_flip(image,steering_angle) for image, steering_angle in zip(CenterImages,y_data)]
    TranslatedImages = np.concatenate([i[0][np.newaxis] for i in val])
    TranslatedSangle = np.array([i[1] for i in val])
    print('Adding shadow to image data. ...')

    # 3. Randomly adding shadow to each image
    RBrightnesImages = np.concatenate([random_brightness(image)[np.newaxis] for image in CenterImages])
    print('Changing Brightness of image data ... ')

    # 4. Concatenating all augmented data
#     X_data = np.concatenate((CenterImages, TranslatedImages, RBrightnesImages))
#     y_data = np.concatenate((y_data, TranslatedSangle, y_data))
    X_data = CenterImages #RBrightnesImages
    y_data = y_data

    # 5. Splitting the data to train, test and valid randomly
    x_train, x_test, y_train, y_test = train_test_split(X_data, y_data, test_size=0.1, random_state=42)
    x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.1, random_state=42)

    print('Train data shape ', X_data.shape)

    with open('/opt/ProcessedData.data','wb') as file:
        pickle.dump([x_test,y_test],file)
        pickle.dump([x_valid,y_valid],file)
        pickle.dump([x_train,y_train],file)
        file.close()

load_dir = '../Data/SampleData/data/'
Drivinglog = pd.read_csv(load_dir + 'driving_log.csv')
Drivinglog = pd.concat([Drivinglog[Drivinglog.steering.ne(0)],Drivinglog[Drivinglog.steering.between(-0.01,0.01)][:300]]).reset_index(drop=True)
Image_Preprocessing(Drivinglog)


#!/usr/bin/env python
# -- coding: utf-8 --

from django.shortcuts import render_to_response
from .models import News
from .models import Book, Slider, Author, Quote, Tag, Publishers
from django.shortcuts import render
from django.shortcuts import HttpResponseRedirect
from django .http import Http404
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from .forms import ProposeForm
from django.core.mail import send_mail
import urllib
import re
from datetime import date
import MySQLdb
from xml.dom.minidom import *

# Books

def index(request):
    p = Book.objects.all()  # [:12]
    paginator = Paginator(p, 3)
    page = request.GET.get('page')
    try:
        p = paginator.page(page)
    except PageNotAnInteger:
        p = paginator.page(1)
    except EmptyPage:
        p = paginator.page(paginator.num_pages)
    s = Slider.objects.all()
    pub = Publishers.objects.all()
    # по категориям
    tv = Book.objects.filter(category='Экранизации')[:8]
    most = Book.objects.filter(category='Самые популярные')[:8]
    best_of_2013 = Book.objects.filter(category='Лучшие 2013')[:8]
    q = Quote.objects.all()
    n = News.objects.filter(category=u'Новости')[:2]
    context = {'p': p, 's': s, 'tv': tv, 'most': most, 'best_of_2013': best_of_2013, 'q': q, 'pub': pub, 'n': n}
    return render(request, 'index.html', context)

def book(request, id):
    d = Book.objects.get(id=id)
    g = Book.objects.filter(genre=d.genre)
    t = Tag.objects.filter(book_id=id)
    data = {'d': d, 'g': g, 't': t}
    return render(request, 'book.html', data)

def author(request, author):
    p = Book.objects.filter(author=author)
    a = Author.objects.filter(name=author)[:1]
    data = {'p': p, 'a': a}
    return render(request, 'author.html', data)

def year(request, year):
    y = Book.objects.filter(year=year)
    data = {'y': y}
    return render(request, 'year.html', data)

def publisher(request, publisher):
    p = Book.objects.filter(publisher=publisher)
    data = {'p': p}
    return render(request, 'publisher.html', data)

def tag(request, tag):
    g = Tag.objects.filter(tag=tag)
    data = {'g': g}
    return render(request, 'tag.html', data)

def category(request, category):
    c = Book.objects.filter(category=category)
    paginator = Paginator(c, 10)
    page = request.GET.get('page')
    try:
        c = paginator.page(page)
    except PageNotAnInteger:
        c = paginator.page(1)
    except EmptyPage:
        c = paginator.page(paginator.num_pages)
    data = {'c': c}
    return render(request, 'category.html', data)

def blog(request):
    n = News.objects.all().order_by('-date')
    data = {'n': n}
    return render(request, 'blog.html', data)

def blog_post(request, id):
    p = News.objects.get(id=id)
    n = News.objects.all()
    data = {'p': p, 'n': n}
    return render(request, 'post.html', data)

def blog_category(request, category):
    p = News.objects.filter(category=category)
    data = {'p': p}
    return render(request, 'blog_category.html', data)

def search_form(request):
    return render(request, 'base.html')

def search(request):
    errors = []
    if 'q' in request.GET:
        q = request.GET['q']
        if not q:
            errors.append('Вы не ввели в строку запрос')
        elif q == u'Поиск книг':
            errors.append('Ищите книги по запросу "Поиск книг"? Интересно...')
        else:
            books = Book.objects.filter(name__icontains=q)
            return render(request, 'search_results.html',
                {'books': books, 'query': q})
    return render(request, 'index.html',
        {'errors': errors})

def rating(request):
    p = Book.objects.order_by('-rating')
    paginator = Paginator(p, 10)
    page = request.GET.get('page')
    try:
        p = paginator.page(page)
    except PageNotAnInteger:
        p = paginator.page(1)
    except EmptyPage:
        p = paginator.page(paginator.num_pages)
    data = {'p': p}
    return render(request, 'rating.html', data)

#def allnews(request):
#    p = News.objects.filter(week=request)
#    context  = {'p':p}
#    return render(request, 'add.html', context)

# testing


import json
import requests
import sys

url = 'http://www.emspost.ru/api/rest'

#print("Original city:")
#originalCityName = input()
originalCityName = sys.argv[1]
#print("Deliviry city:")
#deliviryCityName = input()
deliviryCityName = sys.argv[2]
#print("Weight:")
#weight = input()
weight = sys.argv[3]

payload = {"method":"ems.get.locations", "type":"cities", "plain":"true"}
r = requests.get(url, params=payload)
r = json.loads(r.text)
cities = r["rsp"]["locations"]
cities_d = dict()
try:
	for x in cities:
		cities_d[x["name"]] = x["value"]
	origId = cities_d[originalCityName]
	deliId = cities_d[deliviryCityName]

	payload = {"method":"ems.calculate", "from":origId, "to":deliId, "weight":weight, "value": 11000}
	r = requests.get(url, params=payload)

	r = json.loads(r.text)["rsp"];
	print(json.dumps({ "price": r["price"],
						"time": r["term"]["min"] + "-" + r["term"]["max"],
					 }))
except:
	print('')


from sklearn import linear_model
import numpy as np
from matplotlib import pyplot
from sklearn.metrics import mean_absolute_error
from time import time

with open('../dataset/OnlineNewsPopularity/OnlineNewsPopularity.csv', 'r') as file:
    input = file.readlines()

names = input[0]
names = names.split(',')[2:]
data = input[1:]
data = [line.split(',') for line in data]
data = np.array(data)
data = data[:,2:].astype(float)
data[:,-1:]=np.log(data[:,-1:])

numIters = 100
trainTimeData = [0] * numIters
testTimeData = [0] * numIters
trainError = []
testError = []

for i in range(0,numIters):
    np.random.shuffle(data)
    X = data[:, :-1]
    Y = data[:, -1:]

    numPoints = len(Y)
    pTrain = 0.9
    index = int(numPoints*pTrain)
    Xtr = X[index:,:]
    Ytr = Y[index:,:]
    Xte = X[:index,:]
    Yte = Y[:index,:]

    #Start Timing (data split is not part of timing)
    startTrain = time()
    reg = linear_model.Ridge(alpha=0.05, fit_intercept=True, normalize=True)
    reg.fit(Xtr, Ytr)
    startTest = time()
    Ytr_pred = reg.predict(Xtr)
    Yte_pred = reg.predict(Xte)
    end = time()
    trainError.append(mean_absolute_error(Ytr, Ytr_pred))
    testError.append(mean_absolute_error(Yte, Yte_pred))

    trainTimeData[i] = startTest - startTrain
    testTimeData[i] = end - startTest

print("NumSamples: ",numIters)
print("Mean Training Time: ",np.mean(trainTimeData))
print("Mean Testing Time: ",np.mean(testTimeData))
print("Mean Train Error: ",np.mean(trainError))
print("Mean Test Error: ",np.mean(testError))

np.savetxt('timing_data/train_time_ridge.csv', trainTimeData)
np.savetxt('timing_data/test_time_ridge.csv', testTimeData)
# np.savetxt('../milestone_4/error_data/train_error_ridge.csv', trainError)
# np.savetxt('../milestone_4/error_data/test_error_ridge.csv', testError)


#!/usr/bin/python3
''' Python script that takes in 3 strings and sends a search request to the
Twitter API '''

import requests
from sys import argv
import base64

if __name__ == "__main__":
    consumer_key = argv[1]
    consumer_secret = argv[2]

    key = '{}:{}'.format(consumer_key, consumer_secret).encode('ascii')
    b64_key = base64.b64encode(key)
    b64_key = b64_key.decode('ascii')

    auth_url = 'https://api.twitter.com/oauth2/token'

    headers = {
        'Authorization': 'Basic {}'.format(b64_key),
        'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
        }

    data = {
        'grant_type': 'client_credentials'
    }

    r = requests.post(auth_url, headers=headers, data=data)
    access_token = r.json().get('access_token')

    search_headers = {
        'Authorization': 'Bearer {}'.format(access_token)
    }

    search_params = {
        'q': argv[3],
        'result_type': 'recent',
        'count': 15
    }

    search_url = 'https://api.twitter.com/1.1/search/tweets.json'
    search_resp = requests.get(search_url, headers=search_headers,
                               params=search_params)

    tweet_data = search_resp.json()
    statuses = tweet_data.get('statuses')
    count = 5
    for tuit in statuses:
        print('[{}] {} by {}'.format(tuit.get('id'), tuit.get('text'),
              tuit.get('user').get('name')))
        count -= 1
        if count == 0:
            break


import torch.nn as nn
from torch.nn import ReLU
import math
from dynnet.layers import (
    Input, Linear, BatchNorm, Flatten, CapNorm, Conv2d, Sum)
from dynnet.graph import Graph
from dynnet.filters import SmoothFilter

RESNET_CFGS = {
    50: [3, 4, 6, 3],
    101: [3, 4, 23, 3],
    152: [3, 8, 36, 3]
}

EXPANSION = 4

def block(graph, inp, planes, downsample, stride=1):
    conv1 = graph.add(Conv2d, out_channels=planes, kernel_size=1,
                      bias=False)(inp)
    bn1 = graph.add(BatchNorm)(conv1)
    filt1 = graph.add(SmoothFilter)(bn1)
    conv2 = graph.add(Conv2d, out_channels=planes, kernel_size=3,
                      stride=stride, padding=1, bias=False)(filt1)
    bn2 = graph.add(BatchNorm)(conv2)
    filt2 = graph.add(SmoothFilter)(bn2)
    conv3 = graph.add(Conv2d, out_channels=planes * EXPANSION,
                      kernel_size=1, bias=False)(filt2)
    bn3 = graph.add(BatchNorm)(conv3)
    ss = graph.add(Sum)(bn3, downsample)
    return graph.add(nn.ReLU, inplace=True)(ss)

class ResNet(nn.Module):

    def __init__(self, params):
        super(ResNet, self).__init__()
        num_classes = params['num_classes']
        layers = RESNET_CFGS[int(params['config'])]
        factor = int(params['factor'])
        self.graph = Graph()
        self.inplanes = 64 * factor
        self.inp = self.graph.add(Input, 3, 224, 224)()
        conv1 = self.graph.add(Conv2d, out_channels=self.inplanes,
                               kernel_size=7, stride=2, padding=3,
                               bias=False)(self.inp)
        filt1 = self.graph.add(SmoothFilter)(conv1)
        bn1 = self.graph.add(BatchNorm)(filt1)
        relu = self.graph.add(ReLU, inplace=True)(bn1)
        pool = self.graph.add(nn.MaxPool2d, kernel_size=3,
                              stride=2, padding=1)(relu)
        layer1 = self._make_layer(pool, 64 * factor, layers[0])
        layer2 = self._make_layer(layer1, 128 * factor, layers[1], stride=2)
        layer3 = self._make_layer(layer2, 256 * factor, layers[2], stride=2)
        layer4 = self._make_layer(layer3, 512 * factor, layers[3], stride=2)
        avgpool = self.graph.add(nn.AvgPool2d, 7, stride=1)(layer4)
        flat = self.graph.add(Flatten)(avgpool)
        self.result = self.graph.add(Linear, out_features=num_classes)(flat)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, inp, planes, blocks, stride=1):
        def down_sampler(x):
            if stride != 1 or self.inplanes != planes * EXPANSION:
                downsample = self.graph.add(Conv2d,
                                            out_channels=planes * EXPANSION,
                                            kernel_size=1, stride=stride,
                                            bias=False)(x)
                return self.graph.add(BatchNorm)(downsample)
            else:
                return x

        inp = block(self.graph, inp, planes, down_sampler(inp), stride)
        self.inplanes = planes * EXPANSION
        for i in range(1, blocks):
            inp = block(self.graph, inp, planes, inp)
        return inp

    def forward(self, x):
        return self.graph({self.inp: x}, self.result)[0]