import metrics
import numpy as np
from os import listdir
from tensorflow import keras
from numpy import ndarray, zeros, ones, where
from typing import Tuple, List
from PIL import Image
from utils import save_ndarray_as_image, upscale_image_pixels

def remove_alpha_channel(image:ndarray) -> ndarray:
    return image[:, :, :3] if len(image[0][0]) == 4 else image

def classify_each_pixel(image:ndarray) -> ndarray:
    # Classify red as any pixel that has times 2 the value of blue and green:
    red_double_green = image[:, :, 0] > np.uint16(image[:, :, 1])*2
    red_double_blue = image[:, :, 0] > np.uint16(image[:, :, 2])*2
    builds_pixels = np.where(np.logical_and(red_double_green, red_double_blue), 2, 0)
    # Classify black as any pixel that has all RGB values darker than 20:
    dark_max = 20
    red_dark = image[:, :, 0] < dark_max
    blue_dark = image[:, :, 1] < dark_max
    green_dark = image[:, :, 2] < dark_max
    roads_pixels = np.where(np.logical_and(red_dark, np.logical_and(blue_dark, green_dark)), 1, 0)
    # Unite all classes
    united_classes = builds_pixels + roads_pixels
    classified_pixels = np.where(united_classes > 2, 0, united_classes)
    # Return all classes united:
    return np.uint8(classified_pixels)

def predict_test_images(images_folder:str, predict_folder:str, model_path:str, save_predictions:bool) -> Tuple[List[ndarray], List[ndarray]]:
    print('> loading model ' + model_path)
    model:keras.Model = keras.models.load_model(model_path, compile=False)
    masks_rgb:List[ndarray] = []
    predictions_rgb:List[ndarray] = []

    files_name = tuple(filter(lambda name : 'test' in name, listdir(images_folder)))
    chunks_file_path = [images_folder + chunk_file_name for chunk_file_name in files_name]

    for chunk_index, chunk_file_path in enumerate(chunks_file_path):
        for pair_index, pair in enumerate(np.load(chunk_file_path)['arr_0']):
            print('> Predicted: chunk[%d] pair[%d]' % (chunk_index+1, pair_index+1))
            # Predict returns a list of predictions, so we get only the first one:
            prediction:ndarray = model.predict(np.array([pair[0]]))[0]
            # Its necessary to convert the ndarrays from [-1,1] to range [0,255] and put it only as integers:
            prediction_rgb:np.ndarray = np.uint8(upscale_image_pixels(prediction))
            map_rgb:np.ndarray = np.uint8(upscale_image_pixels(pair[0]))
            mask_rgb:np.ndarray = np.uint8(upscale_image_pixels(pair[1]))
            # Save the masks and predictions in arrays to post check the performance:
            masks_rgb.append(mask_rgb)
            predictions_rgb.append(prediction_rgb)
            # Save predictions if necessary:
            if save_predictions:
                # Generates the name of each prediction:
                group_name:str = 'chunk' + str(chunk_index) + '_pair' + str(pair_index)
                # Save all the images tested:
                save_ndarray_as_image(predict_folder, group_name + '_map', map_rgb)
                save_ndarray_as_image(predict_folder, group_name + '_mask', mask_rgb)
                save_ndarray_as_image(predict_folder, group_name + '_prediction', prediction_rgb)

    return (masks_rgb, predictions_rgb)

def segment_binary_matrix(image:ndarray, class_value:int) -> ndarray:
    return np.uint8(where(image == class_value, ones(image.shape), zeros(image.shape)))

# Matrices with all classes specified:
def classify_pairs(masks:List[ndarray], predictions:List[ndarray]) -> Tuple[List[ndarray], List[ndarray]]:
    print('> Classifying pixels')
    return ([classify_each_pixel(prediction) for prediction in predictions], [classify_each_pixel(remove_alpha_channel(mask)) for mask in masks])

# Binary matrices for each segmentation class that are relevant to apply IoU:
def segment_pairs(predictions_classes:List[np.ndarray], masks_classes:List[np.ndarray], class_label:int) -> Tuple[List[np.ndarray], List[np.ndarray]]:
    print('> Segmenting class matrices')
    return ([segment_binary_matrix(prediction, class_label) for prediction in predictions_classes], [segment_binary_matrix(mask, class_label) for mask in masks_classes])

# Run the validation of one given model:
def run(model_path:str, images_folder:str='./maps/preprocessed/', save_predictions:bool=True) -> None:
    predict_folder = 'maps/predictions/'

    blanks_IoU_results_file = predict_folder + 'IoU Results Blanks.txt'
    blanks_swIoU_results_file = predict_folder + 'swIoU Results Blanks.txt'
    blanks_pError_results_file = predict_folder + 'Pixel Error Results Blanks.txt'
    roads_IoU_results_file = predict_folder + 'IoU Results Roads.txt'
    roads_swIoU_results_file = predict_folder + 'swIoU Results Roads.txt'
    roads_pError_results_file = predict_folder + 'Pixel Error Results Roads.txt'
    builds_IoU_results_file = predict_folder + 'IoU Results Buildings.txt'
    builds_swIoU_results_file = predict_folder + 'swIoU Results Buildings.txt'
    builds_pError_results_file = predict_folder + 'Pixel Error Results Buildings.txt'

    open(blanks_IoU_results_file, 'w+').close()
    open(blanks_swIoU_results_file, 'w+').close()
    open(blanks_pError_results_file, 'w+').close()
    open(roads_IoU_results_file, 'w+').close()
    open(roads_swIoU_results_file, 'w+').close()
    open(roads_pError_results_file, 'w+').close()
    open(builds_IoU_results_file, 'w+').close()
    open(builds_swIoU_results_file, 'w+').close()
    open(builds_pError_results_file, 'w+').close()

    (masks, predictions) = predict_test_images(images_folder, predict_folder, model_path, save_predictions)
    (predictions_classes, masks_classes) = classify_pairs(masks, predictions)

    (predictions_blank_classes, masks_blank_classes) = segment_pairs(predictions_classes, masks_classes, 0)
    (predictions_road_classes, masks_road_classes) = segment_pairs(predictions_classes, masks_classes, 1)
    (predictions_build_classes, masks_build_classes) = segment_pairs(predictions_classes, masks_classes, 2)

    metrics.calculate_IoU('blanks', predictions_blank_classes, masks_blank_classes, blanks_IoU_results_file)
    metrics.calculate_IoU('roads', predictions_road_classes, masks_road_classes, roads_IoU_results_file)
    metrics.calculate_IoU('buildings', predictions_build_classes, masks_build_classes, builds_IoU_results_file)

    metrics.calculate_swIoU('blanks', predictions_blank_classes, masks_blank_classes, blanks_swIoU_results_file)
    metrics.calculate_swIoU('roads', predictions_road_classes, masks_road_classes, roads_swIoU_results_file)
    metrics.calculate_swIoU('buildings', predictions_build_classes, masks_build_classes, builds_swIoU_results_file)

    metrics.calculate_pixel_error('blanks', predictions_blank_classes, masks_blank_classes, blanks_pError_results_file)
    metrics.calculate_pixel_error('roads', predictions_road_classes, masks_road_classes, roads_pError_results_file)
    metrics.calculate_pixel_error('buildings', predictions_build_classes, masks_build_classes, builds_pError_results_file)


#from numba import jit, cuda
import numpy as np
import os
import matplotlib.pyplot as plt
import sfs
import scipy
import tqdm
import argparse
from data_lib import params_linear_2D
from data_lib import soundfield_generation as sg
#import scikit-image as skimage
#from scikit-image.metrics import structural_similarity as ssim

#from skimage.metrics import structural_similarity as ssim
#plt.rcParams.update({
    #"text.usetex": True,
    #"font.family": "sans-serif",
    #"font.sans-serif": ["Helvetica"],
    #'font.size': 20})

idxy = params_linear_2D.idx_lr_gd_y
idx_y = idxy[int(len(idxy)/2):]

print("Running TESTing set generation")

def plot_soundfield(cmap, P, n_f, selection, axis_label_size, tick_font_size, save_path=None, plot_ldspks=True, do_norm=True):
    figure = plt.figure(figsize=(20, 20))
    if do_norm:
        print("np.shape(P) = {}, \n np.shape(P[:, n_f]) = {} ".format(np.shape(P), np.shape(P[:, n_f])))
        #im = sfs.plot2d.amplitude(np.reshape(P[:, n_f], (params_linear_2D.N_sample, params_linear_2D.N_sample)), params_linear_2D.grid2D, xnorm=[0, 0, 0], cmap=cmap, vmin=-1.0, vmax=1.0, colorbar=False)
        #im = sfs.plot2d.amplitude(P[:, n_f], params_linear_2D.grid2D, xnorm=[0, 0, 0], cmap=cmap, vmin=-1.0, vmax=1.0, colorbar=False)
    else:
        print("in else")
        #im = sfs.plot2d.amplitude(np.reshape(P[:, n_f], (params_linear_2D.N_sample, params_linear_2D.N_sample)), params_linear_2D.grid2D,  cmap=cmap, colorbar=False, vmin=P[:, n_f].min(), vmax=P[:, n_f].max(), xnorm=None)
        #im = sfs.plot2d.amplitude(P[:, n_f], params_linear_2D.grid2D, xnorm=None, cmap=cmap, vmin=-1.0, vmax=1.0, colorbar=False)
    if plot_ldspks:
        sfs.plot2d.loudspeakers(params_linear_2D.array.x[selection], params_linear_2D.array.n[selection], a0=1, size=0.18)
    plt.xlabel('$x [m]$', fontsize=axis_label_size), plt.ylabel('$y [m]$', fontsize=axis_label_size)
    plt.tick_params(axis='both', which='major', labelsize=tick_font_size)
    #cbar = plt.colorbar(im, fraction=0.046)
    #cbar.ax.tick_params(labelsize=tick_font_size)
    # cbar.set_label('$NRE~[\mathrm{dB}]$',fontsize=tick_font_size))
    plt.tight_layout()
    if save_path is not None:
        #plt.savefig(save_path)
        print("save_path")
    plt.show()

#@jit(target_backend='cuda')
def main():
    # Arguments parse
    parser = argparse.ArgumentParser(description='Generate data for linear array setup')
    parser.add_argument('--base_dir', type=str, help="Base Data Directory", default='/nas/home/ralessandri/thesis_project/dataset')
    #parser.add_argument('--base_dir', type=str, help="Base Data Directory", default='C:/Users/rales/OneDrive/Desktop/POLIMI/TESI/dataset')
    ##parser.add_argument('--dataset_name', type=str, help="Base Data Directory", default='data_src_wideband_point_W_23_test')
    parser.add_argument('--gt_soundfield', type=bool, help='compute ground truth soundfield', default=True)
    parser.add_argument('--n_missing', type=int, help='number missing loudspeakers', default=0)
    args = parser.parse_args()
    propagate_filters = False
    eval_points = False
    control_points = True
    PLOT_RESULTS = False

    if control_points:
        c_points_x = params_linear_2D.idx_cp_x2
        c_points_y = params_linear_2D.idx_cp_y2

        c_pointsx_y = c_points_x[int(len(c_points_x) / 2):]

    dataset_path = '/nas/home/ralessandri/thesis_project/dataset/test'
    #dataset_path = 'C:/Users/rales/OneDrive/Desktop/POLIMI/TESI/pressure_matching_deep_learning/dataset/linear_array'

    # Setup
    # Grid of points where we actually compute the soundfield
    point = params_linear_2D.point
    #N_pts = len(point)
    N_pts = int((len(params_linear_2D.grid2D[0][0])))
    grid = params_linear_2D.grid2D

    # Secondary Sources Green function
    green_function_sec_sources_path = 'green_function_sec_sources_nl_' + str(params_linear_2D.N_lspks) + '_r_' + str(
        params_linear_2D.rangeX[0]) + '.npy'
    if os.path.exists(os.path.join(dataset_path, green_function_sec_sources_path)):
        G = np.load(os.path.join(dataset_path, green_function_sec_sources_path))
    else:
        G = np.zeros((N_pts, N_pts, params_linear_2D.N_lspks, params_linear_2D.N_freqs), dtype=complex)
        for n_l in tqdm.tqdm(range(params_linear_2D.N_lspks)):
            for n_f in range(len(params_linear_2D.f_axis)):
                # hankel_factor_1 = params_linear_2D.wc[n_f] / params_linear_2D.c  # , (params_linear_2D.N_lspks, 1)
                # print("array_pos = {}\nhankel_factor_2 = point[np] - array_pos = {}".format(params_linear_2D.array_pos, point - params_linear_2D.array_pos))
                # hankel_factor_2 = np.linalg.norm(grid - params_linear_2D.array_pos[n_l])  # , reps=(params_linear_2D.N_freqs, 1)  # ).transpose()
                k = params_linear_2D.wc[n_f] / params_linear_2D.c
                r = np.linalg.norm(grid - params_linear_2D.array_pos[n_l])
                # np.exp(-1j * k * r) / (4 * np.pi)
                # Points, Speakers, Frequencies
                # G[:, :, n_l, n_f] = (1j / 4) * scipy.special.hankel2(0, hankel_factor_1*hankel_factor_2)
                into_G = np.exp(-1j * k * r) / (4 * np.pi)
                # print("into_G = {}".format(np.shape(into_G)))
                G[:, :, n_l, n_f] = into_G  # / r # ! Sto volutamente non tenendo conto del decay!

        print("G is saved, shape = --> {}".format(np.shape(G)))
        np.save(os.path.join(dataset_path, green_function_sec_sources_path), G)

    # Check if array in grid points are equal
    for n_p in range(N_pts):
        if np.sum(np.linalg.norm(point[n_p] - params_linear_2D.array_pos, axis=1) == 0) > 0:
            print(str(n_p))

    N_missing = args.n_missing
    N_lspks = params_linear_2D.N_lspks - N_missing
    if N_missing>0:
        lspks_config_path = 'lspk_config_nl_'+str(params_linear_2D.N_lspks)+'_missing_'+str(N_missing)+'.npy'
        lspk_config_path_global = os.path.join(dataset_path, 'setup', lspks_config_path)
        if os.path.exist(lspk_config_path_global):
            idx_missing = np.load(lspk_config_path_global)
            print('Loaded existing mic config')
        else:
            idx_missing = np.random.choice(np.arange(params_linear_2D.N_lspks), size=N_missing, replace=False)
            print("saved missing indexes, shape --> {}".format(np.shape(idx_missing)))
            np.save(lspk_config_path_global, idx_missing)
        theta_l = np.delete(params_linear_2D.theta_l, idx_missing)
        G = np.delete(G, idx_missing, axis=1)

    if eval_points:
        P_gt = np.zeros((len(params_linear_2D.src_pos_trainT), int(len(params_linear_2D.idx_lr_gd_y)), int(len(params_linear_2D.idx_lr_gd_x)), params_linear_2D.N_freqs), dtype=complex)
    elif control_points:
        P_gt = np.zeros((len(params_linear_2D.src_pos_trainT), int(len(c_points_y)), int(len(c_points_x)), params_linear_2D.N_freqs), dtype=complex)
    else:
        P_gt = np.zeros((len(params_linear_2D.src_pos_trainT), N_pts, N_pts, params_linear_2D.N_freqs),dtype=complex)  # 3204, 64

    d_array = np.zeros((params_linear_2D.N_lspks, N_lspks, params_linear_2D.N_freqs), dtype=complex)
    for n_s in tqdm.tqdm(range(len(params_linear_2D.src_pos_testT))):

        xs = np.append(params_linear_2D.src_pos_testT[n_s], 2)

        # Multiply filters for hergoltz density function
        if propagate_filters:
            P = np.zeros((N_pts, len(params_linear_2D.wc)), dtype=complex)
            for n_p in range(N_pts):
                P[n_p, :] = np.sum(G[n_p] * d_array[n_s], axis=0)

        if args.gt_soundfield:
            for n_f in range(params_linear_2D.N_freqs):
                # Sources, Points, Frequencies
                #P_gt[n_s, :, :, n_f] = (1j / 4) * scipy.special.hankel2(0, (params_linear_2D.wc[n_f] / params_linear_2D.c) * np.linalg.norm(point[:, :2] - xs))
                if eval_points:
                    into_P_gt_eval = sfs.fd.source.point(params_linear_2D.wc[n_f], xs, grid) * np.linalg.norm(grid - xs)
                    for i in params_linear_2D.idx_lr_gd_x:
                        for j in idx_y:
                            into_P_gt_eval[-j, i] = 0
                    into_P_gt_eval_y = into_P_gt_eval[params_linear_2D.idx_lr_gd_y]
                    P_gt[n_s, :, :, n_f] = into_P_gt_eval_y[:,params_linear_2D.idx_lr_gd_x]

                elif control_points:
                    into_P_gt_eval = sfs.fd.source.point(params_linear_2D.wc[n_f], xs, grid) * np.linalg.norm(grid - xs)  # this last multiplication is not valid if I consider the decay!
                    for i in c_points_x:
                        for j in c_pointsx_y:
                            into_P_gt_eval[-j, i] = 0
                    into_P_gt_eval_y = into_P_gt_eval[c_points_y]
                    P_gt[n_s, :, :, n_f] = into_P_gt_eval_y[:, c_points_x]

                else:
                    P_gt[n_s, :, :, n_f] = sfs.fd.source.point(params_linear_2D.wc[n_f], xs, grid) * np.linalg.norm(grid - xs)

    if PLOT_RESULTS:
            # Plot params
            print("Plotting")
            selection = np.ones_like(params_linear_2D.array_pos[:, 0])
            selection = selection == 1  # ?
            #n_s = 32
            n_f = 41
            print(str(params_linear_2D.f_axis[n_f]))
            cmap = 'RdBu_r'
            tick_font_size = 70
            axis_label_size = 90
            plot_soundfield(cmap, P_gt[n_s], n_f, selection, axis_label_size, tick_font_size, save_path=None, plot_ldspks=True,
                            do_norm=False)

    if args.gt_soundfield:
        if control_points:
            print("P_gt saved shape --> {}".format(np.shape(P_gt)))
            np.save(os.path.join(dataset_path, 'gt_soundfield_train_half_cp_double_train.npy'), P_gt)
        else:
            print("P_gt saved shape --> {}".format(np.shape(P_gt)))
            np.save(os.path.join(dataset_path, 'gt_soundfield_train.npy'), P_gt)

    #np.save(os.path.join(dataset_path, 'filters_config_nl_'+str(N_lspks)+'_missing_'+str(N_missing)+'.npy'))

    # P_gt = [sources, control points, frequencies]
    #print("x = {}\ny = {}".format(np.shape(P_gt[:, :, 0]), np.shape(P_gt[0, :, :])))
    #plt.figure(figsize=(10,10))
    #plt.plot(P_gt[:64, :, 0].T, P_gt[0, :, :], 'g*')
    #plt.xlabel('$x [m]$'), plt.ylabel('$y [m]$')
    #plt.xlim(-2, 2)
    #plt.ylim(-2, 2)
    ##plt.setzlim(0, 4)
    ##plt.legend(['eval points', 'control points', 'loudspeakers', 'train sources', 'test sources'])
    #plt.title("#2D")
    #plt.show()

print("Everything Defined")
if __name__ == '__main__':
    print("Main running")
    main()
    print("##   End Generate TESTing   ##")


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from statsmodels.formula.api import ols

churn = pd.read_csv('data/26/churn.csv')
fish = pd.read_csv('data/26/fish.csv')
ad_conversion = pd.read_csv('data/26/ad_conversion.csv')
sp500_yearly_returns = pd.read_csv('data/26/sp500_yearly_returns.csv')
taiwan_real_estate = pd.read_csv('data/26/taiwan_real_estate2.csv')

#%% 1. Simple Linear Regression Modeling
fish.mean()

fish.mass_g.corr(fish.length_cm)

sns.scatterplot('mass_g', 'length_cm', data=fish)
plt.show()

sns.regplot('mass_g', 'length_cm', data=fish, ci=None)
plt.show()

#%%
# Import seaborn with alias sns
import seaborn as sns

# Import matplotlib.pyplot with alias plt
import matplotlib.pyplot as plt

# Draw the scatter plot
sns.scatterplot(x='n_convenience', y='price_twd_msq', data=taiwan_real_estate)

# Show the plot
plt.show()

#%%
# Import seaborn with alias sns
import seaborn as sns

# Import matplotlib.pyplot with alias plt
import matplotlib.pyplot as plt

# Draw the scatter plot
sns.scatterplot(x="n_convenience",
                y="price_twd_msq",
                data=taiwan_real_estate)

# Draw a trend line on the scatter plot of price_twd_msq vs. n_convenience
sns.regplot(x='n_convenience', y='price_twd_msq', data=taiwan_real_estate,
            ci=None,
            scatter_kws={'alpha': 0.5})

# Show the plot
plt.show()

#%% 1.2 Fitting a linear regression
estate_fit = ols('price_twd_msq ~ dist_to_mrt_m', data=taiwan_real_estate)
estate_fit = estate_fit.fit()
print(estate_fit.params)

#%%
# Import the ols function
from statsmodels.formula.api import ols

# Create the model object
mdl_price_vs_conv = ols('price_twd_msq ~ n_convenience', data=taiwan_real_estate)

# Fit the model
mdl_price_vs_conv = mdl_price_vs_conv.fit()

# Print the parameters of the fitted model
print(mdl_price_vs_conv.params)

#%%
# On average, a house with zero convenience stores nearby had a price of 8.2242 TWD per square meter.

#%%
# If you increase the number of nearby convenience stores by one,
# then the expected increase in house price is 0.7981 TWD per square meter.

#%% 1.3 Categorical explanatory variables
fish

sns.displot(data=fish,
            x='mass_g',
            col='species',
            col_wrap=2,
            bins=9)
plt.show()

summary_stats = fish.groupby('species')['mass_g'].mean()
print(summary_stats)

fish_ols = ols('mass_g ~ species', data=fish).fit()
fish_ols.params

fish_ols_zero = ols('mass_g ~ species + 0', data=fish).fit()
fish_ols_zero.params

#%%
# Histograms of price_twd_msq with 10 bins, split by the age of each house
sns.displot(data=taiwan_real_estate,
            x='price_twd_msq',
            col='house_age_years',
            bins=10)

# Show the plot
plt.show()

#%%
# Calculate the mean of price_twd_msq, grouped by house age
mean_price_by_age = taiwan_real_estate.groupby('house_age_years')['price_twd_msq'].mean()

# Print the result
print(mean_price_by_age)

#%%
# Create the model, fit it
mdl_price_vs_age = ols('price_twd_msq ~ house_age_years', data=taiwan_real_estate).fit()

# Print the parameters of the fitted model
print(mdl_price_vs_age.params)

#%%
# Update the model formula to remove the intercept
mdl_price_vs_age0 = ols("price_twd_msq ~ house_age_years + 0", data=taiwan_real_estate).fit()

# Print the parameters of the fitted model
print(mdl_price_vs_age0.params)

#%% 2. Predictions and model objects

#%% 2.1 Making predictions
bream = fish[fish['species'] == 'Bream']
bream.head()

sns.regplot(x='length_cm',
            y='mass_g',
            data=bream,
            ci=None)
plt.show()

bream_ols = ols('mass_g ~ length_cm', data=bream).fit()
bream_ols.params

exp_data = pd.DataFrame({'length_cm': np.arange(20, 41)})
bream_ols.predict(exp_data)

predict_data = exp_data.assign(
    mass_g=bream_ols.predict(exp_data)
)

predict_data

fig = plt.figure()
sns.regplot(x='length_cm',
            y='mass_g',
            ci=None,
            data=bream)
sns.scatterplot(x='length_cm',
                y='mass_g',
                data=predict_data,
                color='r',
                marker='s')
plt.show()

little_bream = pd.DataFrame({'length_cm': [10]})
pred_little_bream = little_bream.assign(
    mass_g=mdl_mass_vs_lenght.predict(little_bream)
)

#%%
# Import numpy with alias np
import numpy as np

# Create explanatory_data
explanatory_data = pd.DataFrame({'n_convenience': np.arange(0, 11)})

# Use mdl_price_vs_conv to predict with explanatory_data, call it price_twd_msq
price_twd_msq = mdl_price_vs_conv.predict(explanatory_data)

# Create prediction_data
prediction_data = explanatory_data.assign(
    price_twd_msq = price_twd_msq)

# Print the result
print(prediction_data)

#%%
# Create a new figure, fig
fig = plt.figure()

sns.regplot(x="n_convenience",
            y="price_twd_msq",
            data=taiwan_real_estate,
            ci=None)
# Add a scatter plot layer to the regplot
sns.scatterplot(x="n_convenience",
                y="price_twd_msq",
                data=prediction_data,
                color='r')

# Show the layered plot
plt.show()

#%%
# Define a DataFrame impossible
impossible = pd.DataFrame({'n_convenience': [-1, 2.5]})

#%% 2.2 Working with model objects
mdl_mass_vs_length = ols("mass_g ~ length_cm", data=bream).fit()
print(mdl_mass_vs_length.params)
print(mdl_mass_vs_length.fittedvalues)
print(mdl_mass_vs_length.resid)
print(mdl_mass_vs_length.summary())

#%%
# Print the model parameters of mdl_price_vs_conv
print(mdl_price_vs_conv.params)

# Print the fitted values of mdl_price_vs_conv
print(mdl_price_vs_conv.fittedvalues)

# Print the residuals of mdl_price_vs_conv
print(mdl_price_vs_conv.resid)

# Print a summary of mdl_price_vs_conv
print(mdl_price_vs_conv.summary())
#%%
# Get the coefficients of mdl_price_vs_conv
coeffs = mdl_price_vs_conv.params

# Get the intercept
intercept = coeffs[0]

# Get the slope
slope = coeffs[1]

# Manually calculate the predictions
price_twd_msq = intercept + slope * explanatory_data
print(price_twd_msq)

# Compare to the results from .predict()
print(price_twd_msq.assign(predictions_auto=mdl_price_vs_conv.predict(explanatory_data)))

#%% 2.3 Regression to the mean
pearson = pd.read_csv('data/26/Pearson.txt', sep="\t", header=None, skiprows=1)
pearson.columns = ['Father', 'Son']

fig = plt.figure()
sns.scatterplot(x='Father',
                y='Son',
                data=pearson)
plt.axline(xy1=(80, 80),
           slope=1,
           linewidth=2,
           color='g')
sns.regplot(x='Father',
            y='Son',
            data=pearson,
            ci=None,
            line_kws={'color': 'black'})
plt.axis('equal')
plt.show()

fig = plt.figure()
plt.axline(xy1=(80, 80),
           slope=1,
           linewidth=2,
           color='g')
sns.regplot(x='Father',
            y='Son',
            data=pearson,
            ci=None,
            line_kws={'color': 'black'})
plt.axis('equal')
plt.show()

mdl_son_vs_father = ols('Son ~ Father',
                        data=pearson).fit()
print(mdl_son_vs_father.params)

really_tall_father = pd.DataFrame(
    {'Father': [90]})
mdl_son_vs_father.predict(really_tall_father)

really_short_father = pd.DataFrame(
    {'Father': [50]})
mdl_son_vs_father.predict(really_short_father)

#%%
# Create a new figure, fig
fig = plt.figure()

# Plot the first layer: y = x
plt.axline(xy1=(0,0), slope=1, linewidth=2, color="green")

# Add scatter plot with linear regression trend line
sns.regplot(x='return_2018',
            y='return_2019',
            data=sp500_yearly_returns,
            ci=None)

# Set the axes so that the distances along the x and y axes look the same
plt.axis('equal')

# Show the plot
plt.show()

#%%
mdl_returns = ols("return_2019 ~ return_2018", data=sp500_yearly_returns).fit()

# Create a DataFrame with return_2018 at -1, 0, and 1
explanatory_data = pd.DataFrame({'return_2018': [-1, 0, 1]})

# Use mdl_returns to predict with explanatory_data
print(mdl_returns.predict(explanatory_data))

#%% 2.4 Transforming variables
perch = fish[fish['species'] == 'Perch']
perch.head()

sns.regplot(x='length_cm',
            y='mass_g',
            data=perch,
            ci=None)
plt.show()

perch['length_cm_cubed'] = perch['length_cm'] ** 3

sns.regplot(x='length_cm_cubed',
            y='mass_g',
            data=perch,
            ci=None)
plt.show()

mdl_perch = ols('mass_g ~ length_cm_cubed', data=perch).fit()
mdl_perch.params

explanatory_data = pd.DataFrame({'length_cm_cubed': np.arange(10, 41, 5) ** 3,
                                 'length_cm': np.arange(10, 41, 5)})

prediction_data = explanatory_data.assign(
    mass_g=mdl_perch.predict(explanatory_data)
)

prediction_data

fig = plt.figure()
sns.regplot(x='length_cm_cubed',
            y='mass_g',
            data=perch,
            ci=None)
sns.scatterplot(data=prediction_data,
                x='length_cm_cubed',
                y='mass_g',
                color='r',
                markers='s')
plt.show()

fig = plt.figure()
sns.regplot(x='length_cm',
            y='mass_g',
            data=perch,
            ci=None)
sns.scatterplot(data=prediction_data,
                x='length_cm',
                y='mass_g',
                color='r',
                markers='s')
plt.show()

#%%
ad_conversion
sns.regplot(x='spent_usd',
            y='n_impressions',
            data=ad_conversion,
            ci=None)

ad_conversion['sqer_spend'] = np.sqrt(ad_conversion['spent_usd'])
ad_conversion['sqer_n_impressions'] = np.sqrt(ad_conversion['n_impressions'])

sns.regplot(x='sqer_spend',
            y='sqer_n_impressions',
            data=ad_conversion,
            ci=None)

#%%
mdl_ad = ols('sqer_n_impressions ~ sqer_spend', data=ad_conversion).fit()

explanatory_data = pd.DataFrame({
    'sqer_spend': np.sqrt(np.arange(0, 601, 100)),
    'spent_usd': np.arange(0, 601, 100)
})

prediction_data = explanatory_data.assign(sqrt_n_impress=mdl_ad.predict(explanatory_data),
                                          n_impression=mdl_ad.predict(explanatory_data) ** 2)

prediction_data

#%%
# Create sqrt_dist_to_mrt_m
taiwan_real_estate["sqrt_dist_to_mrt_m"] = np.sqrt(taiwan_real_estate["dist_to_mrt_m"])

plt.figure()

# Plot using the transformed variable
sns.regplot(x='sqrt_dist_to_mrt_m',
            y='price_twd_msq',
            data=taiwan_real_estate,
            ci=None)
plt.show()

#%%
# Create sqrt_dist_to_mrt_m
taiwan_real_estate["sqrt_dist_to_mrt_m"] = np.sqrt(taiwan_real_estate["dist_to_mrt_m"])

# Run a linear regression of price_twd_msq vs. sqrt_dist_to_mrt_m
mdl_price_vs_dist = ols("price_twd_msq ~ sqrt_dist_to_mrt_m", data=taiwan_real_estate).fit()

# Use this explanatory data
explanatory_data = pd.DataFrame({"sqrt_dist_to_mrt_m": np.sqrt(np.arange(0, 81, 10) ** 2),
                                 "dist_to_mrt_m": np.arange(0, 81, 10) ** 2})

# Use mdl_price_vs_dist to predict explanatory_data
prediction_data = explanatory_data.assign(
    price_twd_msq = mdl_price_vs_dist.predict(explanatory_data)
)

fig = plt.figure()
sns.regplot(x="sqrt_dist_to_mrt_m", y="price_twd_msq", data=taiwan_real_estate, ci=None)

# Add a layer of your prediction points
sns.scatterplot(data=prediction_data, x='sqrt_dist_to_mrt_m', y='price_twd_msq', color='r')
plt.show()

#%%
ad_conversion["qdrt_n_impressions"] = ad_conversion["n_impressions"] ** 0.25
ad_conversion["qdrt_n_clicks"] = ad_conversion["n_clicks"] ** 0.25

mdl_click_vs_impression = ols("qdrt_n_clicks ~ qdrt_n_impressions", data=ad_conversion, ci=None).fit()

explanatory_data = pd.DataFrame({"qdrt_n_impressions": np.arange(0, 3e6+1, 5e5) ** .25,
                                 "n_impressions": np.arange(0, 3e6+1, 5e5)})

# Complete prediction_data
prediction_data = explanatory_data.assign(
    qdrt_n_clicks = mdl_click_vs_impression.predict(explanatory_data)
)

# Print the result
print(prediction_data)
#%%
# Back transform qdrt_n_clicks
prediction_data["n_clicks"] = prediction_data["qdrt_n_clicks"] ** 4

# Plot the transformed variables
fig = plt.figure()
sns.regplot(x="qdrt_n_impressions", y="qdrt_n_clicks", data=ad_conversion, ci=None)

# Add a layer of your prediction points
sns.scatterplot(data=prediction_data, x='qdrt_n_impressions', y='qdrt_n_clicks', color='r')
plt.show()

#%% 3. Assessing model fit

#%% 3.1 Quantifying model fit

print(bream_ols.summary())

bream_ols.rsquared

bream['length_cm'].corr(bream['mass_g']) ** 2

bream_ols.mse_resid

np.sqrt(bream_ols.mse_resid)

# RSE
df = len(bream.index) - 2
np.sqrt(sum(bream_ols.resid ** 2) / df)

# RMSE
np.sqrt(sum(bream_ols.resid ** 2) / len(bream.index))

#%%
# Calculate mse_orig for mdl_click_vs_impression_orig
mse_orig = mdl_click_vs_impression_orig.mse_resid

# Calculate rse_orig for mdl_click_vs_impression_orig and print it
rse_orig = np.sqrt(mse_orig)
print("RSE of original model: ", rse_orig)

# Calculate mse_trans for mdl_click_vs_impression_trans
mse_trans = mdl_click_vs_impression_trans.mse_resid

# Calculate rse_trans for mdl_click_vs_impression_trans and print it
rse_trans = np.sqrt(mse_trans)
print("RSE of transformed model: ", rse_trans)

#%% 3.2 Visualizing model fit
# Resid vs fitted
sns.residplot(x='length_cm',
              y='mass_g',
              data=bream,
              lowess=True)
plt.xlabel('Fitted values')
plt.ylabel('Residuals')
plt.show()

# QQ plot
from statsmodels.api import qqplot
qqplot(data=bream_ols.resid, fit=True, line='45')
plt.show()

# Scale-location plot
model_norm_resud_bream = bream_ols.get_influence().resid_studentized_internal
model_norm_abs_sqrt_bream = np.sqrt(np.abs(model_norm_resud_bream))
sns.regplot(x=bream_ols.fittedvalues,
            y=model_norm_abs_sqrt_bream,
            ci=None,
            lowess=True)
plt.xlabel('Fitted values')
plt.ylabel('Sqrt of abs val of std resid')
plt.show()

#%%
# Plot the residuals vs. fitted values
sns.residplot(x='n_convenience', y='price_twd_msq', data=taiwan_real_estate, lowess=True)
plt.xlabel("Fitted values")
plt.ylabel("Residuals")

# Show the plot
plt.show()

#%%
# Import qqplot
from statsmodels.api import qqplot

# Create the Q-Q plot of the residuals
qqplot(data=mdl_price_vs_conv.resid, fit=True, line="45")

# Show the plot
plt.show()

#%%
# Preprocessing steps
model_norm_residuals = mdl_price_vs_conv.get_influence().resid_studentized_internal
model_norm_residuals_abs_sqrt = np.sqrt(np.abs(model_norm_residuals))

# Create the scale-location plot
sns.regplot(x=mdl_price_vs_conv.fittedvalues, y=model_norm_residuals_abs_sqrt, ci=None, lowess=True)
plt.xlabel("Fitted values")
plt.ylabel("Sqrt of abs val of stdized residuals")

# Show the plot
plt.show()

#%% 3.3 Outliers, leverage, and influence
roach = fish[fish['species'] == 'Roach']
roach.head()

sns.regplot(x='length_cm',
            y='mass_g',
            data=roach,
            ci=None)
plt.show()

roach['extreme_l'] = ((roach['length_cm'] < 15) | (roach['length_cm'] > 26))

fig = plt.figure()
sns.regplot(x='length_cm',
            y='mass_g',
            data=roach,
            ci=None)
sns.scatterplot(x='length_cm',
                y='mass_g',
                data=roach,
                hue='extreme_l')
plt.show()

roach['extreme_m'] = roach['mass_g'] < 1

fig = plt.figure()
sns.regplot(x='length_cm',
            y='mass_g',
            data=roach,
            ci=None)
sns.scatterplot(x='length_cm',
                y='mass_g',
                data=roach,
                hue='extreme_l',
                style='extreme_m')
plt.show()

mdl_roach = ols('mass_g ~ length_cm', data=roach).fit()
summary_roach = mdl_roach.get_influence().summary_frame()
roach['leverage'] = summary_roach['hat_diag']
roach.head()

roach['cooks_dist'] = summary_roach['cooks_d']
roach.head()

roach.sort_values('cooks_dist', ascending=False)

roach_not_short = roach[roach['length_cm'] != 12.9]
sns.regplot(x='length_cm',
            y='mass_g',
            data=roach,
            ci=None,
            line_kws={'color': 'g'})
sns.regplot(x='length_cm',
            y='mass_g',
            data=roach_not_short,
            ci=None,
            line_kws={'color': 'r'})
plt.show()

#%%
# Create summary_info
summary_info = mdl_price_vs_dist.get_influence().summary_frame()

# Add the hat_diag column to taiwan_real_estate, name it leverage
taiwan_real_estate["leverage"] = summary_info["hat_diag"]

# Add the cooks_d column to taiwan_real_estate, name it cooks_dist
taiwan_real_estate['cooks_dist'] = summary_info['cooks_d']

# Sort taiwan_real_estate by cooks_dist in descending order and print the head.
print(taiwan_real_estate.sort_values('cooks_dist', ascending=False).head())

#%% 4. Simple Logistic Regression Modeling

#%% 4.1 Why you need logistic regression
churn.head()

mdl_churn_vs_recency_lm = ols('has_churned ~ time_since_last_purchase', data=churn).fit()
mdl_churn_vs_recency_lm.params

intercept, slope = mdl_churn_vs_recency_lm.params

sns.scatterplot(x='time_since_last_purchase',
                y='has_churned',
                data=churn)
plt.axline(xy1=(0, intercept), slope=slope)
plt.xlim(-10, 10)
plt.ylim(-0.2, 1.2)
plt.show()

from statsmodels.formula.api import logit
mdl_churn_vs_recency_logit = logit('has_churned ~ time_since_last_purchase', data=churn).fit()
mdl_churn_vs_recency_logit.params

sns.regplot(x='time_since_last_purchase',
            y='has_churned',
            data=churn,
            ci=None,
            logistic=True)
plt.axline(xy1=(0, intercept), slope=slope, color='black')
plt.show()

#%%
sns.displot(data=churn, x='time_since_last_purchase', col='has_churned')
plt.show()
sns.displot(data=churn, x='time_since_first_purchase', col='has_churned')
plt.show()

#%%
# Draw a linear regression trend line and a scatter plot of time_since_first_purchase vs. has_churned
sns.regplot(x="time_since_first_purchase",
            y="has_churned",
            data=churn,
            ci=None,
            line_kws={"color": "red"})

# Draw a logistic regression trend line and a scatter plot of time_since_first_purchase vs. has_churned
sns.regplot(x='time_since_first_purchase',
            y='has_churned',
            data=churn,
            ci=None,
            logistic=True,
            line_kws={"color": "blue"})

plt.show()

#%%
# Import logit
from statsmodels.formula.api import logit

# Fit a logistic regression of churn vs. length of relationship using the churn dataset
mdl_churn_vs_relationship = logit('has_churned ~ time_since_first_purchase', data=churn).fit()

# Print the parameters of the fitted model
print(mdl_churn_vs_relationship.params)

#%% 4.2 Predictions and odds ratios
mdl_recency = logit('has_churned ~ time_since_last_purchase', data=churn).fit()

explanatory_data = pd.DataFrame(
    {'time_since_last_purchase': np.arange(-1, 6.25, 0.25)}
)

prediction_data = explanatory_data.assign(has_churned=mdl_recency.predict(explanatory_data))

sns.regplot(x='time_since_last_purchase',
            y='has_churned',
            data=churn,
            ci=None,
            logistic=True)
sns.scatterplot(x='time_since_last_purchase',
                y='has_churned',
                data=prediction_data,
                color='r')
plt.show()

prediction_data['most_likely_outcome'] = np.round(prediction_data['has_churned'])

sns.regplot(x='time_since_last_purchase',
            y='has_churned',
            data=churn,
            ci=None,
            logistic=True)
sns.scatterplot(x='time_since_last_purchase',
                y='most_likely_outcome',
                data=prediction_data,
                color='r')
plt.show()

prediction_data['odds_ratio'] = prediction_data['has_churned'] / (1 - prediction_data['has_churned'])

sns.lineplot(x='time_since_last_purchase',
             y='odds_ratio',
             data=prediction_data)
plt.axhline(y=1, linestyle='dotted')
plt.show()

sns.lineplot(x='time_since_last_purchase',
             y='odds_ratio',
             data=prediction_data)
plt.axhline(y=1, linestyle='dotted')
plt.yscale('log')
plt.show()

prediction_data['log_odds_ratio'] = np.log(prediction_data['odds_ratio'])

#%%
# Create prediction_data
prediction_data = explanatory_data.assign(
    has_churned=mdl_churn_vs_relationship.predict(explanatory_data)
)

# Print the head
print(prediction_data.head())
#%%
# Create prediction_data
prediction_data = explanatory_data.assign(
    has_churned = mdl_churn_vs_relationship.predict(explanatory_data)
)

fig = plt.figure()

# Create a scatter plot with logistic trend line
sns.regplot(x='time_since_first_purchase', y='has_churned', ci=None, logistic=True, data=churn)

# Overlay with prediction_data, colored red
sns.scatterplot(x='time_since_first_purchase',
                y='has_churned',
                data=prediction_data,
                color='r')

plt.show()

#%%
# Update prediction data by adding most_likely_outcome
prediction_data["most_likely_outcome"] = np.round(prediction_data["has_churned"])

fig = plt.figure()

# Create a scatter plot with logistic trend line (from previous exercise)
sns.regplot(x="time_since_first_purchase",
            y="has_churned",
            data=churn,
            ci=None,
            logistic=True)

# Overlay with prediction_data, colored red
sns.scatterplot(x='time_since_first_purchase',
                y='most_likely_outcome',
                data=prediction_data,
                color='r')

plt.show()

#%%
# Update prediction data with odds_ratio
prediction_data["odds_ratio"] = prediction_data["has_churned"] / (1 - prediction_data["has_churned"])

fig = plt.figure()

# Create a line plot of odds_ratio vs time_since_first_purchase
sns.lineplot(x='time_since_first_purchase',
             y='odds_ratio',
             data=prediction_data)

# Add a dotted horizontal line at odds_ratio = 1
plt.axhline(y=1, linestyle="dotted")

plt.show()
#%%
# Update prediction data with log_odds_ratio
prediction_data["log_odds_ratio"] = np.log(prediction_data["odds_ratio"])

fig = plt.figure()

# Update the line plot: log_odds_ratio vs. time_since_first_purchase
sns.lineplot(x="time_since_first_purchase",
             y="log_odds_ratio",
             data=prediction_data)

# Add a dotted horizontal line at log_odds_ratio = 0
plt.axhline(y=0, linestyle="dotted")
plt.yscale('log')
plt.show()

#%% 4.2 Quantifying logistic regression fit
actual_response = churn['has_churned']
predicted_response = np.round(mdl_recency.predict())
outcomes = pd.DataFrame(
    {'actual_response': actual_response,
     'predicted_response': predicted_response}
)
print(outcomes.value_counts(sort=False))

conf_matrix = mdl_recency.pred_table()
print(conf_matrix)

from statsmodels.graphics.mosaicplot import mosaic
mosaic(conf_matrix)

TN = conf_matrix[0, 0]
TP = conf_matrix[1, 1]
FN = conf_matrix[1, 0]
FP = conf_matrix[0, 1]

acc = (TN + TP) / (TN + TP + FN + FP)
acc

sens = TP / (FN + TP)
sens

spec = TN / (TN + FP)
spec

#%%
# Get the actual responses
actual_response = churn['has_churned']

# Get the predicted responses
predicted_response = np.round(mdl_churn_vs_relationship.predict())

# Create outcomes as a DataFrame of both Series
outcomes = pd.DataFrame({'actual_response': actual_response,
                         'predicted_response': predicted_response})

# Print the outcomes
print(outcomes.value_counts(sort = False))

#%%
# Import mosaic from statsmodels.graphics.mosaicplot
from statsmodels.graphics.mosaicplot import mosaic

# Calculate the confusion matrix conf_matrix
conf_matrix = mdl_churn_vs_relationship.pred_table()

# Print it
print(conf_matrix)

# Draw a mosaic plot of conf_matrix
mosaic(conf_matrix)
plt.show()

#%%
# Extract TN, TP, FN and FP from conf_matrix
TN = conf_matrix[0, 0]
TP = conf_matrix[1, 1]
FN = conf_matrix[1, 0]
FP = conf_matrix[0, 1]

# Calculate and print the accuracy
accuracy = (TN + TP) / (TN + TP + FN + FP)
print("accuracy: ", accuracy)

# Calculate and print the sensitivity
sensitivity = TP / (FN + TP)
print("sensitivity: ", sensitivity)

# Calculate and print the specificity
specificity = TN / (TN + FP)
print("specificity: ", specificity)

26 - Introduction to Regression with statsmodels in Python.py


from flask_restful import Resource
from candidatos.modelos.modelos import db, CandidatoSchema, Candidato
from flask import request
from flask_jwt_extended import jwt_required, create_access_token
from datetime import datetime
from celery import Celery

celery_app = Celery(__name__, broker='redis://127.0.0.1:6379/0')
candidato_schema = CandidatoSchema()

class VistaCandidatosC(Resource):
    @jwt_required()
    def post(self):
        nuevo_candidato = Candidato(nombre=request.json['nombre'],\
                                profesion=request.json['profesion'])
        db.session.add(nuevo_candidato)
        db.session.commit()
        return candidato_schema.dump(nuevo_candidato)

class VistaCandidatoC(Resource):
    @jwt_required()
    def put(self, id_candidato):
        candidato = Candidato.query.get_or_404(id_candidato)
        candidato.nombre = request.json.get('nombre', candidato.nombre)
        candidato.profesion = request.json.get('profesion', candidato.profesion)
        db.session.commit()
        return candidato_schema.dump(candidato)


import sys
import openpyxl
import os
from openpyxl import Workbook
from datetime import date
import pandas as pd

def main():
    sales_csv = get_sales_csv()
    orders_directory = create_orders_directory(sales_csv)
    process_sales_data(sales_csv, orders_directory)

# Get path of sales data CSV file from the command line
def get_sales_csv():
    # Check whether command line parameter provided
    # Check whether provide parameter is valid path of file
    if len(sys.argv) < 2:
        print("Error: No Command line parameter is given.")
        sys.exit()
    csv_file = sys.argv[1]
    if not os.path.exists(csv_file):
        print("Error: Given File path does not exist.")

        sys.exit()

    return csv_file

# Create the directory to hold the individual order Excel sheets

def create_orders_directory(sales_csv):
    csv_directory = os.path.dirname(sales_csv)
    today = date.today().strftime('%Y-%m-%d')
    orders_directory = os.path.join(csv_directory, 'Orders' + today)

    if not os.path.exists(orders_directory):
        os.mkdir(orders_directory)

    return orders_directory

    # Get directory in which sales data CSV file resides
    # Determine the name and path of the directory to hold the order data files
    # Create the order directory if it does not already exist

# Split the sales data into individual orders and save to Excel sheets
def process_sales_data(sales_csv, orders_dir):
    # Import the sales data from the CSV file into a DataFrame
    # Insert a new "TOTAL PRICE" column into the DataFrame
    # Remove columns from the DataFrame that are not needed
    # Group the rows in the DataFrame by order ID
    # For each order ID:
        # Remove the "ORDER ID" column
        # Sort the items by item number
        # Append a "GRAND TOTAL" row
        # Determine the file name and full path of the Excel sheet
        # Export the data to an Excel sheet
        # TODO: Format the Excel sheet

    sales_data = pd.read_csv(sales_csv)
    sales_data['TOTAL PRICE'] = sales_data['QUANTITY OF ITEM'] * sales_data['PRICE OF ITEM']
    sales_data = sales_data[['DATE OF ORDER', 'ITEM NUMBER', 'PRODUCT LINE', 'PRODUCT CODE', 'QUANTITY OF ITEM', 'PRICE OF ITEM', 'TOTAL PRICE', 'STATUS', 'CUSTOMER NAME']]
    Orders = sales_data.groupby(by='ORDER ID')

    for Order_id, order_data in Orders:

        Order_data = Order_data.sort_values(by='ITEM NUMBER')
        Order_data = Order_data.reset_index(drop=True)
        grand_total = Order_data['TOTAL PRICE'].sum()
        Order_data = Order_data.append({'ITEM NUMBER': 'GRAND TOTAL', 'TOTAL PRICE': grand_total}, ignore_index=True)

        Order_file = os.path.join(orders_dir, str(Order_id) + ".xlsx")
        Order_data.to_excel(Order_file, Index=False, Engine='openpyxl')
        # TODO: Format the Excel sheet
        wb = openpyxl.load_workbook(Order_file)
        sheet = wb.worksheets[0]
        # Format the Excel sheet
        sheet.column_dimensions['A'].width = 11
        sheet.column_dimensions['B'].width = 13
        sheet.column_dimensions['C'].width = 15
        sheet.column_dimensions['D'].width = 15
        sheet.column_dimensions['E'].width = 15
        sheet.column_dimensions['F'].width = 13
        sheet.column_dimensions['G'].width = 13
        sheet.column_dimensions['H'].width = 10
        sheet.column_dimensions['I'].width = 30

        for row in sheet.item_rows(min_row=2, min_col=4, max_col=5):
            for cell in row:
                cell.number_format = '"$"#,##0.00'
        Workbook.save(Order_file)
        pass

if __name__ == '__main__':
    main()


from tkinter import *
import threading
from tkinter import messagebox
import psutil

from PIL import ImageTk,Image
from scapy.all import Ether, ARP, srp, sniff, conf
from tkinter.ttk import *
# Create the root window
# with specified size and title
root = Tk()
root.configure(background="#0e0d30")
root.title("Minimalistic tool")
#root.geometry("520x300")

label1 = Label(root, text = "Detection of Promiscuous mode and ARP poisoning", background="#000d1a" ,foreground="lightgreen", font="Serif 14 bold").pack(pady=10)
background_image =ImageTk.PhotoImage(file='imagess.jfif')
pic=Label(image=background_image).pack(fill="none",expand=True)

e = threading.Event()

addrs = psutil.net_if_addrs()
OPTIONS = [
   "----"
    ]
for interface in addrs:
    OPTIONS.append(interface)

def get_mac(ip):
    """
    Returns the MAC address of `ip`, if it is unable to find it
    for some reason, throws `IndexError`
    """
    p = Ether(dst='ff:ff:ff:ff:ff:ff')/ARP(pdst=ip)
    result = srp(p, timeout=3, verbose=True)[0]
    return result[0][1].hwsrc

def process(packet):
    # if the packet is an ARP packet
    if packet.haslayer(ARP):
        # if it is an ARP response (ARP reply)
        if packet[ARP].op == 2:
            try:

                # get the real MAC address of the sender
                real_mac = get_mac(packet[ARP].psrc)
                print(real_mac)
                # get the MAC address from the packet sent to us
                response_mac = packet[ARP].hwsrc
                print(response_mac)
                # if they're different, definetely there is an attack
                if real_mac != response_mac:

                    g.insert(0,'YOU ARE BEING ATTACKED')

                    e.set()

            except IndexError:

                # unable to find the real mac
                # may be a fake IP or firewall is blocking packets
                pass

#This is used to call sniff fuction
def sniffs(e):
    g.delete(0,'end')

    iface = variable.get()

    sniff(store=False, prn=process, iface=iface, timeout = 15,stop_filter=lambda p: e.is_set())

    if not e.is_set():

        g.insert(0,'YOU ARE SAFE')
    e.clear()

# this fuction is used to
def promiscs(e1):
    try:
      y.delete(0, 'end')
      ip=e1.get()
      if ip == "":
        y.insert(0, 'ENTER IP ')
        return

      t=get_macs(ip)
      y.insert(0,'ON ')
    except:
      y.insert(0,'OFF')

# here we send a packet with dst as 01:00:00:00:00:00 such that no other device in the network except for a promiscuous mode enabled device recives it.
def get_macs(ip):
    promisc_test = Ether(dst='01:00:00:00:00:00')/ARP(pdst=ip)
    result = srp(promisc_test,timeout=3,verbose=True)[0]
    return result[0][1].hwsrc

# define a function for 2nd toplevel
# window which is not associated with
# any parent window

def open_Toplevel2():

    style = Style()

# This will be adding style, and
# naming that style variable as
# W.Tbutton (TButton is used for ttk.Button).

    style.configure('W.TButton', font =
                   ('calibri', 10, 'bold', 'underline')
                    ,foreground='red')
    # Create widget
    top2 = Toplevel(bg='#0e0d30')

    # define title for window
    top2.title("Promiscious Mode")

    # specify size
    top2.geometry("450x250")
    # Create label
    label = Label(top2,
                  text = "Promiscious Mode", font='Serif 18 bold',background="#0e0d30",foreground="lightgreen").pack()

    #bgimage =ImageTk.PhotoImage(Image.open("back.jpg"))
    #background2=Label(image=bgimage).pack()
    label = Label(top2,
                  text = "IP Address :").place(relx=0.1,rely=0.2,relheight=0.1,relwidth=0.3)
    e1 = Entry(top2)
    e1.place(relx=0.5,rely=0.2,relheight=0.1,relwidth=0.3)
    l=e1.get()
    # Create exit button.
    exit = Button(top2,text = "Exit" , style = 'W.TButton',
                    command = top2.destroy,cursor="X_cursor")
    exit.place(relx=0.1,rely=0.5,relheight=0.1,relwidth=0.3)
    start = Button(top2, text = "Start", command = lambda : threading.Thread(target=promiscs,args=[e1]).start(),cursor="spider"
                    )
    start.place(relx=0.5,rely=0.5,relheight=0.1,relwidth=0.3)
    result = Label(top2,
                  text = "Result :").place(relx=0.1,rely=0.8,relheight=0.1,relwidth=0.3)
    global y
    y = Entry(top2,font='Serif 10 bold')
    y.place(relx=0.5,rely=0.8,relheight=0.1,relwidth=0.3)

    # Display untill closed manually.
    top2.mainloop()

global progress
progress = Progressbar(root, orient = HORIZONTAL,
              length = 100, mode = 'determinate')

# define a function for 1st toplevel
# which is associated with root window.

def open_Toplevel1():

    # Create widget
    top1 = Toplevel(bg='#0e0d30')
    style = Style()

    # Define title for window
    top1.title("ARP POISONING")
    style.configure('W.TButton', font =
                   ('calibri', 10, 'bold', 'underline')
                    ,foreground='red')

    # specify size
    top1.geometry("400x250")

    # Create label
    label = Label(top1,
                  text = "ARP POISONING", font='Serif 18 bold',foreground="lightgreen",background="#0e0d30").pack(pady=10)
    label = Label(top1,
                  text = "Specify interface :").place(relx=0.1,rely=0.22,relheight=0.1,relwidth=0.3)
    global variable
    variable = StringVar(top1)
    variable.set(OPTIONS[0]) # default value

    w = OptionMenu(top1, variable,*OPTIONS)
    w.place(relx=0.5,rely=0.22,relheight=0.1,relwidth=0.3)

    # Create Exit button

    button1 = Button(top1, text = "Exit", style = 'W.TButton',
                     command = top1.destroy,cursor="X_cursor")
    button1.place(relx=0.1,rely=0.5,relheight=0.1,relwidth=0.3)
    button = Button(top1, text = "Start", command = lambda : threading.Thread(target=sniffs(e)).start(), cursor="spider"
                    )
    button.place(relx=0.5,rely=0.5,relheight=0.1,relwidth=0.3)
    label = Label(top1,
                  text = "Result :").place(relx=0.1,rely=0.8,relheight=0.1,relwidth=0.3)

    global g
    g = Entry(top1)
    g.place(relx=0.5,rely=0.8,relheight=0.12,relwidth=0.5)

    # create button to open toplevel2

    # Display untill closed manually
    top1.mainloop()

# Create button to open toplevel1
R1 = Button(root, text = "PROMISCUOUS MODE", command = open_Toplevel2,cursor="target"
                 )

R1.pack(padx=40,pady=10,ipadx=40)

R2 = Button(root, text = "ARP POISONING",command = open_Toplevel1,cursor="target")
R2.pack(padx=40,pady=10,ipadx=55)
#label1.grid()

# position the button
#R1.place(x = 20, y = 100)
#R2.place(x =20, y = 200)

#R3.place(x=20, y = 90)
ourMessage ='“Technology trust is a good thing, but control is a better one.”'
messageVar = Message(root, text = ourMessage)
messageVar.config(bg='lightgreen')
messageVar.pack()
#messageVar.place(x = 400, y = 200)

# Display untill closed manually
root.mainloop()

Detection of ARP spoofing and Promiscuous Mode/Project/project.py


import m4m_sync

from sqlalchemy.exc import InternalError, IntegrityError
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm import joinedload
from sqlalchemy import func

from server.database.models import (
	Object,
	Controller,
	Sensor,
	User,
	UserInfo,
	UserSocialTokens,
)

from server.errors import (
	ConflictError,
	ObjectNotFoundError,
	ObjectExistsError,
	UserNoAccessError,
)
from datetime import datetime

time_field = 'timestamp'

class BaseSqlManager:
	model = None

	def __init__(self, session):
		self.session = session

	def create(self, data):
		obj = self.model(**data)
		try:
			self.session.add(obj)
			self.session.flush()
		except IntegrityError:
			raise ObjectExistsError(object='Record', property='Property')
		except InternalError:
			raise ConflictError()

		self.session.refresh(obj)
		return obj

	def get_all(self):
		return self.session.query(self.model).all()

	def get_by_id(self, id_):
		try:
			return self.session.query(self.model).filter_by(id=id_).one()
		except NoResultFound:
			raise ObjectNotFoundError(object='Record')

	def delete(self, id_):
		self.session.query(self.model).filter_by(id=id_).delete()

class ObjectManager(BaseSqlManager):
	model = Object

	def get_all_for_user(self, user_id: int):
		return self.session.query(self.model) \
			.filter_by(user_id=user_id) \
			.options(joinedload(Object.controllers).joinedload(Controller.sensors)) \
			.all()

	def get_by_id_for_user(self, object_id: int, user_id: int):
		return self.session.query(self.model) \
			.filter_by(user_id=user_id) \
			.filter_by(id=object_id) \
			.one()

	def update_for_user(self, object_id: int, user_id: int, data: dict):
		return self.session.query(self.model) \
			.filter_by(id=object_id) \
			.filter_by(user_id=user_id) \
			.update(data)

	def delete_for_user(self, object_id: int, user_id: int):
		return self.session.query(self.model) \
			.filter_by(id=object_id) \
			.filter_by(user_id=user_id) \
			.delete()

class ControllerManager(BaseSqlManager):
	model = Controller

	def get_for_user(self, controller_id: int, user_id: int):
		return self.session.query(self.model) \
			.join(Controller.object) \
			.filter_by(id=controller_id) \
			.filter_by(user_id=user_id).one()

	def create_for_user(self, user_id: int, data: dict):
		object_id = data['object_id']
		can_access = self.session.query(func.count(Object.id)) \
			.filter_by(id=object_id) \
			.filter_by(user_id=user_id)

		if can_access == 0:
			raise UserNoAccessError()

		return self.create(data)

	def __can_access(self, controller_id: int, user_id: int):
		return self.session.query(func.count(self.model.id)) \
				   .join(Controller.object) \
				   .filter_by(id=controller_id) \
				   .filter_by(user_id=user_id) != 0

	def delete_for_user(self, controller_id: int, user_id: int):
		if not self.__can_access(controller_id, user_id):
			raise UserNoAccessError()

		self.session.query(self.model) \
			.filter_by(id=controller_id) \
			.delete()

	def update_for_user(self, controller_id: int, user_id: int, data: dict):
		can_access = self.session.query(func.count(self.model.id)) \
						 .join(self.model.object) \
						 .filter(self.model.id == controller_id) \
						 .filter_by(user_id=user_id) != 0

		if not self.__can_access(controller_id, user_id) or not can_access:
			raise UserNoAccessError()

		return self.session.query(self.model) \
			.filter_by(id=controller_id) \
			.update(data)

class SensorManager(BaseSqlManager):
	model = Sensor

	default_names = {
		5: 'OBD',
		6: 'GPS',
	}

	def create_or_update(self, _id: str, data: dict):
		query = self.session.query(self.model).filter_by(id=_id)
		if query.scalar():
			query.update(data)
		else:
			name = data.pop('name', self.default_names[data['sensor_type']])
			self.session.add(
				Sensor(
					id=_id,
					name=name,
					activation_date=datetime.now(),
					controller_id=1,
					**data,
				),
			)

	def create_for_user(self, user_id: int, data: dict):
		controller_id = data['controller_id']
		can_access = self.session.query(func.count(Controller.id)) \
						 .join(Controller.object) \
						 .filter(Controller.id == controller_id) \
						 .filter(Object.user_id == user_id) != 0

		if not can_access:
			raise UserNoAccessError()

		return self.create(data)

	def __can_access(self, sensor_id: str, user_id: int):
		return self.session.query(func.count(self.model.id)) \
				   .join(self.model.controller) \
				   .join(Controller.object) \
				   .filter(self.model.id == sensor_id) \
				   .filter(Object.user_id == user_id) != 0

	def get_for_user(self, sensor_id: str, user_id: int):
		return self.session.query(self.model) \
			.join(self.model.controller) \
			.join(Controller.object) \
			.filter(self.model.id == sensor_id) \
			.filter(Object.user_id == user_id) \
			.one()

	def delete_for_user(self, sensor_id: str, user_id: int):
		if not self.__can_access(sensor_id, user_id):
			raise UserNoAccessError()

		return self.session.query(self.model) \
			.filter_by(id=sensor_id) \
			.delete()

	def update_for_user(self, sensor_id: str, user_id: int, data: dict):
		can_access = self.session.query(func.count(self.model.id)) \
						 .join(self.model.controller) \
						 .join(Controller.object) \
						 .filter(self.model.id == sensor_id) \
						 .filter(Object.user_id == user_id) != 0

		if not self.__can_access(sensor_id, user_id) or not can_access:
			raise UserNoAccessError()

		return self.session.query(self.model) \
			.filter_by(id=sensor_id) \
			.update(data)

class UserManager(BaseSqlManager):
	model = User

	def save_new(self, login, pwd_hash):
		user = self.create({
			'login': login,
			'pwd_hash': pwd_hash,
		})

		UserInfoManager(self.session).create({
			'user_id': user.id
		})

		UserSocialTokensManager(self.session).create({
			'user_id': user.id
		})

		ObjectManager(self.session).create({
			'name': 'default',
			'user_id': user.id,
		})

		return user

	def get_by_login(self, login):
		try:
			return self.session.query(self.model).filter_by(login=login).one()
		except NoResultFound:
			raise ObjectNotFoundError(object='User')

class UserInfoManager(BaseSqlManager):
	model = UserInfo

	def get_by_user_id(self, user_id):
		try:
			return self.session.query(self.model).filter_by(user_id=user_id).one()
		except NoResultFound:
			raise ObjectNotFoundError(object='user_info')

	def get_all(self, with_login=False):
		return self.session.query(self.model).options(joinedload(UserInfo.user)).all()

	def update(self, user_id, info):
		return self.session.query(self.model).filter_by(user_id=user_id).update(info)

class UserSocialTokensManager(BaseSqlManager):
	model = UserSocialTokens

	def get_by_user_id(self, user_id: int) -> UserInfo:
		return self.session.query(self.model).filter_by(user_id=user_id).one()

	def update(self, user_id: int, data: dict):
		return self.session.query(self.model).filter_by(user_id=user_id).update(data)

	def sync_user(self, user_token: UserSocialTokens):
		if user_token.yandex_disk:
			store = m4m_sync.YaDiskStore(token=user_token.yandex_disk)
			self.__sync(user_token.user_id, store)
		else:
			controllers = self.session.query(Controller) \
				.options(joinedload(Controller.sensors)) \
				.filter(Object.user_id == user_token.user_id) \
				.all()
			for controller in controllers:
				for sensor in controller.sensors:
					self.session.delete(sensor)
				self.session.delete(controller)

	def __sync(self, user_id: int, store: m4m_sync.stores.BaseStore):
		remote_controllers = store.get_controllers()
		local_controllers = self.session.query(Controller) \
			.options(joinedload(Controller.object), joinedload(Controller.sensors)) \
			.filter(Object.user_id == user_id) \
			.all()

		local_controllers_macs = [local_controller.mac for local_controller in local_controllers]
		remote_controllers_macs = [remote_controller.mac for remote_controller in remote_controllers]

		for local_controller in local_controllers:
			if local_controller.mac not in remote_controllers_macs:
				for local_sensor in local_controller.sensors:
					self.session.delete(local_sensor)
				self.session.delete(local_controller)

		for remote_controller in remote_controllers:
			if remote_controller.mac in local_controllers_macs:
				local_controller = next(filter(lambda x: x.mac == remote_controller.mac, local_controllers))
				if local_controller.name != remote_controller.name:
					local_controller.name = remote_controller.name
			else:
				object_id, = self.session.query(Object.id).filter_by(user_id=user_id).first()
				local_controller = Controller(
					name=remote_controller.name,
					mac=remote_controller.mac,
					object_id=object_id,
				)
				self.session.add(local_controller)

			remote_sensors = store.get_sensors(remote_controller)
			local_sensors = self.session.query(Sensor).filter_by(controller_id=local_controller.id)

			local_sensor_ids = [local_sensor.id for local_sensor in local_sensors]
			remote_sensor_ids = [remote_sensor.id for remote_sensor in remote_sensors]

			for local_sensor in local_sensors:
				if local_sensor.id not in remote_sensor_ids:
					self.session.delete(local_sensor)

			for remote_sensor in remote_sensors:
				if remote_sensor.id in local_sensor_ids:
					local_sensor = next(filter(lambda x: x.id == remote_sensor.id, local_sensors))
					if local_sensor.name != remote_sensor.name:
						local_sensor.name = remote_sensor.name
				else:
					local_sensor = Sensor(
						name=remote_sensor.name,
						id=remote_sensor.id,
						controller_id=local_controller.id,
					)
					self.session.add(local_sensor)


"""
Data-loading classes.
"""
import h5py
import torch
import torch.utils.data as data
import threading as thread
import queue
import numpy as np
from random import randint, shuffle

# ===== ===== Queue-based Multiprocessing HDF5 Loader class ===== =====
from utils.worker_fns import ecoli_worker_fn, bucket_worker_fn
class QueueLoader(object):
    """
    The QueueLoader reads from an HDF5 file and uses multiple cpu-based worker processes
    to continuously push new (signal,sequence) batches onto the queue; the workers perform
    (sampling => one-hot encoding => padding) prior to pushing them onto the queue.

    A Queue object is exposed, but the individual worker threads are not exposed.

    Upon dequeuing, a tuple of batched sequences is either returned as-is (for CPU processing)
    or is dispatched to CUDA via `*.cuda()` during the dequeue operation; the tuple is
    of the form ().
    """
    def __init__(self, dataset_path, num_signal_levels=256, num_workers=1, queue_size=50, batch_size=8, sample_lengths=(90,110),
                 max_iters=100, epoch_size=1000, worker_fn='ecoli'):
        """
        Construct a QueueLoader.
        """
        # save settings:
        self.dataset_path = dataset_path
        self.num_signal_levels = num_signal_levels
        self.num_workers = num_workers
        self.queue_size = queue_size
        self.batch_size = batch_size
        self.sample_lengths = sample_lengths
        self.max_iters = max_iters
        self.epoch_size = epoch_size
        self.num_epochs = 0
        self._cuda = False

        # open HDF5 file:
        try:
            self.dataset = h5py.File(dataset_path, 'r')
        except:
            raise Exception("Could not open HDF5 file: {}".format(dataset_path))

        # list of top-level read names into train/validation set:
        self.reads = list(self.dataset.keys())
        valid_set_size = int(0.3 * len(self.reads)) # 30% of reads go to validation
        self.train_reads = self.reads[valid_set_size:]
        self.valid_reads = self.reads[0:valid_set_size]

        # construct queue, re-entrant lock, and data counter:
        self.train_queue = queue.Queue(queue_size)
        self.valid_queue = queue.Queue(queue_size)
        self.queue_lock = thread.Lock()
        self.global_counter = 0
        self.stop_event = thread.Event()

        # create worker function as a closure:
        assert (worker_fn in ['ecoli', 'buckets'])
        if worker_fn == 'buckets': raise Exception("ERR: bucketed dataset is currently unsupported.")
        def queue_worker(q, reads, stop_evt):
            while True:
                if stop_evt.is_set(): break
                if (self.global_counter > self.max_iters): break
                datavals = ecoli_worker_fn(self.dataset, reads, batch_size=batch_size, sample_lengths=sample_lengths)
                with self.queue_lock:
                    q.put( datavals )
                    self.incr_global_ctr()
        self.queue_worker = queue_worker

        # create workers:
        self.workers = []
        for k in range(num_workers):
            _train_worker = thread.Thread(target=self.queue_worker,
                                          args=(self.train_queue, self.train_reads, self.stop_event),
                                          daemon=True)
            _train_worker.start()
            self.workers.append(_train_worker)
        # only need one validation worker since we dequeue from it so rarely:
        _valid_worker = thread.Thread(target=self.queue_worker,
                                      args=(self.valid_queue, self.valid_reads, self.stop_event),
                                      daemon=True)
        _valid_worker.start()
        self.workers.append(_valid_worker)

    def incr_global_ctr(self):
        self.global_counter += 1
        if (self.global_counter % self.epoch_size == 0): self.num_epochs += 1

    def dequeue(self, from_queue="train"):
        """
        Dequeue a new data sequence tuple from the queue as a list of Variables.
        """
        assert (from_queue in ['train', 'valid'])
        if from_queue == "train":
            try:
                vals = self.train_queue.get(timeout=1)
            except:
                raise StopIteration
        if from_queue == "valid":
            try:
                vals = self.valid_queue.get(timeout=1)
            except:
                raise StopIteration
        if self._cuda:
            return [torch.autograd.Variable(val.cuda()) for val in vals]
        else:
            return [torch.autograd.Variable(val) for val in vals]

    def close(self):
        """
        Gracefully join all threads and close HDF5 file handle.
        """
        # join the processes, with timeout of 2 seconds:
        self.stop_event.set()
        [worker.join(timeout=2) for worker in self.workers]

        # finally close HDF5 dataset file:
        if self.dataset: self.dataset.close()

    def cuda(self):
        self._cuda = True

    def cpu(self):
        self._cuda = False

# ===== ===== Basic HDF5 Loader class ===== =====
class Loader(object):
    """
    Signal and base-sequence loader. Takes a list of datasets in the form of an HDF5 and swaps between them.

    At each call of fetch(), this loader does the following:
    1) randomly chooses a bucket;
    2) randomly chooses a pair of signals and sequences in that bucket and one-hots the signal;
    3) returns them.
    """
    def __init__(self, dataset_path, num_signal_levels=256, max_iters=100, num_epochs=1):
        """Open handle to HDF5 dataset file."""
        self.dataset_path = dataset_path
        self._dataset = h5py.File(dataset_path, 'r')
        self._closed = False
        self.num_epochs = num_epochs
        self.max_iters = max_iters
        self._load_metadata()
        self.counter = 0
        self.epochs = 0
        self.num_signal_levels = num_signal_levels
        self._cuda = False

    def cuda(self):
        self._cuda = True

    def close(self):
        """Close the dataset."""
        self._dataset.close()
        self._closed = True

    def is_closed(self):
        """True if closed, False if opened."""
        return self._closed

    def fetch(self, bucket=None, entry=None):
        """
        Fetch a random signal/sequence pair from the dataset.

        Performs the following steps:
        1) choose a random bucket if bucket == None (else if int, choose that bucket);
        2) choose a random (signal,sequence) pair if entry == None (else if int, choose that entry);
        3) convert to one-hot encoding;
        4) convert to torch tensor format:
             signal ~ torch.FloatTensor variable of shape (1, num_levels, len(signal))
             sequence ~ torch.LongTensor variable of shape (1, len(sequence))
        5) if self.cuda == True, then convert to CUDA variables; return pair of variables.
        """
        ### check if we've hit the maximum number of timesteps; if yes, raise StopIteration
        self._maybe_stop()

        ### choose a random bucket and a random entry:
        bucket_id = randint(0, self._num_buckets-1) if (bucket == None) else bucket
        data_id = randint(0, self._buckets_data[bucket_id]['dataset_size']-1) if (entry == None) else entry

        ### fetch signal, sequence:
        signal = self.one_hot_signal(
            torch.from_numpy(self._dataset['bucket_{}'.format(bucket_id)]['signals']['{}'.format(data_id)][:]).long(),
            num_levels=self.num_signal_levels)
        seq = torch.from_numpy(
            self._dataset['bucket_{}'.format(bucket_id)]['reads']['{}'.format(data_id)][:]).long().unsqueeze(0)

        ### update timestep
        self._tick()

        ### return CUDA or CPU:
        if not self._cuda: return (torch.autograd.Variable(signal), torch.autograd.Variable(seq))
        return (torch.autograd.Variable(signal.cuda()), torch.autograd.Variable(seq.cuda()))

    ### Static Helper Functions:
    @staticmethod
    def one_hot_signal(signal, num_levels):
        """
        One-hot-encode a signal.

        Args:
        * signal: a list of ndarrays; these are the signals themselves.

        Returns:
        A one-hot-encoded signal.
        """
        return torch.zeros(1, num_levels, signal.size(0)).scatter_(1, signal.unsqueeze(0).unsqueeze(0), 1.)

    ### Helper Methods:
    def _load_metadata(self):
        """Load metadata values as attributes."""
        self._num_buckets = self._dataset['meta'].attrs['num_buckets']
        self._bucket_size = self._dataset['meta'].attrs['bucket_size']
        self._signals_path = self._dataset['meta'].attrs['signals_path']
        self._reads_path = self._dataset['meta'].attrs['reads_path']
        self._lengths_tsv_path = self._dataset['meta'].attrs['lengths_tsv_path']
        self._buckets_data = {}
        for k in range(self._num_buckets):
            self._buckets_data[k] = {
                'dataset_size': self._dataset['bucket_{}'.format(k)].attrs['dataset_size'],
                'max_read_length': self._dataset['bucket_{}'.format(k)].attrs['max_read_length'],
                'min_read_length': self._dataset['bucket_{}'.format(k)].attrs['min_read_length'],
                'max_signal_length': self._dataset['bucket_{}'.format(k)].attrs['max_signal_length'],
                'min_signal_length': self._dataset['bucket_{}'.format(k)].attrs['min_signal_length'],
                # n.b.: you have to index these to load them into memory; this is a space-saving
                # technique by design
                'read_lengths': self._dataset['bucket_{}'.format(k)]['read_lengths'],
                'signal_lengths': self._dataset['bucket_{}'.format(k)]['signal_lengths'],
            }
        self._num_sequences = sum([self._buckets_data[k]['dataset_size'] for k in range(self._num_buckets)])

    def _tick(self):
        """Update counter and maybe update epoch"""
        self.counter += 1
        if (self.counter != 0 and self.counter % self._num_sequences == 0): self.epochs += 1

    def _maybe_stop(self):
        """Return True if we are done; return False otherwise"""
        if self.epochs == self.num_epochs or self.counter == self.max_iters:
            raise StopIteration


from utils.utils import *
from pipeline.training.train_alexnet_mnist_debug import *
import pipeline.training.train_smallnet_mnist as mn

def train_alexnet_mnist_0001(config_data, tab_level=0, verbose=0):
	print('train_alexnet_mnist_0001().')
	from pipeline.data.load_data import load_mnist_0001
	from utils.loss import compute_loss

	data_loader = load_mnist_0001(config_data, verbose=0)
	state_tracker = setup_state_tracker(config_data, verbose=verbose, tab_level=tab_level)
	net = new_or_load_model(state_tracker,config_data, verbose=verbose, tab_level=tab_level)
	criterion, optimizer = mn.setup_training_tools_0001(net, config_data,
		verbose=verbose, tab_level=tab_level+1)

	pm.printv('Start training...'%(), tab_level=tab_level)
	total_iter_in_this_run = 0
	l_epoch = 1 + state_tracker.get_latest_saved_epoch()

	for n_epoch in range(l_epoch, l_epoch + config_data['general']['epoch']):
		state_tracker.setup_for_this_epoch(n_epoch)
		for i, data in enumerate(data_loader,0):
			optimizer.zero_grad()

			x, y0 = data
			x = x.to(this_device)

			if DEBUG_train_loop_0001(DEBUG_train_alexnet_mnist_LOOP_SIGNAL, net, x, y0,
				tab_level=tab_level, verbose=verbose): return

			y = net(x)

			loss = compute_loss(criterion, y.squeeze(3).squeeze(2).cpu(), y0)
			loss.backward()
			optimizer.step()

			# FOR LOGGING
			total_iter_in_this_run += 1
			state_tracker.store_loss_by_epoch(loss.item(), n_epoch)

			stop_iter, stop_epoch = DEBUG_train_loop_0002(DEBUG_train_alexnet_mnist_LOOP_SIGNAL2,
				i, n_epoch - l_epoch , tab_level=tab_level+1, verbose=verbose)
			if stop_iter: break
		state_tracker.update_epoch()
		if stop_epoch: break
	state_tracker.update_state(total_iter_in_this_run, config_data)
	mn.save_model_by_n_th_run(net, state_tracker,tab_level=tab_level, verbose=verbose)
	state_tracker.display_end_state(tab_level=tab_level+1, verbose=verbose)

def setup_state_tracker(config_data, for_training=True, verbose=0, tab_level=0):
	from utils.state_tracker import StateTracker
	ckpt_path = os.path.join(config_data['working_dir'],'checkpoint')
	tracker_name = 'alexnet_mnist_0001_'
	training_series_name = tracker_name + str(config_data['training']['series_name'])
	state_tracker = StateTracker(
		mode='load_latest_if_available',
		ckpt_path=ckpt_path,
		training_series_name=training_series_name,
		load_this_n_th_run=None,
		verbose=verbose,
		tab_level=tab_level+1)
	state_tracker = state_tracker.load_latest_if_available(state_tracker.latest_saved_run,for_training=for_training)
	state_tracker.display_simple_state(tab_level=tab_level+1, verbose=verbose)
	return state_tracker

def new_or_load_model(state_tracker, config_data, verbose=0, tab_level=0):
	pm.printvm("new_or_load_model(). latest_saved_run:%s"%(str(state_tracker.latest_saved_run)),
		 tab_level=tab_level,verbose=verbose, verbose_threshold=50)
	from models.networks import AlexNetLike

	if config_data['data_from_torch']['mnist']['resize'] is None:
		net = AlexNetLike(verbose=verbose, tab_level=tab_level)
	else:
		IMG_SIZE = config_data['data_from_torch']['mnist']['resize']
		net = AlexNetLike(set_default_params=False, relprop_mode=config_data['lrp']['relprop_mode'], verbose=verbose, tab_level=tab_level)
		if IMG_SIZE == (140,140):
			from models.networks_Alex import AlexNet_custom_setting0001
			net = AlexNet_custom_setting0001(net)
		else:
			raise Exception('Please manually set params')

	from pipeline.training.train_smallnet_mnist import new_or_load_model_inner
	net = new_or_load_model_inner(state_tracker, net, verbose=verbose, tab_level=tab_level+1)
	return net


from flask import render_template, flash, Blueprint, redirect,url_for
from app.models.hotel import Hotel
from flask_login import current_user
from app.manage_property.forms import ManageForm

manage_blueprint = Blueprint("manage_property",
                             __name__,
                             template_folder="templates/manage_property")

@manage_blueprint.route("/manage", methods=["GET", "POST"])
def manage_property():
    form = ManageForm()

    if form.validate_on_submit():
        hotel = Hotel.query.filter_by(user_id=current_user.id).first()
        hotel_name = form.hotel_name.data
        address = form.address.data
        room_quantity = form.room_quantity.data
        stars = form.stars.data
        if hotel_name:
            hotel.hotel_name = hotel_name
            Hotel.update()
        if address:
            hotel.address = address
            Hotel.update()
        if room_quantity:
            hotel.room_quantity = room_quantity
            Hotel.update()
        if stars:
            hotel.stars = stars
            Hotel.update()
        # old_info = [hotel.hotel_name, hotel.address, hotel.room_quantity, hotel.stars]
        # new_info = [hotel_name, address, room_quantity, stars]
        # for item in new_info:
        #     if item:
        #         old_info[new_info.index(item)] = item
        #         Hotel.update()
        #         print(old_info[new_info.index(item)])

        flash("Item(s) updated successfully")
        return redirect(url_for("manage_property.manage_property"))

    return render_template("manage_property.html", form=form)


from duty.objects import dp, SignalEvent, __version__
from duty.utils import ment_user, format_response
import time

@dp.signal_event_register('инфо', 'инфа', 'info')
def sinfo(event: SignalEvent) -> str:
    if event.msg['from_id'] not in event.db.trusted_users:
        message_id = event.send(event.responses['not_in_trusted'])
        time.sleep(3)
        event.api.msg_op(3, msg_id=message_id)
        return "ok"

    owner = event.api('users.get', user_ids=event.db.owner_id)[0]

    event.send(format_response(
        event.responses['info_duty'],
        версия=__version__,
        владелец=ment_user(owner),
        чаты=len(event.db.chats.keys()),
        ид=event.chat.iris_id,
        имя=event.chat.name
    ))
    return "ok"


"""added phone number

Revision ID: 6114f633fefd
Revises: 03c901835d1c
Create Date: 2022-07-11 15:26:00.045673

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '6114f633fefd'
down_revision = '03c901835d1c'
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    # op.alter_column('posts', 'id',
    #            existing_type=sa.INTEGER(),
    #            nullable=True,
    #            autoincrement=True)
    op.add_column('users', sa.Column(
        'phone_number', sa.String(), nullable=True))
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('users', 'phone_number')
    # op.alter_column('posts', 'id',
    #            existing_type=sa.INTEGER(),
    #            nullable=False,
    #            autoincrement=True)
    # ### end Alembic commands ###


import torch
import numpy as np
from PIL import Image
import os, random, math
import sys

sys.path.append(os.path.abspath(os.path.join(__file__, "..", "..")))
from black_box_tool import get_84_transform

sys.path.append(os.path.abspath(os.path.join(__file__, "..", "..")))

def save_image_batch(imgs, output, col=None, size=None, pack=True):
    if isinstance(imgs, torch.Tensor):
        imgs = (imgs.detach().clamp(0, 1).cpu().numpy()*255).astype('uint8')
    base_dir = os.path.dirname(output)
    if base_dir!='':
        os.makedirs(base_dir, exist_ok=True)
    if pack:
        imgs = pack_images( imgs, col=col ).transpose( 1, 2, 0 ).squeeze()
        imgs = Image.fromarray( imgs )
        if size is not None:
            if isinstance(size, (list,tuple)):
                imgs = imgs.resize(size)
            else:
                w, h = imgs.size
                max_side = max( h, w )
                scale = float(size) / float(max_side)
                _w, _h = int(w*scale), int(h*scale)
                imgs = imgs.resize([_w, _h])
        imgs.save(output)
    else:
        #output_filename = output.strip('.png')
        output_filename = output[:-4]
        for idx, img in enumerate(imgs):
            img = Image.fromarray( img.transpose(1, 2, 0).squeeze() )
            img.save(output_filename+'-%d.png'%(idx))

def pack_images(images, col=None, channel_last=False, padding=1):
    # N, C, H, W
    if isinstance(images, (list, tuple) ):
        images = np.stack(images, 0)
    if channel_last:
        images = images.transpose(0,3,1,2) # make it channel first
    assert len(images.shape)==4
    assert isinstance(images, np.ndarray)

    N,C,H,W = images.shape
    if col is None:
        col = int(math.ceil(math.sqrt(N)))
    row = int(math.ceil(N / col))

    pack = np.zeros( (C, H*row+padding*(row-1), W*col+padding*(col-1)), dtype=images.dtype )
    for idx, img in enumerate(images):
        h = (idx // col) * (H+padding)
        w = (idx % col) * (W+padding)
        pack[:, h:h+H, w:w+W] = img
    return pack

def normalize(tensor, mean, std, reverse=False):
    if reverse:
        _mean = [ -m / s for m, s in zip(mean, std) ]
        _std = [ 1/s for s in std ]
    else:
        _mean = mean
        _std = std

    _mean = torch.as_tensor(_mean, dtype=tensor.dtype, device=tensor.device)
    _std = torch.as_tensor(_std, dtype=tensor.dtype, device=tensor.device)
    tensor = (tensor - _mean[None, :, None, None]) / (_std[None, :, None, None])
    return tensor

class Normalizer(object):
    def __init__(self, mean, std):
        self.mean = mean
        self.std = std

    def __call__(self, x, reverse=False):
        return normalize(x, self.mean, self.std, reverse=reverse)

class ImagePool2(object):
    def __init__(self,args, root,num_classes,transform,max_batch_per_class):
        self.args=args
        self.root_support=os.path.join(root,'support')
        self.root_query = os.path.join(root, 'query')
        #support pool
        for c_abs in range(num_classes):
            os.makedirs(os.path.join(self.root_support,str(c_abs)), exist_ok=True)
            os.makedirs(os.path.join(self.root_query, str(c_abs)), exist_ok=True)
        self._idx = [0]*num_classes
        self.max_batch_per_class=max_batch_per_class
        self.ready_class=[]
        self.transform=transform

    def add(self, imgs, c_abs_list=None,synthesis_batch_size_per_class=None,mode=None):
        c_abs_targets=torch.LongTensor(c_abs_list*synthesis_batch_size_per_class)
        for c_abs in c_abs_list:
            if mode=='support':
                root=os.path.join(self.root_support,str(c_abs))
            if mode=='query':
                root=os.path.join(self.root_query,str(c_abs))
            imgs_c=imgs[c_abs_targets==c_abs]
            save_image_batch(imgs_c, os.path.join( root, "%d.png"%(self._idx[c_abs]) ), pack=False)
            self._idx[c_abs]+=1
            self._idx[c_abs]=self._idx[c_abs]%self.max_batch_per_class
            if c_abs not in self.ready_class:
                self.ready_class.append(c_abs)

    def get_random_task(self,num_w,num_s,num_q):
        select_way=random.sample(self.ready_class,num_w)
        support_x=[]
        query_x=[]
        support_y_abs=[]
        query_y_abs = []
        for c_relative,c_abs in enumerate(select_way):
            c_abs_path = os.path.join(self.root_support, str(c_abs))
            image_name_list = os.listdir(c_abs_path)
            image_name_list = random.sample(image_name_list, (num_s+num_q))
            if self.args.dataset!='mix':
                if self.args.dataset == 'omniglot':
                    select_image = [self.transform(Image.open(os.path.join(c_abs_path, image_name)).convert('L')) for image_name in image_name_list]
                else:
                    select_image = [self.transform(Image.open(os.path.join(c_abs_path, image_name)).convert('RGB')) for image_name in image_name_list]
            elif self.args.dataset=='mix':
                if c_abs<=63:
                    transform=get_84_transform(self.args, dataset='cifar100')
                else:
                    transform=get_84_transform(self.args, dataset='cub')
                select_image = [transform(Image.open(os.path.join(c_abs_path, image_name)).convert('RGB')) for
                                image_name in image_name_list]

            select_image = torch.stack(select_image, dim=0)
            support_x.append(select_image[:num_s])
            query_x.append(select_image[num_s:(num_s + num_q)])
            support_y_abs.append(torch.LongTensor([c_abs] * num_s))
            query_y_abs.append(torch.LongTensor([c_abs] * num_q))
        return torch.cat(support_x,dim=0),torch.cat(support_y_abs,dim=0),torch.cat(query_x,dim=0),torch.cat(query_y_abs,dim=0),select_way


import os
import requests
from dotenv import load_dotenv
from flask import Blueprint, request, jsonify

api = Blueprint('api', __name__, url_prefix='/api')

# .env ファイルから API キーを取得
load_dotenv()
X_API_KEY = os.getenv('API_KEY')

base_url = 'https://publicapi.taittsuu.com/publicapi/v0.1'
headers = {'content-type': 'application/json', 'X-API-KEY': X_API_KEY}
public_timelines_url = '/publictimelines'
users_url = '/users'
taiitsus_url = '/taiitsus'
liking_users_url = '/liking_users'
following_url = '/following'
followers_url = '/followers'
search_uel = '/search'

# パブリックタイムラインの取得
@api.route(public_timelines_url, methods=['GET'])
def public_timelines():
    req = request.args
    next_num = req.get('next')
    next_param = f'?next={next_num}' if next_num else ''
    res = requests.get(
            f'{base_url}{public_timelines_url}{next_param}',
            headers=headers
        )
    return res.json()

# ユーザーのタイーツ取得
@api.route(f'{users_url}/<user_id>{taiitsus_url}', methods=['GET'])
def user_taiitsus(user_id):
    req = request.args
    next_num = req.get('next')
    next_param = f'?next={next_num}' if next_num else ''
    res = requests.get(
            f'{base_url}{users_url}/{user_id}{taiitsus_url}{next_param}',
            headers=headers
        )
    return res.json()

# タイーツ情報取得
@api.route(f'{taiitsus_url}/<taiitsu_id>', methods=['GET'])
def taiitsu(taiitsu_id):
    res = requests.get(
            f'{base_url}{taiitsus_url}/{taiitsu_id}',
            headers=headers
        )
    return res.json()

# タイーツのいいね一覧取得
@api.route(f'{taiitsus_url}/<taiitsu_id>{liking_users_url}', methods=['GET'])
def liking_users(taiitsu_id):
    req = request.args
    next_num = req.get('next')
    next_param = f'?next={next_num}' if next_num else ''
    res = requests.get(
            f'{base_url}{taiitsus_url}/{taiitsu_id}{liking_users_url}{next_param}',
            headers=headers
        )
    return res.json()

# ユーザー情報取得
@api.route(f'{users_url}/<user_id>', methods=['GET'])
def user(user_id):
    res = requests.get(
            f'{base_url}{users_url}/{user_id}',
            headers=headers
        )
    return res.json()

# ユーザーのフォロー一覧取得
@api.route(f'{users_url}/<user_id>{following_url}', methods=['GET'])
def user_following(user_id):
    req = request.args
    next_num = req.get('next')
    next_param = f'?next={next_num}' if next_num else ''
    res = requests.get(
            f'{base_url}{users_url}/{user_id}{following_url}{next_param}',
            headers=headers
        )
    return res.json()

# ユーザーのフォロワー一覧取得
@api.route(f'{users_url}/<user_id>{followers_url}', methods=['GET'])
def user_followers(user_id):
    req = request.args
    next_num = req.get('next')
    next_param = f'?next={next_num}' if next_num else ''
    res = requests.get(
            f'{base_url}{users_url}/{user_id}{followers_url}{next_param}',
            headers=headers
        )
    return res.json()

# ユーザー検索
@api.route(f'{search_uel}{users_url}', methods=['GET'])
def search_users():
    req = request.args
    query_word = req.get('query')
    if not query_word:
        return jsonify({'error': {
            'code': 400,
            'message': 'クエリパラメータ"query"は必須です。'
        }})
    next_num = req.get('next')
    params = f'?query={query_word}&next={next_num}' if next_num else f'?query={query_word}'
    res = requests.get(
            f'{base_url}{search_uel}{users_url}{params}',
            headers=headers
        )
    return res.json()

# エラーのハンドリング
@api.errorhandler(400)
@api.errorhandler(404)
def error_handler(error):
    return jsonify({'error': {
        'code': error.description['code'],
        'message': error.description['message']
    }}), error.code


import click
import pandas as pd
from tqdm import tqdm

from ..apply import predict_separator
from ..io import (
    append_column_to_hdf5,
    append_predictions_cta,
    read_telescope_data_chunked,
    drop_prediction_column,
    drop_prediction_groups,
    load_model,
)
from ..configuration import AICTConfig
from ..logging import setup_logging

@click.command()
@click.argument("configuration_path", type=click.Path(exists=True, dir_okay=False))
@click.argument("data_path", type=click.Path(exists=True, dir_okay=False))
@click.argument("model_path", type=click.Path(exists=True, dir_okay=False))
@click.option("-v", "--verbose", help="Verbose log output", is_flag=True)
@click.option(
    "-N",
    "--chunksize",
    type=int,
    help="If given, only process the given number of events at once",
)
@click.option("-y", "--yes", help="Do not prompt for overwrites", is_flag=True)
def main(configuration_path, data_path, model_path, chunksize, yes, verbose):
    """
    Apply loaded model to data.

    CONFIGURATION_PATH: Path to the config yaml file.
    DATA_PATH: path to the FACT/CTA data.
    MODEL_PATH: Path to the pickled model.

    The program adds the following columns to the inputfile:
        `output_name`: the output of model.predict_proba for the
        class name given in the config file.

    If `output_name` is not given in the config file,
    the default value of "gamma_prediction" will be used.
    """
    log = setup_logging(verbose=verbose)

    config = AICTConfig.from_yaml(configuration_path)
    model_config = config.separator

    prediction_column_name = model_config.output_name

    if config.data_format == "CTA":
        drop_prediction_groups(data_path, group_name=model_config.output_name, yes=yes)
    elif config.data_format == "simple":
        drop_prediction_column(
            data_path,
            group_name=config.events_key,
            column_name=prediction_column_name,
            yes=yes,
        )

    log.debug("Loading model")
    model = load_model(model_path)
    log.debug("Loaded model")

    df_generator = read_telescope_data_chunked(
        data_path,
        config,
        chunksize,
        model_config.columns_to_read_apply,
        feature_generation_config=model_config.feature_generation,
    )

    if config.data_format == "CTA":
        chunked_frames = []

    for df_data, start, stop in tqdm(df_generator):
        prediction = predict_separator(df_data[model_config.features], model)
        if config.data_format == "CTA":
            df_data.reset_index(inplace=True)
            for tel_id, group in df_data.groupby("tel_id"):
                d = group[["obs_id", "event_id"]].copy()
                d[prediction_column_name] = prediction[group.index]
                chunked_frames.append(d)
                append_predictions_cta(
                    data_path,
                    d,
                    f"/dl2/event/telescope/{model_config.output_name}/tel_{tel_id:03d}",
                )
        elif config.data_format == "simple":
            append_column_to_hdf5(
                data_path, prediction, config.events_key, model_config.output_name
            )

    # combine predictions
    if config.data_format == "CTA":
        array_table = f"/dl2/event/subarray/{model_config.output_name}"
        d = (
            pd.concat(chunked_frames)
            .groupby(["obs_id", "event_id"], sort=False)
            .agg(["mean", "std"])
        )
        d.columns = d.columns.droplevel(0)
        d.reset_index().to_hdf(data_path, array_table, mode="a", format="table")

if __name__ == "__main__":
    # pylint: disable=no-value-for-parameter
    main()


import unittest

from em.bot import get_bot
from em.mmapi import send_message

class TestMattermostAPI(unittest.TestCase):
    bot = get_bot()

    def test_send_message(self):
        user_id = 'jxs5xm3b37djjfqohbtp4urpho'
        message = "I've matched token123@nix.co.il with your account.\n" \
                  "Next, click the button at the bottom of this message to get a verification email from cloudflare.\n" \
                  "You will only be sent a single email per address. You must complete verification to receive emails."
        actions = [
            {
                'id': 'hooks',
                'name': 'aaa',
                'integration': {
                    'url': 'http://email-manager.web.svc/hooks/send_email',
                    'context': {
                        'token': 'token123',
                    }
                }
            }
        ]
        result = send_message(self.bot.driver, user_id, message, actions)

        self.assertEqual(True, False)  # add assertion here

if __name__ == '__main__':
    unittest.main()