import pandas as pd
import matplotlib.pyplot as plt

# Load the CSV data into a DataFrame
df = pd.read_csv("suicide_reates.csv")

# Menu for user's choice
print("Select a graph to display:")
print("1. Bar Graph")
print("2. Histogram")
print("3. Scatter Plot")
print("4. Line Graph")  # Add Line Graph option

choice = input("Enter your choice (1/2/3/4): ")

# Convert the choice to an integer
choice = int(choice)

# Check the user's choice and create the corresponding plot
if choice == 1:
    # Bar Graph
    plt.figure(figsize=(10, 6))
    plt.bar(df["YEAR"], df["ESTIMATE"])
    plt.title("Bar Graph: Suicide Death Rates Over Time")
    plt.xlabel("Year")
    plt.ylabel("Death Rate (per 100,000 population)")
    plt.grid(axis="y")
    plt.show()
elif choice == 2:
    # Histogram
    plt.figure(figsize=(10, 6))
    plt.hist(df["ESTIMATE"], bins=5, edgecolor="k")
    plt.title("Histogram: Suicide Death Rates")
    plt.xlabel("Death Rate (per 100,000 population)")
    plt.ylabel("Frequency")
    plt.grid(axis="y")
    plt.show()
elif choice == 3:
    # Scatter Plot
    plt.figure(figsize=(10, 6))
    plt.scatter(df["YEAR"], df["ESTIMATE"], s=50, alpha=0.7)
    plt.title("Scatter Plot: Suicide Death Rates Over Time")
    plt.xlabel("Year")
    plt.ylabel("Death Rate (per 100,000 population)")
    plt.grid(True)
    plt.show()
elif choice == 4:
    # Line Graph
    plt.figure(figsize=(10, 6))
    plt.plot(df["YEAR"], df["ESTIMATE"], marker='o', linestyle='-')
    plt.title("Line Graph: Suicide Death Rates Over Time")
    plt.xlabel("Year")
    plt.ylabel("Death Rate (per 100,000 population)")
    plt.grid(True)
    plt.show()
else:
    print("Invalid choice. Please select 1, 2, 3, or 4.")


import torch
import pandas as pd
import numpy as np
from tqdm.notebook import tqdm
from joblib import Parallel, delayed #conda install -c anaconda joblib

class DatasetHandler:

    def __init__(self, dataframes, config_dict):
        """
        cols_input, cols_target, dataframes_descriptions,
        keep_na=False, include_all_timestamps_between=None, all_timestamps_intervals=None,
        cols_channels_input=None, cols_channels_target=None, as_float32=True):

        dataframes - list of dataframes, supposed to work per year. Each dataframe contains np.arrays with dtype = object
            The shapes of np.arrays are assumed to be [c,h,w] for meteorology dats or [1] for dust, and assuming
            target or input can be only dust or only meteorology
        dataframes_description - a must have description of the dataset (please describe the channels).
            e.g.: {"input":{0:"Z500",1:"PV310"},"target":{0:"dust_0",1:"lags_0"}}
        cols_channels_input/target - dict with {cols:[idxs]}, e.g. {"PV":[0], "Z":None, "U":[1,3]}.
            If None - will use all channels (for dust dataframes keep it None)
        include_all_timestamps_between - list of [first,last] to include all dates between them, both are pandas DateTime.
            If include_all_timestamps_between==[], will init first and last from dataframes.
        all_timestamps_intervals - the intervals of all timestamps dataframe, default to "3h"
        as_float32 - if False, will set to float64 on numpy to torch conversion. Defaults to True

        """
        self.dataframes = dataframes
        self.cols_input = config_dict["cols_input"]
        self.cols_target = config_dict["cols_target"]
        self.cols_to_keep = self.cols_input+self.cols_target
        self.cols_channels_input = config_dict["cols_channels_input"]
        self.cols_channels_target = config_dict["cols_channels_target"]
        self.dataframes_descriptions = config_dict["dataframes_descriptions"]
        self.keep_na = config_dict["keep_na"]
        self.replace_na_in_target = config_dict["replace_na_in_target"]
#         self.mask_tensor = config_dict["mask_tensor"]
        self.as_float32 = config_dict["as_float32"]
        self.wanted_year = config_dict["wanted_year"] # can be set to None
        self.include_all_timestamps_between =  config_dict["include_all_timestamps_between"]
        self.all_timestamps_intervals = config_dict["all_timestamps_intervals"] or "3h"
#         self.order_of_cols = config_dict["order_of_cols"]
        if self.include_all_timestamps_between: self.init_all_timestamps()
        self.shapes = {"input": None, "target": None}
        self.col_channels_idxs = {"input": {}, "target": {}}
        self.init_cols_channels()
        self.combine_dataframes()
        self.init_shapes_and_idxs()

    def init_all_timestamps(self):
        first = self.dataframes[0].index[0]
        last = self.dataframes[0].index[-1]
        if len(self.dataframes)>1:
            for df in self.dataframes[1:]:
                if df.index[0] < first: first = df.index[0]
                if df.index[-1] > last: last = df.index[-1]
        self.include_all_timestamps_between = [first, last]
        all_dates =  pd.date_range(start=first, end=last, freq=self.all_timestamps_intervals, tz="UTC")
        all_dates_df = pd.DataFrame({},index=all_dates)
        if self.wanted_year is not None:
            all_dates_df = all_dates_df[all_dates_df.index.year==self.wanted_year]
        self.dataframes.append(all_dates_df)

    def init_cols_channels(self):
        cols_channels_input_new = {col: None for col in self.cols_input}
        cols_channels_target_new = {col: None for col in self.cols_target}
        cols_channels_new = [cols_channels_input_new, cols_channels_target_new]
        for i,cols_channels_list in enumerate([self.cols_channels_input,self.cols_channels_target]):
            if cols_channels_list is not None:
                all_cols = self.cols_input if cols_channels_list==self.cols_channels_input else self.cols_target
                for col in cols_channels_list:
                    if col not in all_cols:
                        print(f"Error! {col} from cols_channels is not in cols list: {all_cols}. Aborting...")
                        return None
                    cols_channels_new[i][col] = cols_channels_list[col]
        self.cols_channels_input,self.cols_channels_target = cols_channels_new

    def combine_dataframes(self, ):
        # If include_all_timestamps - assume first and last are not None (else - ignore them)
        print("Combining dataframes...")
        self.combined_dataframe = self.dataframes[0]
        if len(self.dataframes)>1:
            for df in self.dataframes[1:]:
                self.combined_dataframe = self.combined_dataframe.join(df, how="outer")
        self.combined_dataframe = self.combined_dataframe[self.cols_to_keep]
        if not self.keep_na:
            print(f"Removing NaN values: from length of {len(self.combined_dataframe)} ...")
            self.combined_dataframe = self.combined_dataframe.dropna(how="any")
            print(f"... keeping {len(self.combined_dataframe)}...")
        elif self.replace_na_in_target is not None:
            print(f"Replacing NaN values with {self.replace_na_in_target}...")
            for col in self.cols_target:
                for i in range(len(self.combined_dataframe[col])):
                    if np.isnan(self.combined_dataframe[col][i]):
                        d = self.combined_dataframe.index[i]
                        self.combined_dataframe.loc[d,col]=self.replace_na_in_target
        print("...Done! Fixing shapes of singular data cols...")
        good_sample_idx = self.get_good_combined_idx()
        for col in self.cols_to_keep:
            if len(self.combined_dataframe[col][good_sample_idx].shape)<=1:
                self.combined_dataframe[col] = self.combined_dataframe[col].astype("object")
                self.combined_dataframe[col] = [p for p in np.expand_dims(np.array(self.combined_dataframe[col]),1)]
        print("...Done! The resulting dataframe is kept in self.combined_dataframe:")
        self.print_statistics(self.combined_dataframe)

    def print_statistics(self, df):
        print("Number of values per column:")
        print(df.count())
        print(f"Dates: {df.index[:5]}\n...\n{df.index[-5:]}")

    def get_good_combined_idx(self):
        for i,date in enumerate(self.combined_dataframe.index):
            if(self.combined_dataframe[i:i+1].isna().values.any()):
                continue
            return i
        print("Did not find any row without NaN")
        return None

    def get_shapes_of_data(self, x):
        shapes = x.shape
        if len(shapes)<=1:
            return 1,0,0
        else:
            return shapes

    def init_shapes_and_idxs(self):
        """
            shapes are assumed to be [c,h,w] for meteorology data, or [1] for dust, and [h,w] are the same for
            for all inputs and targets (separately)
        """
        good_idx = self.get_good_combined_idx() # An idx with full row in self.combined_dataframe
        dataset_types = ["input","target"]
#         if self.cols_target == []:
#             dataset_types = ["input"]
#         if self.cols_input == []:
#             dataset_types = ["target"]
        for dataset_type in ["input","target"]:
            channels_counter = 0
            cols_list = self.cols_input if dataset_type=="input" else self.cols_target
            if cols_list==[]:
                self.shapes[dataset_type] = None
                continue
            sample_data = self.combined_dataframe[cols_list[0]][good_idx]
            _,h_all,w_all=self.get_shapes_of_data(sample_data)
            channels_from_cols = self.cols_channels_input if dataset_type=="input" else self.cols_channels_target
            for col in cols_list:
                x = self.combined_dataframe[col][good_idx]
                if channels_from_cols[col] is not None:
                    channels_to_keep = np.array(channels_from_cols[col])
                    x = x[channels_to_keep,:,:]
                c,h,w=self.get_shapes_of_data(x)
                if w!=w_all or h!=h_all:
                    print(f"Bad shapes of input parameters! {h,w} and in {col} not {h_all,w_all}. Aborting...")
                    return
                self.col_channels_idxs[dataset_type][col] = np.arange(channels_counter,channels_counter+c)
                channels_counter+=c
            self.shapes[dataset_type] = [channels_counter,h_all,w_all]

    def create_tensor_from_dataset_type(self, dataset_type):
        shape = self.shapes[dataset_type]
        if shape is None:
            return None
        N,C,H,W = len(self.combined_dataframe), shape[0], shape[1], shape[2]
        if H==0 and W==0:
            x = torch.zeros([N,C],dtype=torch.float32)
        else:
            x = torch.zeros([N,C,H,W],dtype=torch.float32)
        cols_list = self.cols_input if dataset_type=="input" else self.cols_target
        for col in cols_list:
            channels = self.col_channels_idxs[dataset_type][col]
            channels_from_df = np.arange(self.combined_dataframe[col][0].shape[0])
            channels_from_cols = self.cols_channels_input if dataset_type=="input" else self.cols_channels_target
            if channels_from_cols[col] is not None:
                channels_from_df = np.array(channels_from_cols[col])
            if H==0 and W==0:
                x[:,channels] = torch.tensor([c.astype("float32") for c in self.combined_dataframe[col]])
            else:
                x_tf = torch.tensor([c.astype("float32") for c in self.combined_dataframe[col]])
                x[:,channels,:,:] = x_tf[:,channels_from_df,:,:]
        return x

    def create_and_save_dataset(self, dir_path, base_filename):
        """
            Creates a dataset from self.combined_dataframe and saves it
            A directroy named "metadata" has to be created inside dir_path
        """
        x_input = self.create_tensor_from_dataset_type("input")
        x_target = self.create_tensor_from_dataset_type("target")
        timestamps = self.combined_dataframe.index
        self.save_dataset(dir_path, base_filename, x_input, x_target, timestamps)

    def save_dataset(self, dir_path, base_filename, x_input, x_target, timestamps):
        filename_input = dir_path+"/"+base_filename+"_input.pkl"
        filename_target = dir_path+"/"+base_filename+"_target.pkl"
        filename_timestamps = dir_path+"/"+base_filename+"_timestamps.pkl"
        filename_description = dir_path+"/metadata/"+base_filename+"_descriptions.pkl"
        if x_input is not None: torch.save(x_input, filename_input)
        if x_target is not None: torch.save(x_target, filename_target)
        if timestamps is not None: torch.save(timestamps, filename_timestamps)
        if self.dataframes_descriptions is not None: torch.save(self.dataframes_descriptions, filename_description)

    @staticmethod
    def create_and_save_one_dataset_from_path(dataframes_paths, dataset_arguments, save_as):
        dataframes = [torch.load(path) for path in dataframes_paths]
        try:
            handler = DatasetHandler(dataframes, dataset_arguments)
            handler.create_and_save_dataset(save_as["dir_path"], save_as["base_filename"])
        except KeyError:
            print("...An error occured (most likely all are NaN's), skipping dataset......")

    @staticmethod
    def create_and_save_datasets_from_paths(dataframes_paths, datasets_arguments, save_as_list, njobs=3):
        """
            datasets_arguments is a list of dictionaries required to construct each dataset, while the dataframes are given
            as a list of paths to be loaded when constructing and saving the dataset
            dataframes_paths: the paths to dataframes to be loaded
            datasets_arguments:
            {
                cols_input
                cols_target
                dataframes_descriptions
                keep_na
                include_all_timestamps_between
                all_timestamps_intervals
                cols_channels_input
                cols_channels_target
                as_float32
                wanted_year
            }
            save_as_list: [{"dir_path":..., "base_filename":...},]
        """
        num_datasets_to_save = len(dataframes_paths)
        Parallel(n_jobs=njobs,verbose=100)(delayed(
            DatasetHandler.create_and_save_one_dataset_from_path)(dataframes_paths[i], datasets_arguments[i], save_as_list[i])
                for i in range(num_datasets_to_save)
        )


from django.urls import path

from . import views

urlpatterns = [
    path("", views.index, name="index"),
    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("register", views.register, name="register"),
    path("allPosts",views.allPosts,name="allPosts"),
    path("profilePage\<str:userid>",views.profilePage,name="profilePage"),
    path("follow\<str:userid>",views.follow,name="follow"),
    path("unfollow\<str:userid>",views.unfollow,name="unfollow"),
    path("following",views.following,name="following"),

    # API Routes
    path("posts/<int:number>",views.savePost,name="savePost"),
    path("like",views.like,name="like")
]


import math
import pytest
from datatable import f, dt, update
from tests import assert_equals

stypes_int = dt.ltype.int.stypes
stypes_float = dt.ltype.real.stypes
stypes_str = dt.ltype.str.stypes
stypes_all = [dt.bool8, dt.obj64] + stypes_int + stypes_float + stypes_str

#-------------------------------------------------------------------------------
# Assign None
#-------------------------------------------------------------------------------

def test_assign_none_single():
    DT = dt.Frame(A=range(5))
    DT[:, f.A] = None
    assert_equals(DT, dt.Frame(A=[None]*5, stype=dt.int32))

def test_assign_none_new():
    DT = dt.Frame(A=range(5))
    DT[:, "B"] = None
    assert_equals(DT, dt.Frame(A=range(5), B=[None]*5,
                               stypes={"A": dt.int32, "B": dt.bool8}))

def test_assign_none_all():
    DT = dt.Frame([[True, False], [1, 5], [999, -12], [34.2, math.inf],
                   [0.001, 1e-100], ['foo', 'bar'], ['buzz', '?']],
                  names=list("ABCDEFG"),
                  stypes=['bool', 'int8', 'int64', 'float32', 'float64',
                          'str32', 'str64'])
    DT[:, :] = None
    assert_equals(DT, dt.Frame([[None, None]] * 7,
                               names=tuple("ABCDEFG"),
                               stypes=(dt.bool8, dt.int8, dt.int64, dt.float32,
                                       dt.float64, dt.str32, dt.str64)))

#-------------------------------------------------------------------------------
# Assign boolean values
#-------------------------------------------------------------------------------

def test_assign_boolean():
    DT = dt.Frame(A=[None])
    DT[:, "A"] = True
    assert_equals(DT, dt.Frame(A=[True]))

def test_assign_boolean2():
    DT = dt.Frame(A=[True, False, True])
    DT[:, "A"] = False
    assert_equals(DT, dt.Frame(A=[False] * 3))

@pytest.mark.parametrize("stype", stypes_int + stypes_float + stypes_str)
def test_assign_boolean_to_different_type(stype):
    DT = dt.Frame(A=[5, 7], stype=stype)
    assert DT.stype == stype
    DT[:, "A"] = False
    assert_equals(DT, dt.Frame(A=[False, False], stype=bool))

def test_assign_boolean_partial():
    DT = dt.Frame(A=range(5))
    DT[2, "B"] = False
    assert_equals(DT, dt.Frame(A=range(5), B=[None, None, False, None, None]))

#-------------------------------------------------------------------------------
# Assign integer values
#-------------------------------------------------------------------------------

def test_assign_integer_out_of_range():
    DT = dt.Frame(A=[1, 2, 3], stype=dt.int8)
    assert DT.stype == dt.int8
    DT[:, "A"] = 5000000
    assert_equals(DT, dt.Frame(A=[5000000] * 3, stype=dt.int32))

def test_assign_integer_out_of_range_to_subset():
    DT = dt.Frame(A=range(10), stype=dt.int8)
    assert DT.stype == dt.int8
    DT[:3, "A"] = 999
    assert_equals(DT, dt.Frame(A=[999, 999, 999, 3, 4, 5, 6, 7, 8, 9],
                               stype=dt.int16))
    DT[-1, "A"] = 10**10
    assert_equals(DT, dt.Frame(A=[999, 999, 999, 3, 4, 5, 6, 7, 8, 10**10],
                               stype=dt.int64))

def test_assign_int_overflow():
    # When integer overflows, it becomes a float64 value
    DT = dt.Frame(A=range(5), B=[0.0]*5)
    DT[:, "A"] = 10**100
    DT[:, "B"] = 10**100
    assert_equals(DT, dt.Frame(A=[1.0e100]*5, B=[1.0e100]*5))

@pytest.mark.parametrize("stype", [dt.bool8] + stypes_str)
def test_assign_integer_to_different_type(stype):
    DT = dt.Frame(A=[5], stype=stype)
    assert DT.stype == stype
    DT[:, "A"] = 777
    assert_equals(DT, dt.Frame(A=[777], stype=dt.int32))

#-------------------------------------------------------------------------------
# Assign float values
#-------------------------------------------------------------------------------

def test_assign_to_newcolumn_subset():
    DT = dt.Frame(A=range(5))
    DT[[1, 4], "B"] = 3.7
    assert_equals(DT, dt.Frame(A=range(5), B=[None, 3.7, None, None, 3.7]))

def test_assign_float_upcast():
    # When float32 overflows, it is converted to float64
    DT = dt.Frame(A=[1.3, 2.7], stype=dt.float32)
    DT[:, "A"] = 1.5e+100
    assert_equals(DT, dt.Frame(A=[1.5e100, 1.5e100]))

def test_assign_to_float32_column():
    DT = dt.Frame(A=range(5), stype=dt.float32)
    DT[:, "A"] = 3.14159
    assert_equals(DT, dt.Frame(A=[3.14159] * 5, stype=dt.float32))

@pytest.mark.parametrize("stype", [dt.bool8] + stypes_int + stypes_str)
def test_assign_float_to_different_type(stype):
    DT = dt.Frame(A=[5], stype=stype)
    assert DT.stype == stype
    DT[:, "A"] = 3.14159265
    assert_equals(DT, dt.Frame(A=[3.14159265]))

#-------------------------------------------------------------------------------
# Assign string values
#-------------------------------------------------------------------------------

def test_assign_string_to_str64():
    DT = dt.Frame(A=["wonder", None, "ful", None], stype=dt.str64)
    DT[:, "A"] = "beep"
    assert_equals(DT, dt.Frame(A=["beep"] * 4, stype=dt.str64))

@pytest.mark.parametrize("stype", [dt.bool8] + stypes_int + stypes_float)
def test_assign_string_to_different_type(stype):
    DT = dt.Frame(A=[5], stype=stype)
    assert DT.stype == stype
    DT[:, "A"] = 'what?'
    assert_equals(DT, dt.Frame(A=['what?']))

def test_assign_str_to_empty_frame():
    # See issue #2043, the assignment used to deadlock
    DT = dt.Frame(W=[], stype=dt.str32)
    DT[f.W == '', f.W] = 'yay!'
    assert_equals(DT, dt.Frame(W=[], stype=dt.str32))

#-------------------------------------------------------------------------------
# Assign types
#-------------------------------------------------------------------------------

def test_assign_type_simple():
    DT = dt.Frame(A=range(5))
    DT["A"] = dt.float64
    assert_equals(DT, dt.Frame(A=range(5), stype=dt.float64))

def test_assign_type_to_many_columns():
    DT = dt.Frame(A=[True, False], B=[3, 7], C=[2.4, 9.9])
    DT[:, :] = dt.int64
    assert_equals(DT, dt.Frame(A=[1, 0], B=[3, 7], C=[2, 9], stype=dt.int64))

def test_assign_type_to_some_columns():
    DT = dt.Frame(A=['bii', 'doo'], B=[3, 5], C=[4, 4])
    DT[:, int] = float
    assert_equals(DT, dt.Frame(A=['bii', 'doo'], B=[3.0, 5.0], C=[4.0, 4.0]))

@pytest.mark.parametrize("st", stypes_all)
def test_assign_stypes(st):
    DT = dt.Frame(A=[1])
    DT["A"] = st
    assert DT.stype == st

@pytest.mark.parametrize("pt", [bool, int, float, str, object])
def test_assign_python_type(pt):
    DT = dt.Frame(A=[7])
    DT["A"] = pt
    if pt is object:
        assert DT.stype == dt.obj64
    else:
        assert type(DT[0, 0]) is pt

@pytest.mark.parametrize("lt", [dt.ltype.bool, dt.ltype.int, dt.ltype.real,
                                dt.ltype.str, dt.ltype.obj])
def test_assign_ltype(lt):
    DT = dt.Frame(A=[7])
    DT["A"] = lt
    assert DT.ltypes[0] == lt

def test_assign_type_without_stype_change():
    # Verify that if a column already has same ltype, then that ltype will
    # not change upon the assignment
    DT = dt.Frame([[0], [1], [2], [3], [4], [5]],
                  stypes=[dt.bool8, dt.int8, dt.int16, dt.int32, dt.int64,
                          dt.float32])
    DT[:, "C0":"C5"] = int
    assert_equals(DT, dt.Frame([[0], [1], [2], [3], [4], [5]],
                               stypes=[dt.int32, dt.int8, dt.int16, dt.int32,
                                       dt.int64, dt.int32]))

def test_assign_stype_to_new_column():
    DT = dt.Frame(S=range(5))
    DT["N"] = dt.float64
    DT["M"] = dt.float32
    assert_equals(DT, dt.Frame(S=range(5), N=[None]*5, M=[None]*5,
                               stypes=dict(N=dt.float64, M=dt.float32)))

def test_assign_bad_type():
    DT = dt.Frame(A=range(5))
    with pytest.raises(ValueError, match="Unknown type <class 'type'> used in "
                                         "the replacement expression"):
        DT["A"] = type

def test_assign_different_types():
    DT = dt.Frame(A=range(5), B=list("ABCDE"))
    assert DT.stypes == (dt.int32, dt.str32)
    DT[:, update(A=dt.float32, B=dt.str64)]
    assert_equals(DT, dt.Frame(A=range(5), B=list("ABCDE"),
                               stypes=dict(A=dt.float32, B=dt.str64)))

def test_assign_types_partial():
    DT = dt.Frame(A=range(5))
    with pytest.raises(ValueError, match="Partial reassignment of Column's "
                                         "type is not possible"):
        DT[:2, "A"] = str

#-------------------------------------------------------------------------------
# Assign range
#-------------------------------------------------------------------------------

def test_assign_range():
    DT = dt.Frame(A=[3, 4, 0])
    DT["B"] = range(3)
    assert_equals(DT, dt.Frame(A=[3, 4, 0], B=[0, 1, 2]))

def test_assign_range2():
    DT = dt.Frame(A=[7]*7, stype=dt.float32)
    DT["A"] = range(3, 10)
    assert_equals(DT, dt.Frame(A=range(3, 10)))

def test_assign_range_subframe():
    DT = dt.Frame(A=range(20))
    DT[10:, "A"] = range(10)
    assert_equals(DT, dt.Frame(A=list(range(10))*2))

def test_assign_range_compute():
    DT = dt.Frame(A=[5, 10, 100])
    DT["B"] = f.A * range(3)
    assert_equals(DT, dt.Frame(A=[5, 10, 100], B=[0, 10, 200]))


# -*- coding: utf-8 -*-
import logging

from .utils import parse_date
from .config import Config

import xmltodict

logger = logging.getLogger()

class XMLParser(object):
    def __init__(self, xml, config=None):
        self.xml = xml
        self.errors = None
        if config is None:
            config = Config()
        self.config = config
        self.parse_xml(xml)
        logger.debug(self.__dict__)

    def parse_xml(self, xml):
        try:
            parsed = xmltodict.parse(xml, encoding="iso-8859-1")
        except Exception as e:
            logger.debug('Cannot parse the returned xml "%s" -> "%s"', xml, e)
            parsed = {}

        if 'errors' in parsed:
            self.errors = parsed['errors']['error']

        return parsed

class PagSeguroNotificationResponse(XMLParser):
    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        parsed = super(PagSeguroNotificationResponse, self).parse_xml(xml)
        if self.errors:
            return
        transaction = parsed.get('transaction', {})
        for k, v in transaction.items():
            setattr(self, k, v)

class PagSeguroPreApprovalNotificationResponse(XMLParser):
    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        parsed = super(PagSeguroPreApprovalNotificationResponse,
                       self).parse_xml(xml)
        if self.errors:
            return
        transaction = parsed.get('transaction', {})
        for k, v in transaction.items():
            setattr(self, k, v)

class PagSeguroPreApprovalCancel(XMLParser):
    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        parsed = super(PagSeguroPreApprovalCancel, self).parse_xml(xml)
        if self.errors:
            return
        transaction = parsed.get('transaction', {})
        for k, v in transaction.items():
            setattr(self, k, v)

class PagSeguroCheckoutSession(XMLParser):
    def __init__(self, xml, config=None):
        self.session_id = None
        super(PagSeguroCheckoutSession, self).__init__(xml, config)

    def parse_xml(self, xml):
        parsed = super(PagSeguroCheckoutSession, self).parse_xml(xml)
        if self.errors:
            return
        session = parsed.get('session', {})
        self.session_id = session.get('id')

class PagSeguroPreApprovalPayment(XMLParser):
    def __init__(self, xml, config=None):
        self.code = None
        super(PagSeguroPreApprovalPayment, self).__init__(xml, config)

    def parse_xml(self, xml):
        parsed = super(PagSeguroPreApprovalPayment, self).parse_xml(xml)
        if self.errors:
            return
        result = parsed.get('result', {})
        self.code = result.get('transactionCode')
        self.date = parse_date(result.get('date'))

class PagSeguroCheckoutResponse(XMLParser):
    def __init__(self, xml, config=None):
        self.code = None
        self.date = None
        self.payment_url = None
        self.payment_link = None
        self.transaction = None
        super(PagSeguroCheckoutResponse, self).__init__(xml, config)

    def parse_xml(self, xml):
        parsed = super(PagSeguroCheckoutResponse, self).parse_xml(xml)
        if self.errors:
            return
        checkout = parsed.get('checkout', {})
        self.code = checkout.get('code')
        self.date = parse_date(checkout.get('date'))

        self.payment_url = self.config.PAYMENT_URL % self.code

        # this is used only for transparent checkout process
        self.transaction = parsed.get('transaction', {})
        self.payment_link = self.transaction.get('paymentLink')

class PagSeguroTransactionSearchResult(XMLParser):
    current_page = None
    total_pages = None
    results_in_page = None
    transactions = []

    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        parsed = super(PagSeguroTransactionSearchResult, self).parse_xml(xml)
        if self.errors:
            return
        search_result = parsed.get('transactionSearchResult', {})
        self.transactions = search_result.get('transactions', {})
        self.transactions = self.transactions.get('transaction', [])
        if not isinstance(self.transactions, list):
            self.transactions = [self.transactions]
        self.current_page = search_result.get('currentPage', None)
        if self.current_page is not None:
            self.current_page = int(self.current_page)
        self.results_in_page = search_result.get('resultsInThisPage', None)
        if self.results_in_page is not None:
            self.results_in_page = int(self.results_in_page)
        self.total_pages = search_result.get('totalPages', None)
        if self.total_pages is not None:
            self.total_pages = int(self.total_pages)

class PagSeguroPreApproval(XMLParser):

    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        parsed = super(PagSeguroPreApproval, self).parse_xml(xml)
        if self.errors:
            return
        result = parsed.get('preApproval', {})
        self.name = result.get('name', None)
        self.code = result.get('code', None)
        self.date = parse_date(result.get('date'))
        self.tracker = result.get('tracker', None)
        self.status = result.get('status', None)
        self.reference = result.get('reference', None)
        self.last_event_date = result.get('lastEventDate', None)
        self.charge = result.get('charge', None)
        self.sender = result.get('sender', {})

class PagSeguroPreApprovalSearch(XMLParser):

    current_page = None
    total_pages = None
    results_in_page = None
    pre_approvals = []

    def __getitem__(self, key):
        getattr(self, key, None)

    def parse_xml(self, xml):
        parsed = super(PagSeguroPreApprovalSearch, self).parse_xml(xml)
        if self.errors:
            return
        search_result = parsed.get('preApprovalSearchResult', {})
        self.pre_approvals = search_result.get('preApprovals', {})
        self.pre_approvals = self.pre_approvals.get('preApproval', [])
        if not isinstance(self.pre_approvals, list):
            self.pre_approvals = [self.pre_approvals]
        self.current_page = search_result.get('currentPage', None)
        if self.current_page is not None:
            self.current_page = int(self.current_page)
        self.results_in_page = search_result.get('resultsInThisPage', None)
        if self.results_in_page is not None:
            self.results_in_page = int(self.results_in_page)
        self.total_pages = search_result.get('totalPages', None)
        if self.total_pages is not None:
            self.total_pages = int(self.total_pages)


# importint the necessary lirbraries

# import tensorflow.keras as kerasfrom __future__ import print_function
from __future__ import print_function

import keras
from keras.layers import Convolution2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import *
from keras.layers.normalization import BatchNormalization
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint
from keras import initializers
from keras.preprocessing import image
from keras.preprocessing.image import img_to_array
from keras import layers
from keras.models import load_model
import matplotlib.image as mpimg

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import tensorflow as tf
from keras import backend as K
from keras.layers import Layer
from keras import activations
from keras import utils
from keras import models
from keras.models import Model
from keras.callbacks import ModelCheckpoint
from keras.optimizers import Adam, SGD

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
from os import listdir
import os
from imutils import paths
import numpy as np
import pickle
import matplotlib.pyplot as plt
import cv2
import random
import os

class Length(layers.Layer):
    """
    Compute the length of vectors. This is used to compute a Tensor that has the same shape with y_true in margin_loss.
    Using this layer as model's output can directly predict labels by using `y_pred = np.argmax(model.predict(x), 1)`
    inputs: shape=[None, num_vectors, dim_vector]
    output: shape=[None, num_vectors]
    """

    def call(self, inputs, **kwargs):
        return K.sqrt(K.sum(K.square(inputs), -1) + K.epsilon())

    def compute_output_shape(self, input_shape):
        return input_shape[:-1]

    def get_config(self):
        config = super(Length, self).get_config()
        return config

class Mask(layers.Layer):
    """
    Mask a Tensor with shape=[None, num_capsule, dim_vector] either by the capsule with max length or by an additional
    input mask. Except the max-length capsule (or specified capsule), all vectors are masked to zeros. Then flatten the
    masked Tensor.
    For example:
        ```
        x = keras.layers.Input(shape=[8, 3, 2])  # batch_size=8, each sample contains 3 capsules with dim_vector=2
        y = keras.layers.Input(shape=[8, 3])  # True labels. 8 samples, 3 classes, one-hot coding.
        out = Mask()(x)  # out.shape=[8, 6]
        # or
        out2 = Mask()([x, y])  # out2.shape=[8,6]. Masked with true labels y. Of course y can also be manipulated.
        ```
    """

    def call(self, inputs, **kwargs):
        if type(inputs) is list:  # true label is provided with shape = [None, n_classes], i.e. one-hot code.
            assert len(inputs) == 2
            inputs, mask = inputs
        else:  # if no true label, mask by the max length of capsules. Mainly used for prediction
            # compute lengths of capsules
            x = K.sqrt(K.sum(K.square(inputs), -1))
            # generate the mask which is a one-hot code.
            # mask.shape=[None, n_classes]=[None, num_capsule]
            mask = K.one_hot(indices=K.argmax(x, 1), num_classes=x.get_shape().as_list()[1])

        # inputs.shape=[None, num_capsule, dim_capsule]
        # mask.shape=[None, num_capsule]
        # masked.shape=[None, num_capsule * dim_capsule]
        masked = K.batch_flatten(inputs * K.expand_dims(mask, -1))
        return masked

    def compute_output_shape(self, input_shape):
        if type(input_shape[0]) is tuple:  # true label provided
            return tuple([None, input_shape[0][1] * input_shape[0][2]])
        else:  # no true label provided
            return tuple([None, input_shape[1] * input_shape[2]])

    def get_config(self):
        config = super(Mask, self).get_config()
        return config

def squash(vectors, axis=-1):
    """
    The non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0
    :param vectors: some vectors to be squashed, N-dim tensor
    :param axis: the axis to squash
    :return: a Tensor with same shape as input vectors
    """
    s_squared_norm = K.sum(K.square(vectors), axis, keepdims=True)
    scale = s_squared_norm / (1 + s_squared_norm) / K.sqrt(s_squared_norm + K.epsilon())
    return scale * vectors

class CapsuleLayer(layers.Layer):
    """
    The capsule layer. It is similar to Dense layer. Dense layer has `in_num` inputs, each is a scalar, the output of the
    neuron from the former layer, and it has `out_num` output neurons. CapsuleLayer just expand the output of the neuron
    from scalar to vector. So its input shape = [None, input_num_capsule, input_dim_capsule] and output shape = \
    [None, num_capsule, dim_capsule]. For Dense Layer, input_dim_capsule = dim_capsule = 1.

    :param num_capsule: number of capsules in this layer
    :param dim_capsule: dimension of the output vectors of the capsules in this layer
    :param routings: number of iterations for the routing algorithm
    """

    def __init__(self, num_capsule, dim_capsule, routings=3,
                 kernel_initializer='glorot_uniform',
                 **kwargs):
        super(CapsuleLayer, self).__init__(**kwargs)
        self.num_capsule = num_capsule
        self.dim_capsule = dim_capsule
        self.routings = routings
        self.kernel_initializer = initializers.get(kernel_initializer)

    def build(self, input_shape):
        assert len(input_shape) >= 3, "The input Tensor should have shape=[None, input_num_capsule, input_dim_capsule]"
        self.input_num_capsule = input_shape[1]
        self.input_dim_capsule = input_shape[2]

        # Transform matrix
        self.W = self.add_weight(shape=[self.num_capsule, self.input_num_capsule,
                                        self.dim_capsule, self.input_dim_capsule],
                                 initializer=self.kernel_initializer,
                                 name='W')

        self.built = True

    def call(self, inputs, training=None):
        # inputs.shape=[None, input_num_capsule, input_dim_capsule]
        # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule]
        inputs_expand = K.expand_dims(inputs, 1)

        # Replicate num_capsule dimension to prepare being multiplied by W
        # inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule]
        inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1])

        # Compute `inputs * W` by scanning inputs_tiled on dimension 0.
        # x.shape=[num_capsule, input_num_capsule, input_dim_capsule]
        # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule]
        # Regard the first two dimensions as `batch` dimension,
        # then matmul: [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule].
        # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule]
        inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled)

        # Begin: Routing algorithm ---------------------------------------------------------------------#
        # The prior for coupling coefficient, initialized as zeros.
        # b.shape = [None, self.num_capsule, self.input_num_capsule].
        b = tf.zeros(shape=[K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule])

        assert self.routings > 0, 'The routings should be > 0.'
        for i in range(self.routings):
            # c.shape=[batch_size, num_capsule, input_num_capsule]
            c = tf.nn.softmax(b, dim=1)

            # c.shape =  [batch_size, num_capsule, input_num_capsule]
            # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
            # The first two dimensions as `batch` dimension,
            # then matmal: [input_num_capsule] x [input_num_capsule, dim_capsule] -> [dim_capsule].
            # outputs.shape=[None, num_capsule, dim_capsule]
            outputs = squash(K.batch_dot(c, inputs_hat, [2, 2]))  # [None, 10, 16]

            if i < self.routings - 1:
                # outputs.shape =  [None, num_capsule, dim_capsule]
                # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
                # The first two dimensions as `batch` dimension,
                # then matmal: [dim_capsule] x [input_num_capsule, dim_capsule]^T -> [input_num_capsule].
                # b.shape=[batch_size, num_capsule, input_num_capsule]
                b += K.batch_dot(outputs, inputs_hat, [2, 3])
        # End: Routing algorithm -----------------------------------------------------------------------#

        return outputs

    def compute_output_shape(self, input_shape):
        return tuple([None, self.num_capsule, self.dim_capsule])

    def get_config(self):
        config = {
            'num_capsule': self.num_capsule,
            'dim_capsule': self.dim_capsule,
            'routings': self.routings
        }
        base_config = super(CapsuleLayer, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

def PrimaryCap(inputs, dim_capsule, n_channels, kernel_size, strides, padding):
    """
    Apply Conv2D `n_channels` times and concatenate all capsules
    :param inputs: 4D tensor, shape=[None, width, height, channels]
    :param dim_capsule: the dim of the output vector of capsule
    :param n_channels: the number of types of capsules
    :return: output tensor, shape=[None, num_capsule, dim_capsule]
    """
    output = layers.Conv2D(filters=dim_capsule * n_channels, kernel_size=kernel_size, strides=strides, padding=padding,
                           name='primarycap_conv2d')(inputs)
    outputs = layers.Reshape(target_shape=[-1, dim_capsule], name='primarycap_reshape')(output)
    return layers.Lambda(squash, name='primarycap_squash')(outputs)

"""
# The following is another way to implement primary capsule layer. This is much slower.
# Apply Conv2D `n_channels` times and concatenate all capsules
def PrimaryCap(inputs, dim_capsule, n_channels, kernel_size, strides, padding):
    outputs = []
    for _ in range(n_channels):
        output = layers.Conv2D(filters=dim_capsule, kernel_size=kernel_size, strides=strides, padding=padding)(inputs)
        outputs.append(layers.Reshape([output.get_shape().as_list()[1] ** 2, dim_capsule])(output))
    outputs = layers.Concatenate(axis=1)(outputs)
    return layers.Lambda(squash)(outputs)
"""

def CapsNet(input_shape, n_class, routings):
    """
    A Capsule Network on MNIST.
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    :param routings: number of routing iterations
    :return: Two Keras Models, the first one used for training, and the second one for evaluation.
            `eval_model` can also be used for training.
    """
    x = layers.Input(shape=input_shape)

    # Layer 1: Just a conventional Conv2D layer
    conv1 = layers.Conv2D(filters=128, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x)

    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid')

    # Layer 3: Capsule layer. Routing algorithm works here.
    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
                             name='digitcaps')(primarycaps)

    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
    # If using tensorflow, this will not be necessary. :)
    out_caps = Length(name='capsnet')(digitcaps)

    # Decoder network.
    y = layers.Input(shape=(n_class,))
    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training
    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction

    # Shared Decoder model in training and prediction
    decoder = models.Sequential(name='decoder')
    decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
    decoder.add(layers.Dense(1024, activation='relu'))
    decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
    decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))

    # Models for training and evaluation (prediction)
    train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
    eval_model = models.Model(x, [out_caps, decoder(masked)])

    # manipulate model
    noise = layers.Input(shape=(n_class, 16))
    noised_digitcaps = layers.Add()([digitcaps, noise])
    masked_noised_y = Mask()([noised_digitcaps, y])
    manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
    return train_model, eval_model, manipulate_model

def margin_loss(y_true, y_pred):
    """
    Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it.
    :param y_true: [None, n_classes]
    :param y_pred: [None, num_capsule]
    :return: a scalar loss value.
    """
    L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \
        0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))

    return K.mean(K.sum(L, 1))

def capsule_prediction(image_path):
    # define the model
    model, eval_model, manipulate_model = CapsNet(input_shape=(28, 28, 3),
                                                  n_class=10,
                                                  routings=3)
    eval_model.summary()
    dir_name = os.path.dirname(os.path.abspath(__file__))
    eval_model.load_weights(dir_name + "/best_weights_capsule_train.h5")

    # setting up predictions
    new_img = None
    try:
        new_img = image.load_img(image_path, target_size=(28, 28))
    except OSError:
        return {'error': 'could not identify image'}

    the_image = image.load_img(image_path)
    img = image.img_to_array(new_img)
    img = np.expand_dims(img, axis=0)
    img = img / 255

    prediction, y_recon = eval_model.predict(img)
    li = ["Bacterial_spot", "Early_blight", "healthy", "Late_blight", "Leaf_Mold", "mosaic_virus", "Septoria_leaf_spot",
          "spider_mite", "Target_Spot", "Yellow_Leaf_Curl"]
    # decode the results into a list of tuples (class, description, probability)
    # (one such list for each sample in the batch)
    class_name = li[prediction.argmax()]

    '''
    d = prediction.flatten()
    j = d.max()
    for index,item in enumerate(d):
        if item == j:
            class_name = li[index]
    '''

    # ploting image with predicted class name
    '''plt.figure(figsize=(4, 4))
    plt.imshow(the_image)
    plt.axis('off')
    plt.title(class_name)'''
    K.clear_session()
    return {'disease': class_name}


import logging

logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.INFO)

def do_add_remove_maglev(test, table, add, remove, exp_table, exp_prev_ele_map):
    logger.info("===Add/Remove Maglev test===")
    if add:
        logger.info("Adding {} to the table.".format(add))
        table.add(add)
    if remove:
        logger.info("Removing {} from the table.".format(remove))
        table.remove(remove)
    act_table = table.get_table()
    act_prev_ele_map = table.get_prev_elements_map()
    test.assertEqual(exp_table, act_table)
    test.assertEqual(act_prev_ele_map, exp_prev_ele_map)
    logger.info("Number of elements replaced {}".format(
        table.elements_replaced))
    logger.info("Number of elements replacing {}".format(
        table.elements_replacing))


import numpy as np
import pandas as pd
import sqlite3

from FoodTables.FoundationTable import FoundationTable
from FoodTables.MarketTable import MarketTable

VOLUME_CONVERSIONS_TABLE = "volume_conversions"
MASS_CONVERSIONS_TABLE = "mass_conversions"
FOOD_TABLE = "food"

con = sqlite3.connect("data.sqlite")

def reset_db():
    con.execute(f"DROP TABLE IF EXISTS {VOLUME_CONVERSIONS_TABLE}")
    con.execute(f"CREATE TABLE IF NOT EXISTS `{VOLUME_CONVERSIONS_TABLE}` "
                f"(`id` TEXT NOT NULL, `name` TEXT NOT NULL, `equiv` REAL NOT NULL, PRIMARY KEY(`id`))")

if __name__ == '__main__':
    reset_db()

    u_df = pd.read_csv("conversions.csv", dtype={"id": "string", "name": "string", "equiv": np.float64})
    u_df.to_sql(
        name=VOLUME_CONVERSIONS_TABLE,
        con=con,
        if_exists="append",
        index=False
    )

    pd.concat(
        [
            FoundationTable().get_df(),
            MarketTable().get_df()
        ],
        ignore_index=True
    ).to_sql(
        name=FOOD_TABLE,
        con=sqlite3.connect("data.sqlite"),
        if_exists="replace",
        index=False
    )


# 스포츠 기사 크롤링

import requests
from bs4 import BeautifulSoup as bs

from apps.resources.models import Article

def crawling_entertain_news():
    entertain_url = "https://entertain.naver.com"

    entertain_url_home = entertain_url + "/home"

    response = requests.get(entertain_url_home)
    soup = bs(response.text, "html.parser")

    ranking = soup.find("div", {"class": "rank_lst"})
    lis = ranking.findAll("li")

    entertain_news_list = []

    for li in lis:
        entertain_news_list.append(entertain_url + li.find("a")["href"])

    # 연예기사 넘 많아서 5개로 줄임
    entertain_news_list = entertain_news_list[:5]

    parsed_news = []

    for news in entertain_news_list:
        response = requests.get(news)
        soup = bs(response.text, "html.parser")
        title = (
            soup.select(".end_tit")[0]
            .text.replace("\n", "")
            .replace("\t", "")
            .replace("\r", "")
        )
        content = (
            soup.find("div", {"id": "articeBody"})
            .text.replace("\n", "")
            .replace("\t", "")
            .replace("\r", "")
        )
        url = news
        if len(content) > 500 & len(content) < 4000:
            parsed_news.append(
                {
                    "title": title,
                    "content": content,
                    "url": url,
                }
            )

    if len(parsed_news) > 0:
        for news in parsed_news:
            if Article.objects.filter(title=news["title"]).exists():
                continue
            else:
                Article.objects.create(
                    kind=Article.ArticleKindChoices.ENTERTAINMENT,
                    title=news["title"],
                    content=news["content"],
                    url=news["url"],
                )


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('api', '0019_auto_20150121_1722'),
    ]

    operations = [
        migrations.AlterField(
            model_name='contestentry',
            name='jam_points',
            field=models.DecimalField(default=0, max_digits=9, decimal_places=1),
            preserve_default=True,
        ),
    ]


import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

class clusteredData():
    def __init__(self,dataSet)-> None:
        self.__dataSet = dataSet
        self.__centriods,self.__belongsTo = elbowMethod(dataSet,8)
        #self.__centriods,self.__belongsTo = kmeans(3,dataSet)
        #self.plot()

    def plot(self) -> None:
        k = len(self.__centriods)
        colors = ['r', 'g','y']
        #color=iter(cm.rainbow(np.linspace(0,1,k)))
        __, ax = plt.subplots()

        for i in range(len(self.__dataSet)):
            x = self.__dataSet[i][0]
            y = self.__dataSet[i][1]
            ax.plot(x, y, (colors[int(self.__belongsTo[i])] + 'o'))

        for i in range(k):
            ax.plot(self.__centriods[i][0], self.__centriods[i][1], 'bo')
        plt.show()

    def getBelongsTo(self):
        return self.__belongsTo
    def getCentriods(self):
        return self.__centriods

def euclidian(a, b):
    return np.linalg.norm(a-b)

def kmeans(k: int, dataSet: np.array):
    numInstances, numFeatures = dataSet.shape

    # intializes the centriods at a random starting point
    startingIndexes = np.random.randint(0, numInstances - 1, size=k)
    unique = False
    while(not unique):
        indices = set()
        unique = True
        for index in startingIndexes:
            if index in indices:
                unique = False
                startingIndexes = np.random.randint(0, numInstances - 1, size=k)
                break
            else:
                indices.add(index)

    centriods = dataSet[startingIndexes]

    oldCentriods = np.zeros(centriods.shape)
    belongsTo = np.zeros((numInstances, 1))
    norm = euclidian(centriods,oldCentriods)

    while norm > 0:
        norm = euclidian(centriods,oldCentriods)

        for  dataIndex, data in enumerate(dataSet):
            distanceToCentriod = np.zeros((k,1))
            for centriodIndex, centroid in enumerate(centriods):
                distanceToCentriod[centriodIndex] = euclidian(centroid,data)

            belongsTo[dataIndex][0] = np.argmin(distanceToCentriod)

        tmpCentriods = np.zeros((k, numFeatures))
        for index in range(len(centriods)):
            # data points assigned to cluster
            assignedData = [i for i in range(len(belongsTo)) if belongsTo[i] == index]
            centriod = np.mean(dataSet[assignedData], axis=0)
            tmpCentriods [index, :] = centriod

        oldCentriods = centriods
        centriods = tmpCentriods

    return centriods, belongsTo

def getLine(x1,y1,x2,y2):
    m =(y2-y1)/(x2-x1)
    b = y2 - (m*x2)
    return m,b

def getClusters(centriods, belongsTo, dataSet):
    clusters = []
    for centriod in centriods:
        cluster = {}
        cluster["centriod"] = centriod
        cluster["dataPoints"] = []
        clusters.append(cluster)

    for itemIndex, item in enumerate(belongsTo):
        clusters[int(item)]["dataPoints"].append(dataSet[itemIndex])

    return clusters

def getClusterMean(cluster):
    distance = 0
    for datapoint in cluster["dataPoints"]:
        distance += euclidian(cluster["centriod"],datapoint)
    try:
        return distance/len(cluster["dataPoints"])
    except ZeroDivisionError:
        print("zero sized cluster")
        return None

def plotElbowMethod(datas):
    #color=iter(cm.rainbow(np.linspace(0,1,k)))
    __, ax = plt.subplots()

    for dataIndex, data in enumerate(datas):
        x = dataIndex
        y = data["sse"]
        ax.plot(x, y, "bo")
    plt.show()

def elbowMethod(dataSet, maxK = 8,plot =False):
    datas = []
    for i in range(1,maxK+1):
        data = {}
        centriods, belongsTo = kmeans(i,dataSet)
        data['centriods'] = centriods
        data['belongsTo'] = belongsTo
        data['sse'] = 0
        clusters = getClusters(centriods, belongsTo, dataSet)
        for cluster in clusters:
            mean = getClusterMean(cluster)
            for dataPoint in cluster["dataPoints"]:
                data['sse'] += (euclidian(dataPoint,cluster["centriod"]) - mean )**2
        datas.append(data)

    #plots elbow curve
    if plot:
        plotElbowMethod(datas)

    #gets line of starting and end points
    m,b = getLine(0,datas[0]['sse'],len(datas)-1,datas[len(datas)-1]['sse'])
    best = 0
    length = 0

    for i in range(len(datas)):
        y = m*i +b
        #print(y-datas[i]["sse"])
        if y - datas[i]["sse"] > length:
            best = i
            length = y - datas[i]["sse"]
    print(best+1)
    return datas[best]['centriods'], datas[best]['belongsTo']


import logging
from google.cloud import secretmanager
from google.cloud import pubsub

# format logs
formatter = '%(asctime)s - %(levelname)s - %(message)s'
logging.basicConfig(format=formatter, level=logging.DEBUG)
logging.basicConfig()
logger = logging.getLogger(__name__)

# get secrets
secrets = secretmanager.SecretManagerServiceClient()
gcp_project_id = secrets.access_secret_version(request={"name": "projects/952416783871/secrets/gcp_project_id/versions/1"}).payload.data.decode()

# connect to resources
publisher = pubsub.PublisherClient()

# sends FEC file paths to be imported into BigQuery
def federal_fec_ingest_queue_import(message, context):

    # list of files to import from Google Cloud Storage
    files = [
        "weball22/weball22.txt",
        "cn22/cn.txt",
        "ccl22/ccl.txt",
        "webl22/webl22.txt",
        "cm22/cm.txt",
        "webk22/webk22.txt",
        "indiv22/itcont.txt",
        "pas222/itpas2.txt",
        "oth22/itoth.txt",
        "oppexp22/oppexp.txt",
        "independent_expenditure_2022/independent_expenditure_2022.csv",
        "ElectioneeringComm_2022/ElectioneeringComm_2022.csv",
        "CommunicationCosts_2022/CommunicationCosts_2022.csv"
    ]

    for filepath in files:

        # sends a message to Pub/Sub to import the files
        topic = 'projects/' + gcp_project_id + '/topics/federal_fec_ingest_import_bigquery'
        publisher.publish(topic, b'import FEC file', filepath=filepath)
        logger.info(' - '.join(['COMPLETED', 'file sent to be imported', filepath]))

    # return number of files queued
    return len(files)


import numpy as np
# from sklearn.metrics import accuracy_score
from sortedcontainers import SortedList
import math
import matplotlib.pyplot as plt

class ml_model(object):
    """
    Parameters
    ------------
    method : str
        which ML lib
    learning_rate : float (default: 0.01)
        Learning rate (between 0.0 and 1.0)
    num_epochs : int (default: 100)
        How many run to train dataset.
    shuffle : bool (default: True)
        Shuffles training data every epoch

    Attributes
    ------------
    w_ : 1d-array, float
        Weights
    errors_ : list, float
    """

    def __init__(self, method=None, learning_rate=0.01,
                 num_epochs=100, shuffle=None):
        self.method = method
        self.learning_rate = learning_rate
        self.num_epochs = num_epochs
        self.shuffle = shuffle
        self.w_ = []
        self.cost_ = []
        # print("ml_model: __init__")

    def fit(self, X_train, Y, standardize=False):

        # Dimensions, Features of X_Train
        self.N, self.D = X_train.shape
        print("X_train has {0} samples with {1} \
            features".format(self.N, self.D))

        self.Y = Y

        # print("X_train:", X_train)

        if(standardize is True):
            X_std = np.copy(X_train)
            for i in range(self.D):
                X_std[:, i] = \
                    (X_train[:, i] - X_train[:, i].mean()) / X_train[:, i].std()

            self.X_train = np.copy(X_std)

        else:
            self.X_train = np.copy(X_train)

        # print("X_train_std:", self.X_train)

        if (self.shuffle is True):
            # init w_ as random numbers
            self.w_ = np.random.randn(1 + self.D)

            # shuffle X_train, Y
            # r = np.random.permutation(self.N)
            # self.X_train = self.X_train[r]
            # self.Y = self.Y[r]

        else:
            # init w_ as zeros with w0
            self.w_ = np.zeros((1 + self.D))

        print("shape of X_train:", self.X_train.shape)
        print("shape of w_:", self.w_.shape)
        print("shape of Y:", self.Y.shape)

    def activation_fn(self, z, activation=None):

            if(activation is None):
                return z
            elif(activation == "sign"):
                return np.piecewise(z, [z < 0, z >= 0], [-1, 1])
            elif(activation == "step"):
                return np.piecewise(z, [z < 0, z >= 0], [0, 1])
            elif(activation == "sigmoid"):
                return 1.0 / (1.0 + np.exp(-z))
            elif(activation == "tanh"):
                return (1.0 - np.exp(-2*z))/(1.0 + np.exp(-2*z))

    def net_input(self, X_data):
        # net input: w0x0 + w1x1... + wixi
        # print("net_input:")
        # print("shape of X_data:", X_data.shape)
        # print("shape of self.w_[1:]:", self.w_[1:].shape)
        return np.dot(X_data, self.w_[1:]) + self.w_[0]

    def predict(self):
        pass

    def r2_evl(self, Y, Y_hat):
        d1 = Y - Y_hat
        d2 = Y - Y.mean()
        # print("Shape of d1:", d1.shape)
        # print("Shape of d2:", d2.shape)
        r2 = 1 - (d1.dot(d1) / d2.dot(d2))
        print("R2:", r2)

    def score(self, Y, Y_hat):
        print('Misclassified samples: %d' % (Y != Y_hat).sum())
        # print('Accuracy: %.2f' % accuracy_score(Y, Y_hat))
        print('Score:', np.mean(Y_hat == Y))

class myPerceptron(ml_model):

    def __init__(self, method=None, learning_rate=0.01,
                 num_epochs=100, shuffle=True, activation="sign"):
        # print("myPerceptron: __init__")
        super().__init__(method, learning_rate, num_epochs, shuffle)
        self.activation = activation

    """
    Parameters
    ------------
    X_train : array, float
        Dataset for traninig
    Y : array, float
        "True" Y
    """
    def fit(self, X_train, Y, standardize=False):
        super().fit(X_train, Y, standardize)
        self.errors_ = []
        for epoch in range(self.num_epochs):
            errors = 0
            for xi, y in zip(self.X_train, Y):
                y_hat = self.predict(xi)
                error = y - y_hat
                update = self.learning_rate * error
                self.w_[1:] += update * xi
                self.w_[0] += update
                errors += int(error != 0.0)
            self.errors_.append(errors)
            # print('errors:', errors)

        print("final w:\n", self.w_, "\nepochs:",
              (epoch+1), "/", self.num_epochs)
        plt.plot(range(1, len(self.errors_) + 1),
                 self.errors_, marker='o')
        plt.xlabel('Epochs')
        plt.ylabel('Number of erros')
        plt.show()

    def predict(self, X_test, standardize=False):
        # print("shape of X_test:", X_test.shape)
        if(standardize is True):
            for i in range(self.D):
                X_test[:, i] = \
                 (X_test[:, i] - X_test[:, i].mean()) / X_test[:, i].std()

        z = self.net_input(X_test)
        # print("z:",z)
        Y_hat = super().activation_fn(z, self.activation)
        # print("Y_hat:",Y_hat)
        return Y_hat

class myAdaline(ml_model):

    def __init__(self, method=None, learning_rate=0.0001,
                 num_epochs=200, shuffle=True, mini_batch=False):
        # print("myAdaline: __init__")
        super().__init__(method, learning_rate, num_epochs, shuffle)
        self.mini_batch = mini_batch

    """
    Parameters
    ------------
    X_train : array, float
        Dataset for traninig
    Y : array, float
        "True" Y
    """
    def _update_weights(self, xi, target):
        """Apply Adaline learning rule to update the weights"""
        output = super().activation_fn(self.net_input(xi), activation=None)
        error = (target - output)
        # print("shape of xi:", xi.shape)
        # print("shape of error:", error.shape)
        self.w_[1:] += self.learning_rate * xi.dot(error)
        self.w_[0] += self.learning_rate * error
        cost = 0.5 * error**2
        return cost

    def fit(self, X_train, Y, standardize=False):
        super().fit(X_train, Y, standardize)
        cost = 0
        avg_cost = 0
        for epoch in range(self.num_epochs):
            if(self.mini_batch is False):
                z = self.net_input(self.X_train)
                output = super().activation_fn(z, activation=None)
                error = (self.Y - output)
                self.w_[1:] += self.learning_rate * \
                    np.dot(self.X_train.T, error)
                self.w_[0] += self.learning_rate * error.sum()
                cost = (error**2).sum() / 2.0
                self.cost_.append(cost)
            else:
                cost = []
                for xi, target in zip(X_train, Y):
                    cost.append(self._update_weights(xi, target))
                avg_cost = sum(cost) / len(Y)
                self.cost_.append(avg_cost)

        if(self.mini_batch is False):
            print("final w:\n", self.w_, "\nFinal cost:\n", cost,
                  "\nepochs:\n", (epoch+1), "/", self.num_epochs)
        else:
            print("final w:\n", self.w_, "\nFinal cost:\n", avg_cost,
                  "\nepochs:\n", (epoch+1), "/", self.num_epochs)

        plt.plot(range(1, len(self.cost_) + 1), self.cost_, marker='o')
        plt.xlabel('Epochs')
        plt.ylabel('Sum-squared-error')
        plt.title('Adaline - Learning rate: {0}'.format(self.learning_rate))
        plt.tight_layout()
        plt.show()

    def partial_fit(self, X_train, Y):
        """Fit training data without reinitializing the weights"""
        if Y.ravel().shape[0] > 1:
            for xi, target in zip(X_train, Y):
                self._update_weights(xi, target)
        else:
            self._update_weights(X_train, Y)
        return self

    def predict(self, X_test, standardize=False):
        print("shape of X_test:", X_test.shape)
        if(standardize is True):
            for i in range(self.D):
                X_test[:, i] = (X_test[:, i] - X_test[:, i].mean()) / X_test[:, i].std()

        z = self.net_input(X_test)
        # output = super().activation_fn(z, activation=None)
        Y_hat = super().activation_fn(z, activation="sign")
        return Y_hat

class myLogistic(ml_model):

    def __init__(self, method=None, learning_rate=0.0001,
                 num_epochs=200, shuffle=True):
        # print("myLogistic: __init__")
        super().__init__(method, learning_rate, num_epochs, shuffle)

    """
    Parameters
    ------------
    X_train : array, float
        Dataset for traninig
    Y : array, float
        "True" Y
    """
    def fit(self, X_train, Y, standardize=False):
        super().fit(X_train, Y, standardize)
        cost = 0
        for epoch in range(self.num_epochs):
            z = self.net_input(self.X_train)
            # print("z:", z)
            output = super().activation_fn(z, activation='sigmoid')
            # print("Shape of output:", output.shape)
            # print("output:", output)
            error = (self.Y - output)
            self.w_[1:] += self.learning_rate * np.dot(self.X_train.T, error)
            self.w_[0] += self.learning_rate * error.sum()
            cost = -Y.dot(np.log(output)) - ((1 - Y).dot(np.log(1 - output)))
            self.cost_.append(cost)

        print("final w:\n", self.w_, "\nFinal cost:\n", cost,
              "\nepochs:\n", (epoch+1), "/", self.num_epochs)

        plt.plot(range(1, len(self.cost_) + 1), self.cost_, marker='o')
        plt.xlabel('Epochs')
        plt.ylabel('Sum-squared-error')
        plt.title('Adaline - Learning rate: {0}'.format(self.learning_rate))
        plt.tight_layout()
        plt.show()

    def predict(self, X_test, standardize=False):
        print("shape of X_test:", X_test.shape)
        if(standardize is True):
            for i in range(self.D):
                X_test[:, i] = (X_test[:, i] - X_test[:, i].mean()) / X_test[:, i].std()
        z = self.net_input(X_test)
        # output = super().activation_fn(z, activation='sigmoid')
        Y_hat = super().activation_fn(z, activation="step")
        return Y_hat

class myKNN(ml_model):

    def __init__(self, K):
        self.K = K

    """
    Parameters
    ------------
    X_train : array, float
        Dataset for traninig
    Y : array, float
        "True" Y
    """
    def fit(self, X_train, Y):
        self.X_train = X_train
        self.Y = Y
        # print("self.Y:", self.Y)
        # print("self.X_train:", self.X_train)

    def predict(self, X_test):
        print("shape of X_test:", X_test.shape)
        Y_pred = np.zeros((X_test.shape[0]))
        # print("X_test:", X_test)
        for i, xi_test in enumerate(X_test):
            sl = SortedList()
            for j, xi_train in enumerate(self.X_train):
                diff = xi_test - xi_train
                dist = math.sqrt(diff.dot(diff))
                if(len(sl) < self.K):
                    sl.add((dist, self.Y[j]))
                else:
                    if(dist < sl[-1][0]):
                        del sl[-1]
                        sl.add((dist, self.Y[j]))
            # print("sl:", sl)

            # vote
            votes = {}

            for _, v in sl:
                # print("v:", v)
                votes[v] = votes.get(v, 0) + 1
            max_votes = 0
            max_votes_class = -1
            for v, count in votes.items():
                if count > max_votes:
                    max_votes = count
                    max_votes_class = v
            Y_pred[i] = max_votes_class
        return Y_pred

class myBayes(ml_model):

    def __init__(self, naive=True, pdf='gaussian'):
        self.naive = naive
        self.pdf = pdf

    """
    Parameters
    ------------
    X_train : array, float
        Dataset for traninig
    Y : array, float
        "True" Y
    """
    def fit(self, X_train, Y, smoothing=1e-2):
        self.X_train = X_train
        self.Y = Y

        # print("self.Y:", self.Y)
        # print("self.X_train:", self.X_train)
        N, D = X_train.shape
        print("X_train has {0} samples with {1} features".format(
            N, D))

        self.lable_Prob_in_Y = {}
        labels = set(Y)
        self.lable_numbers = len(labels)

        if self.pdf is 'gaussian':
            if self.naive is True:
                self.X_train_mean_var_in_Y = {}
                for c in labels:
                    such_X_train = X_train[Y == c]
                    self.X_train_mean_var_in_Y[c] = {
                        "mean": such_X_train.mean(axis=0),
                        'var': np.var(such_X_train) + smoothing
                    }
                    self.lable_Prob_in_Y[c] = float(len(Y[Y == c])) / len(Y)

                print("len of X_train_mean_var_in_Y:", len(self.X_train_mean_var_in_Y))
                # print("X_train_mean_var_in_Y:", self.X_train_mean_var_in_Y)
            else:
                self.X_train_mean_cov_in_Y = {}
                for c in labels:
                    such_X_train = X_train[Y == c]
                    self.X_train_mean_cov_in_Y[c] = {
                        "mean": such_X_train.mean(axis=0),
                        'cov': np.cov(such_X_train.T) + np.eye(D)*smoothing
                    }
                    self.lable_Prob_in_Y[c] = float(len(Y[Y == c])) / len(Y)

                print("len of X_train_mean_cov_in_Y:", len(self.X_train_mean_cov_in_Y))
                # print("X_train_mean_cov_in_Y:", self.X_train_mean_cov_in_Y)
            print("len of X_train_Prob_in_Y:", len(self.lable_Prob_in_Y))
            # print("lable_Prob_in_Y:", self.lable_Prob_in_Y)
        else:
            pass

    def predict(self, X_test):

        from scipy.stats import multivariate_normal as mvn

        N = X_test.shape[0]
        print("shape of X_test:", X_test.shape)
        Y_pred_P = np.zeros((N, self.lable_numbers))

        if self.naive is True:
            for c, g in self.X_train_mean_var_in_Y.items():
                mean, var = g["mean"], g["var"]
                Y_pred_P[:, c] = mvn.logpdf(X_test, mean=mean, cov=var) + np.log(self.lable_Prob_in_Y[c])
                # print("c:{0}, g:{1}".format(c, g))
        else:
            for c, g in self.X_train_mean_cov_in_Y.items():
                mean, cov = g["mean"], g["cov"]
                Y_pred_P[:, c] = mvn.logpdf(X_test, mean=mean, cov=cov) + np.log(self.lable_Prob_in_Y[c])
                # print("c:{0}, g:{1}".format(c, g))

        # print("Y_pred_P:{0}".format(Y_pred_P))
        Y_pred = np.argmax(Y_pred_P, axis=1)
        return Y_pred

class myMLP(ml_model):

    def activation_derivative(self, z, activation=None):

            if(activation is None):
                return 0
            elif(activation == "sigmoid"):
                return (super().activation_fn(z, "sigmoid")*(1-super().activation_fn(z, "sigmoid")))
            elif(activation == "tanh"):
                return (1 + super().activation_fn(z, "tanh"))*(1 - super().activation_fn(z, "tanh"))

    def __init__(self, net_arch=[2, 3, 2], activation_h='tanh', l2=0.,
                 num_epochs=100, learning_rate=0.001,
                 shuffle=True, minibatch_size=1):
        self.activation_h = activation_h
        self.layers = len(net_arch)
        self.net_arch = net_arch
        self.l2 = l2
        self.num_epochs = num_epochs
        self.learning_rate = learning_rate
        self.minibatch_size = minibatch_size
        self.shuffle = shuffle

    def _forward(self, X):
        """Compute forward propagation step"""

        A_in = [X]

        # step 1: net input of hidden layer
        # [n_samples, n_features] dot [n_features, n_hidden]
        # -> [n_samples, n_hidden]
        # [n_samples, n_hidden] dot [i layer n_hidden, i+1 n_hidden]
        # print("shape of X:", X.shape)
        for i in range(len(self.weights)-1):
            print("shape of A_in[{0}]:{1}".format(i, A_in[i].shape))
            print("shape of weights({0}):{1}".format(i, self.weights[i].shape))
            z_h = np.dot(A_in[i], self.weights[i])
            a_h = self.activation_fn(z_h, self.activation_h)
            print("shape of a_h({0}):{1}".format(i, a_h.shape))
            # add the bias for the next layer
            ones = np.ones((1, X.shape[0]))
            a_h = np.concatenate((ones.T, a_h), axis=1)
            print("Shape of a_h with bias:", a_h.shape)
            A_in.append(a_h)
            print("Len of A_in:", len(A_in))

        # step 2: net input of output layer
        # [n_samples, n_hidden] dot [n_hidden, n_classlabels]
        # -> [n_samples, n_classlabels]
        z_o = np.dot(A_in[-1], self.weights[-1])
        a_o = self.activation_fn(z_o, self.activation_h)

        return A_in, a_o

    def _compute_cost(self, y, y_hat):
        """Compute cost function.

        Parameters
        ----------
        y : array, shape = (n_samples, n_labels)
            true class labels.
        y_hat : array, shape = [n_samples, n_output_units]
            Activation of the output layer (forward propagation)

        Returns
        ---------
        cost : float
            Regularized cost

        """
        """
        L2_term = (self.l2 *
                   (np.sum(self.w_h ** 2.) +
                    np.sum(self.w_out ** 2.)))
        """
        L2_term = 0

        y = y.reshape(y_hat.shape[0], y_hat.shape[1])
        print("shape of y:", y.shape)
        print("shape of y_hat:", y_hat.shape)
        term1 = -y * (np.log(y_hat))
        term2 = (1. - y) * np.log(1. - y_hat)
        print("shape of term1:", term1.shape)
        print("shape of term2:", term2.shape)
        cost = np.sum(term1 - term2) + L2_term
        return cost

    """
    Parameters
    ------------
    X_train : array, float
        Dataset for traninig
    Y : array, float
        "True" Y
    """
    def fit(self, X_train, Y, standardize=False):

        # Dimensions, Features of X_Train
        self.N, self.D = X_train.shape
        print("X_train has {0} samples with {1} features".format(self.N, self.D))

        self.Y = Y

        # print("X_train:", X_train)

        if(standardize is True):
            X_std = np.copy(X_train)
            for i in range(self.D):
                X_std[:, i] = \
                    (X_train[:, i] - X_train[:, i].mean()) / X_train[:, i].std()

            self.X_train = np.copy(X_std)

        else:
            self.X_train = np.copy(X_train)

        # print("X_train_std:", self.X_train)

        ones = np.ones((1, self.X_train.shape[0]))
        self.X_train = np.concatenate((ones.T, self.X_train), axis=1)
        print("Shape of X_train with bias:", self.X_train.shape)

        self.weights = []
        if (self.shuffle is True):
            # init weights as random numbers
            print("init weights as random numbers:")
            for layer in range(len(self.net_arch) - 1):
                w_ = np.random.randn(self.net_arch[layer] + 1, self.net_arch[layer+1])
                print("for layer {0} to {1}".format(layer, layer+1))
                print("shape of w_:", w_.shape)
                self.weights.append(w_)

            # shuffle X_train, Y
            # r = np.random.permutation(self.N)
            # self.X_train = self.X_train[r]
            # self.Y = self.Y[r]

        else:
            # init w_ as zeros with w0
            print("init weights as 0:")
            for layer in range(len(self.net_arch) - 1):
                w_ = np.zeros(self.net_arch[layer] + 1, self.net_arch[layer+1])
                print("for layer {0} to {1}".format(layer, layer+1))
                print("shape of w_:", w_.shape)
                self.weights.append(w_)

        print("shape of X_train:", self.X_train.shape)
        print("len of weights:", len(self.weights))
        print("shape of Y:", self.Y.shape)

        # iterate over training epochs
        for i in range(self.num_epochs):

            indices = np.arange(X_train.shape[0])

            for start_idx in range(0, indices.shape[0] - self.minibatch_size +
                                   1, self.minibatch_size):
                batch_idx = indices[start_idx:start_idx + self.minibatch_size]
                A_in, y_hat = self._forward(self.X_train[batch_idx])

                ##################
                # Backpropagation
                ##################

                # [n_samples, n_classlabels]
                print("shape of y[batch_idx]:", y[batch_idx].shape)
                print("shape of y_hat:", y_hat.shape)
                delta_out = y_hat - y[batch_idx].reshape(y_hat.shape[0], y_hat.shape[1])
                print("shape of delta_out:", delta_out.shape)

        # error for the output layer
        #self._compute_cost(Y, y_hat)

mlp = myMLP(net_arch=[2, 4, 1], minibatch_size=4)
X = np.array([[0, 0],
                     [0, 1],
                     [1, 0],
                     [1, 1]])

y = np.array([0, 1, 1, 0])

mlp.fit(X, y)


import torch
print(torch.__version__)
torch.get_default_dtype()
torch.get_num_threads()
torch.set_default_dtype(torch.float64)
torch.get_default_dtype()
tensor_arr = torch.Tensor([[1, 2, 3], [4, 5, 6]])
torch.is_tensor(tensor_arr)
torch.numel(tensor_arr)  #Gives the number of elements in the tensor
tensor_uninitialized = torch.Tensor(2, 2)
tensor_uninitialized
tensor_initialized = torch.rand(2, 2)
tensor_initialized
temsor_cpu_int = torch.Tensor([5, 3]).type(torch.IntTensor)
tensor_cuda_int = torch.Tensor([5, 3]).type(torch.cuda.IntTensor)
torch_cpu_short = torch.ShortTensor([1.0, 2.0, 3.0])
tensor_cuda_short = torch.cuda.ShortTensor([1.0, 2.0, 3.0]) #cuda commands create tensor in GPU
tensor_float = torch.cuda.HalfTensor([1.0, 2.0, 3.0]) #takes half the memory
torch_cpu_fill = torch.full((2, 6), fill_value = 10)
torch_cuda_fill = torch.full((2, 6), fill_value = 5).type(torch.cuda.FloatTensor)
tensor_of_ones_cpu = torch.ones([2, 4], dtype = torch.int32)
tensor_of_ones_cuda = torch.ones([2, 4], dtype = torch.int32).type(torch.cuda.IntTensor)
tensor_of_zeros__cpu = torch.zeros_like(tensor_of_ones_cpu)
tensor_of_zeros_gpu = torch.zeros_like(tensor_of_ones_cuda)
tensor_eye_cpu = torch.eye(5)
tensor_eye_gpu = torch.eye(5).type(torch.cuda.FloatTensor)
non_zero_indices = torch.nonzero(tensor_eye_cpu)
non_zero_indices = torch.nonzero(tensor_of_ones_cuda)
i = torch.tensor([[0, 1, 1],
                 [2, 2, 0]])
v = torch.tensor([3, 4, 5], dtype = torch.float32)
sparse_tensor = torch.sparse_coo_tensor(i, v, [2, 5])
sparse_tensor.data

#Simple operations on Tensors
initial_tensor = torch.rand(2, 3)
initial_tensor.fill_(10)
new_tensor = initial_tensor.add(5)
initial_tensor.add_(8)
new_tensor.sqrt_()
x = torch.linspace(start = 0.1, end = 10, steps = 15)
tensor_chunk = torch.chunk(x, 3, 0)
tensor_1 = tensor_chunk[0]
tensor_2 = tensor_chunk[1]
tensor_3 = torch.tensor([3.0, 4.0, 5.0])
torch.cat((tensor_1, tensor_2, tensor_3), 0)
random_tensor = torch.Tensor([[20, 8, 40], [5, 43, 6], [78, 54, 6]])
random_tensor[0, 1]
random_tensor[1:, 1:]
random_tensor.size()
resized_tensor = random_tensor.view(9)
resized_tensor.size()
random_tensor[2, 2] = 200.0
resized_tensor

#Elementwise and Matrix Operations
tensor_unsqueeze = torch.unsqueeze(random_tensor, 2)
tensor_unsqueeze.size()
tensor_transpose = torch.transpose(initial_tensor, 0, 1)
sorted_tensor, sorted_indices = torch.sort(random_tensor)
tensor_float = torch.FloatTensor([-1.1, -2.2, -3.3])
tensor_abs = torch.abs(tensor_float)
rand1 = torch.abs(torch.rand(2,3))
rand2 = torch.abs(torch.rand(2,3))
add1 = rand1 + rand2
tensor_just = torch.Tensor([[1, 2, 3], [-1, -2, -3]])
tensor_div = torch.div(tensor_just, tensor_just + 0.3)
tensor_mul = torch.mul(tensor_just, tensor_just)
tensor_clamp = torch.clamp(tensor_just, min = -0.2, max = 2)
t1 = torch.Tensor([2, 3])
t2 = torch.Tensor([4, 5])
dot_product = torch.dot(t1, t2)
matrix = torch.Tensor([[1, 2, 3], [4, 5, 6]])
vector = torch.Tensor([0, 1, 2])
matrix_vector = torch.mv(matrix, vector)
another_matrix = torch.Tensor([[10,20], [30, 0], [0,50]])
matrix_mul = torch.mm(matrix, another_matrix)
torch.argmax(matrix_mul, dim = 1)
torch.argmin(matrix_mul, dim = 1)

#Converting between PyTorch and numpy tensors
import numpy as np
tensor = torch.rand(4,3)
type(tensor)
numpy_from_tensor = tensor.numpy()
type(numpy_from_tensor)
torch.is_tensor(tensor)
torch.is_tensor(numpy_from_tensor)
numpy_from_tensor[0, 0] = 100.0
numpy_from_tensor
tensor
numpy_arr = np.array([[1.0, 2.0, 3.0], [10.0, 20.0, 30.0], [100.0, 200.0, 300.0]])
tensor_from_numpy = torch.from_numpy(numpy_arr)
type(tensor_from_numpy)
torch.is_tensor(tensor_from_numpy)
tensor_from_numpy_arr = torch.as_tensor(numpy_arr)


# -*- coding: utf-8 -*-

import json
import os
from locale import getdefaultlocale
from .internationalization import SUPPORTED_LANGUAGES

_lang = getdefaultlocale()[0]
for _supported_lang in SUPPORTED_LANGUAGES.keys():
    if _lang in _supported_lang or _supported_lang in _lang:
        _lang = _supported_lang
        break
else:
    _lang = 'en'

SETTINGS_FILE = os.path.expanduser('~/.wbwr_cfg')
DEFAULT_SETTINGS = {
    'blink_warning_shown': False,
    'theme': 1,
    'speed': 125,
    'cpf': 20,
    'language': _lang,
    'font': 'serif',
    'pause_on_image': True,
    'font_size_focus': 45
}

class _settings_type(dict):
    def __getitem__(self, name):
        try:
            return super(_settings_type, self).__getitem__(name)
        except KeyError:
            val = DEFAULT_SETTINGS[name]
            self[name] = val
            return val

    def save(self):
        with open(SETTINGS_FILE, 'w') as file:
            json.dump(self, file)

    def load(self):
        self.clear()
        with open(SETTINGS_FILE, 'r') as file:
            for k, v in json.load(file).items():
                self[k] = v

Settings = _settings_type()
try:
    Settings.load()
except OSError:
    pass  # Settings will be populated and saved on demand


from django import forms
from .models import Booking
from django.core.exceptions import ValidationError

class BookingForm(forms.ModelForm):
    """Form for the booking model."""
    class Meta:
        model = Booking
        fields = (
            'first_name',
            'last_name',
            'phone_number',
            'email_address',
            'date',
            'service_type',
        )

    date = forms.DateField(
        widget=forms.widgets.DateInput(
            attrs={
                'type': 'date'}))

    def __init__(self, *args, **kwargs):

        super().__init__(*args, **kwargs)


from requests.exceptions import ReadTimeout

from django.http import JsonResponse

from ..keycloak_services import check_ldap_connection
from ..keycloak_services import check_ldap_authentication
from ..keycloak_services import request_access_token

def process_request(request, parameter_keys, kc_call):
    token = request_access_token()
    parameters = []
    missing_parameters = []
    for key in parameter_keys:
        parameter = request.GET.get(key, None)
        if parameter:
            parameters.append(parameter)
        else:
            missing_parameters.append(key)
    if missing_parameters:
        error = "parameters missing: {keys}".format(keys=missing_parameters)
        return JsonResponse(
            {'errorMessage': error}, status=400
        )
    status = 400
    json_data = {'errorMessage': "LDAP test error"}
    try:
        check_response = kc_call(
            'master', token, *parameters, timeout=0.4)
        if check_response.status_code == 204:
            status = 200
            json_data = {'successMessage': "LDAP test success"}
        else:
            status = check_response.status_code
            json_data = check_response.json()
    except ReadTimeout:
        status = 408
        json_data['errorMessage'] = "Keycloak: no response"
    finally:
        return JsonResponse(json_data, status=status)  # noqa: B012,
        # return inside finally blocks cause exceptions to be silenced

def verify_connection(request):
    return process_request(request, ['url'], check_ldap_connection)

def verify_authentication(request):
    return process_request(
        request,
        ['url', 'bind_dn', 'bind_credential'],
        check_ldap_authentication,
    )


import ddt
import unittest,requests
@ddt.ddt
class TestLogin(unittest.TestCase):
	@ddt.file_data(r'E:\login.yml')
	@ddt.unpack
	def test_run(self,**kwargs):
		method = kwargs.get('method')
		url = kwargs.get('url')
		data = kwargs.get('data',{})
		header = kwargs.get('header',{})
		is_json = kwargs.get('is_json',0)
		cookie = kwargs.get('cookie',{})
		check = kwargs.get('check')
		if method=='post':
			if is_json:
				r = requests.post(url,json=data,headers=header,cookies=cookie)
			else:
				r = requests.post(url,data=data, headers=header, cookies=cookie)
		else:
			r = requests.get(url,params=data,headers=header, cookies=cookie)
		for c in check:
			self.assertIn(c,r.text)


import logging

from . import BinanceBrokers
from .requests import unauthorizrd_request

from core.config import (
    BINANCE_SPOT_KLINES_URL,
    BINANCE_UM_KLINES_URL,
    BINANCE_CM_KLINES_URL,

    BINANCE_SPOT_MARKET_INFO_URL,
    BINANCE_UM_MARKET_INFO_URL,
    BINANCE_CM_MARKET_INFO_URL,
)

async def get_klines(
    broker: str,
    symbol: str,
    interval: str,
    limit: int = 500,
) -> dict:
    """The function gets klines history from Binance

    Args:
        symbol (str): symbol like BTCUSDT
        interval (str): interval (timeframe) like '15m'
        stop (asyncio.Event): stop event from the bot
        limit (int, optional): limit of klines. Defaults to 500.

    Returns:
        dict: parsed JSON response
    """

    logger = logging.getLogger('get_klines')
    params = {
        'symbol': symbol.upper(),
        'interval': interval,
        'limit': limit,
    }

    if broker == 'Binance-spot':
        url = BINANCE_SPOT_KLINES_URL
    elif broker == 'Binance-UM-Futures':
        url = BINANCE_UM_KLINES_URL
    elif broker == 'Binance-CM-Futures':
        url = BINANCE_CM_KLINES_URL
    else:
        raise ValueError(f'Wrong broker {broker}')

    return await unauthorizrd_request(broker, url, 'get', params, logger)

async def get_market_info(broker: str) -> dict:
    """The function gets exchange info (symbols, limits)

    Args:
        brokers (BinanceBrokers): Binance brokers enum

    Returns:
        dict: parsed JSON data
    """
    logger = logging.getLogger('get_market_info')
    if broker == 'Binance-spot':
        url = BINANCE_SPOT_MARKET_INFO_URL
    elif broker == 'Binance-UM-Futures':
        url = BINANCE_UM_MARKET_INFO_URL
    elif broker == 'Binance-CM-Futures':
        url = BINANCE_CM_MARKET_INFO_URL
    else:
        raise ValueError(f'Wrong broker {broker}')

    return await unauthorizrd_request(broker, url, 'get', {}, logger)


'''
텔레포트 3
시간 제한	메모리 제한	제출	정답	맞은 사람	정답 비율
2 초	512 MB	303	122	101	43.348%
문제
수빈이는 크기가 무한대인 격자판 위에 살고 있다. 격자판의 각 점은 두 정수의 쌍 (x, y)로 나타낼 수 있다.

제일 처음에 수빈이의 위치는 (xs, ys)이고, 집이 위치한 (xe, ye)로 이동하려고 한다.

수빈이는 두 가지 방법으로 이동할 수 있다. 첫 번째 방법은 점프를 하는 것이다. 예를 들어 (x, y)에 있는 경우에 (x+1, y), (x-1, y), (x, y+1), (x, y-1)로 이동할 수 있다. 점프는 1초가 걸린다.

두 번째 방법은 텔레포트를 사용하는 것이다. 텔레포트를 할 수 있는 방법은 총 세 가지가 있으며, 미리 정해져 있다. 텔레포트는 네 좌표 (x1, y1), (x2, y2)로 나타낼 수 있으며, (x1, y1)에서 (x2, y2)로 또는 (x2, y2)에서 (x1, y1)로 이동할 수 있다는 것이다. 텔레포트는 10초가 걸린다.

수빈이의 위치와 집의 위치가 주어졌을 때, 집에 가는 가장 빠른 시간을 구하는 프로그램을 작성하시오.

입력
첫째 줄에 xs와 ys가, 둘째 줄에 xe, ye가 주어진다. (0 ≤ xs, ys, xe, ye ≤ 1,000,000,000)

셋째 줄부터 세 개의 줄에는 텔레포트의 정보 x1, y1, x2, y2가 주어진다. (0 ≤ x1, y1, x2, y2 ≤ 1,000,000,000)

입력으로 주어지는 모든 좌표 6개는 서로 다르다.

출력
수빈이가 집에 가는 가장 빠른 시간을 출력한다.
'''
# check all teleport-use ways with brute-force
import sys
from collections import defaultdict
from itertools import permutations

xs, ys = map(int, sys.stdin.readline().split())
xe, ye = map(int, sys.stdin.readline().split())
m_route = abs(xe-xs) + abs(ye-ys)

tels = defaultdict(tuple)
for i in range(3):
    x1, y1, x2, y2 = list(map(int, sys.stdin.readline().split()))
    tels[(x1, y1)], tels[(x2, y2)] = (x2, y2), (x1, y1)

for n in range(1,4):
    for q in list(permutations(tels.keys(),n)):
        x, y, cnt = xs, ys, 0
        for xt, yt in q:
            new_x, new_y = tels[(xt, yt)]
            if abs(xt-x) + abs(yt-y) + 10 < abs(new_x-x) + abs(new_y-y):
                cnt += abs(xt-x) + abs(yt-y) + 10
                x, y = new_x, new_y
            else:
                cnt = m_route
                break
        m_route = cnt + abs(xe-x) + abs(ye-y) if cnt + abs(xe-x) + abs(ye-y) < m_route else m_route
print(m_route)

# It exceeds time limit
import sys
from collections import deque, defaultdict

xs, ys = map(int, sys.stdin.readline().split())
xe, ye = map(int, sys.stdin.readline().split())
m_route = abs(xe-xs) + abs(ye-ys)
print(m_route)
tels = defaultdict(tuple)
for i in range(3):
    x1, y1, x2, y2 = list(map(int, sys.stdin.readline().split()))
    tels[(x1, y1)], tels[(x2, y2)] = (x2, y2), (x1, y1)
grid_x = max([k[0] for k in tels.keys()] + [xs, xe]) + 1
grid_y = max([k[1] for k in tels.keys()] + [ys, ye]) + 1

visited = [[ m_route for _ in range(grid_y)] for __ in range(grid_x)]
visited[xs][ys] = 0
queue = deque([(xs, ys, 0)])
while queue:
    x, y, cnt = queue.popleft()
    if cnt > visited[xe][ye]: continue
    if tels[(x, y)]:
        new_x, new_y = tels[(x, y)]
        if visited[new_x][new_y] > cnt + 10:
            queue.append((new_x, new_y, cnt + 10))
            visited[new_x][new_y] = cnt + 10
    for dx, dy in [(1, 0), (0, 1), (-1, 0), (0, -1)]:
        if x + dx < 0 or x + dx >= grid_x or y + dy < 0 or y + dy >= grid_y: continue
        if visited[x+dx][y+dy] > cnt+1:
            queue.append((x+dx, y+dy, cnt+1))
            visited[x+dx][y+dy] = cnt+1
print(visited[xe][ye])