"""Defines URL patterns for Project_logs"""

from django.conf.urls import url
from . import views

urlpatterns = [

	url(r'^$', views.index, name='index'),
    url(r'^all-posts/$', views.contents, name='contents'),
    url(r'^post/(?P<slug>[-\w]+)/$', views.post, name='post'),
    url(r'^new-post/$', views.new_post, name='new_post'),
    url(r'^add-images/(?P<slug>[-\w]+)/$', views.ImageView.as_view(), name='add_images'),
]


from rest_framework import serializers
from datasets.models import DatasetLabel
from datasets.models.dataset import Dataset

class DatasetLabelSerializer(serializers.ModelSerializer):
    class Meta:
        model = DatasetLabel
        exclude = ['dataset']

    def create(self, validated_data):
        dataset_id = self.context.get('view').kwargs.get('dataset_pk')
        dataset = Dataset.objects.get(id=dataset_id)
        validated_data['dataset'] = dataset
        return super().create(validated_data)


from marvin.tools.maps import Maps
from marvin.tools.cube import Cube
import matplotlib.pyplot as plt

maps = Maps(plateifu='8714-12703')
print(maps)
# get an emission line map
haflux = maps.emline_gflux_ha_6564
values = haflux.value
ivar = haflux.ivar
mask = haflux.mask
haflux.plot()

cube = Cube(plateifu='8714-12703')
# get a spaxel by slicing cube[i,j]
spaxel = cube[16, 16]
flux = spaxel.flux
wave = flux.wavelength
ivar = flux.ivar
mask = flux.mask
flux.plot()

# plt.show()


"""
functions that analyze positions/values and identify appropriate moves
"""

from navigation import *
from itertools import groupby

valid = [1,2,3,4,5,6,7,8,9]

def find_possible_moves(moveset):
    """ finds any move(s) with value 0 in a single set of squares
        returns a list of potential moves, i.e. ((x,y), value)
    """
    values = [v for v in valid if v not in [m[1] for m in moveset if m[1] != 0]]
    return [(m[0], v) for m in moveset if m[1] == 0 for v in values]

    missing = [m for m in moveset if m[1] == 0]
    if len(missing) == 1:
        values = [v for v in valid if v not in [m[1] for m in moveset if m[1] != 0]][0]
        return [(missing[0][0], values)]

def find_impossible_moves(moveset):
    """ finds empty spaces and reports which values cannot be placed there
    """
    values = [v[1] for v in moveset if v[1] != 0]
    return [(v[0], values) for v in moveset if v[1] == 0]

def flatten_impossible_moves(movesets):
    """ movesets: a collection of items each representing the impossible moves
        (e.g. (3,3) can't be a 4,5, or 6) for each of the 9-item sets

    flattens these by unioning the impossible moves for each location.
    """
    g = groupby(sorted((m for ms in movesets for m in ms), key=lambda x: x[0]), lambda x: x[0])
    return [(x, set([val for yy in y for val in yy[1]])) for (x,y) in g]

def resolve_moves(possible_moves, impossible_moves=[]):
    """
    possible_moves: sets of potentially valid moves within each 9-item set (row/col/box)

    resolve which 'possible moves' are valid across those collections
    if there is only one possible move across collections then it is a necessary move
    if there is no possible move across collections then something is wrong
    if there are multiple possible moves, continue

    returns set of necessary moves
    """
    moves = []
    impossible = dict(impossible_moves)

    for place in places():
        # find matching moves from each 9-item set that apply to this 'place'; get the possible values from each (drop the place)
        sets = [set(m[1] for m in ms if place in m) for ms in possible_moves if any(place in m for m in ms)]

        if not any(sets):
            continue

        # find which values would be valid according to each 9-item sets' constraints
        valid_moves = sets[0].intersection(*sets)
        if place in impossible:
            valid_moves = valid_moves - set(impossible[place])

        if len(valid_moves) == 0:
            raise ValueError(place)
        elif len(valid_moves) == 1:
            moves.append((place, valid_moves.pop()))

    return moves


from gcsa.google_calendar import GoogleCalendar
from datetime import datetime
from tkinter import *
import webbrowser

root = Tk()
root.title("Google Calendar")
root.geometry("1000x700")
calendar = GoogleCalendar('yourgmail@gmail.com')

y = 10
for event in calendar:
    y += 1
    x = 0
    s = StringVar()
    s = event.start.strftime('%A')
    if datetime.now().strftime('%A') != s:
        break
    date = event.start.strftime('  %m/%d/%y')
    time = event.start.strftime("%I:%M%p")

    print(s, date, '-->', time, event.description)
    label_frame = LabelFrame(root, text=time)

    label_frame.pack(padx=10, pady=10)
    L1 = Label(label_frame, text=s)
    L1.grid(row=y, column=x, padx=10, pady=30)
    x += 10
    L2 = Label(label_frame, text=date)
    L2.grid(row=y, column=x, padx=10, pady=30)
    x += 10
    L3 = Label(label_frame, text=time)
    L3.grid(row=y, column=x, padx=10, pady=30)
    x += 10
    L4 = Label(label_frame, text=event.description)
    L4.grid(row=y, column=x, padx=10, pady=30)
    location = str(event.location)
    print(len(location))
    new = 1

    def OpenUrl(url):
        print(url)
        webbrowser.open(url, new=new)

    if len(location) != 4:
        Btn = Button(root, text="Open Google Maps", command=lambda aurl=location: OpenUrl(aurl)())
        Btn.pack()
root.mainloop()


import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split

from src.data.preprocess.hatespeech.preprocess_rdforest_hatespeech import (
    generate_tfidf_vectors,
)
from src.features.hatespeech.featureseng_helperfunc_hatespeech import (
    combine_feature_space,
)
from src.features.hatespeech.featureseng_rdforest_hatespeech import (
    contains_not_sexist_but,
)
from src.utils.preprocess_text_helpers import normalizer

DATA = pd.read_csv("../../../DATA/external/hatespeech/hs_data.csv")

DATA["normalized"] = normalizer(DATA["text"])
X = generate_tfidf_vectors(DATA["normalized"].to_list())["vectors"]

DATA["contains_not_sexist_but"] = DATA["text"].apply(contains_not_sexist_but)
X = combine_feature_space(X, DATA["contains_not_sexist_but"].values)

Y = pd.get_dummies(DATA["annotation"])["misogynistic"]
X_TRAIN, X_TEST, Y_TRAIN, Y_TEST = train_test_split(X, Y, test_size=0.3)

CLF = RandomForestClassifier(
    bootstrap=True,
    class_weight=None,
    criterion="gini",
    max_depth=None,
    max_features="auto",
    max_leaf_nodes=None,
    min_impurity_decrease=0.0,
    min_impurity_split=None,
    min_samples_leaf=1,
    min_samples_split=2,
    min_weight_fraction_leaf=0.0,
    n_estimators=100,
    n_jobs=1,
    oob_score=False,
    random_state=None,
    verbose=0,
    warm_start=False,
)

CLF.fit(X_TRAIN, Y_TRAIN)

FEATURE_NAMES = generate_tfidf_vectors(DATA["normalized"].to_list())[
    "FEATURE_NAMES"
] + ["contains_not_sexist_but"]
FEATURE_IMPORTANCE = pd.Series(
    CLF.feature_importances_, index=FEATURE_NAMES
).sort_values(ascending=False)

Y_PRED = CLF.predict(X_TEST)

# Model Accuracy, how often is the classifier correct?
print("Accuracy:", metrics.accuracy_score(Y_TEST, Y_PRED))

# Creating a bar plot
sns.barplot(x=FEATURE_IMPORTANCE[0:10], y=FEATURE_IMPORTANCE.index[0:10])
# Add labels to your graph
plt.xlabel("Feature Importance Score")
plt.ylabel("Features")
plt.title("Visualizing Important Features")
plt.legend()
plt.show()


import data_preparation_functions as prepfunctions
import mlxtend.frequent_patterns as pm
import matplotlib.pyplot as plt
import ds_functions as ds
import pandas as pd
import os

graphsDir = './Results/Pattern Mining/'
if not os.path.exists(graphsDir):
    os.makedirs(graphsDir)

bin_strategies = ['Uniform', 'Quantile', 'Kmeans']
n_bins = [3, 5, 10]

values = {
    'with 3 bins': [18200, 20500, 20000],
    'with 5 bins': [18200, 12000, 18000],
    'with 10 bins': [12500, 5400, 10500],
}

plt.figure(figsize=(7,7))
ds.multiple_bar_chart(bin_strategies, values, ylabel='Number of Patterns')
plt.suptitle('Number of Patterns for a support of 0.01')
plt.savefig(graphsDir + 'HFCR Number of Patterns')

values = {
    'with 3 bins': [30, 59, 29],
    'with 5 bins': [45, 81, 40],
    'with 10 bins': [52, 50, 40],
}

plt.figure(figsize=(7,7))
ds.multiple_bar_chart(bin_strategies, values, ylabel='Lift Score')
plt.suptitle('Lift of the top 10% for a support of 0.01')
plt.savefig(graphsDir + 'HFCR Lift Score')

bin_strategies = ['0.25', '0.57']


from django.shortcuts import render,redirect

from django.http import JsonResponse

# Create your views here.

import json

# 導入 model
from .models import *

# 用戶查詢,訂單資料
def api_query_user_order_view(request):
    if request.method == 'POST':
        if 'user_name' in request.session and 'uid' in request.session:
            uid = request.session['uid'][0]
            user_order_number = request.POST['order_number']
            user_order_date_begin = request.POST['order_date_begin']
            user_order_date_end = request.POST['order_date_end']

            #  根據前端發送的值做判斷查詢(如果沒有 user_order_number ,則直接查詢日期的區間)
            if user_order_number == '':
                #  1.多筆查詢
                user_order_msg = Order_info.objects.filter(user_id=uid, order_time__range=(user_order_date_begin, user_order_date_end)).order_by('-id')
            else:
                #  2.查詢單筆
                # 如果有 user_order_number 則只要查詢 訂單編號即可
                user_order_msg = Order_info.objects.filter(user_id=uid, order_number=user_order_number)

            # 組數據給前端設置頁面
            user_order_msg_dict = {
                'user_order_status': 0,
            }

            # 設置 user_order_msg_list 來組 訂單數據格式
            user_order_msg_list = []

            # 如果 user_order_msg 存在,則表示有數據
            if user_order_msg:
                #  將 user_order_msg 狀態 改為 1
                user_order_msg_dict['user_order_status'] = 1

                for i in user_order_msg:
                    user_order_info_dic = {
                        'recipient_name': i.recipient_name,
                        'recipient_phone_number': i.recipient_phone_number,
                        'recipient_address': i.recipient_address,
                        'order_total_price': i.order_total_price,
                        'order_number': i.order_number,
                        'order_time': i.order_time,
                        'order_status': i.order_status,
                        'order_check_time': i.order_check_time,
                        'order_pay_chose_pay': i.pay.chose_pay,
                        'order_pay_send_price': i.pay.send_price,
                        'user_order_info_detail': [],
                    }
                    for j in i.order_info_detail_set.filter(orders_id=i.id):
                        user_order_info_dic['user_order_info_detail'].append({
                            'goods_id': j.goods_id,
                            'goods_name': j.goods.title,
                            'goods_img_url': j.goods_img_url,
                            'goods_color': j.goods_color,
                            'goods_color_img': str(Goods_color.objects.filter(goods_id=j.goods_id, goods_color=j.goods_color)[0].goods_color_img),
                            'goods_size': j.goods_size,
                            'goods_amount': j.goods_amount,
                            'goods_price': j.goods_price,
                            'goods_singo_total_price': j.goods_total_price,
                        })

                    user_order_msg_list.append(user_order_info_dic)

                user_order_msg_dict['user_order_info_msg'] = user_order_msg_list
            return JsonResponse(user_order_msg_dict)

        else:
            return redirect('/')
    else:
        return redirect('/')


import math
import random

# import gym
import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.distributions import Normal
import pickle, argparse

from IPython.display import clear_output
import matplotlib.pyplot as plt
from matplotlib import animation
from IPython.display import display
import IPython
from custom_env_w import Point

use_cuda = torch.cuda.is_available()
device   = torch.device("cuda" if use_cuda else "cpu")

parser = argparse.ArgumentParser()
parser.add_argument('--flname', type=str, default='policy_net_cnstr_')
parser.add_argument('--pen_1', type=float, default=-10)
parser.add_argument('--pen_2', type=float, default=-20)
args = parser.parse_args()

class ReplayBuffer:

	def __init__(self, capacity):
		self.capacity = capacity
		self.buffer = []
		self.position = 0

	def push(self, state, action, reward, next_state, done):

		if len(self.buffer) < self.capacity:
			self.buffer.append(None)
		self.buffer[self.position] = (state, action, reward, next_state, done)
		self.position = (self.position + 1) % self.capacity

	def sample(self, batch_size):

		batch = random.sample(self.buffer, batch_size)
		state, action, reward, next_state, done = map(np.stack, zip(*batch))
		return state, action, reward, next_state, done

	def __len__(self):
		return len(self.buffer)

def plot(frame_idx, rewards):

	clear_output(True)
	plt.figure(figsize=(20,5))
	plt.subplot(131)
	plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))
	plt.plot(rewards)
	plt.show()

class ValueNetwork(nn.Module):

	def __init__(self, state_dim, hidden_dim, init_w=3e-3):
		super(ValueNetwork, self).__init__()

		self.linear1 = nn.Linear(state_dim, hidden_dim)
		self.linear2 = nn.Linear(hidden_dim, hidden_dim)
		self.linear3 = nn.Linear(hidden_dim, 1)

		self.linear3.weight.data.uniform_(-init_w, init_w)
		self.linear3.bias.data.uniform_(-init_w, init_w)

	def forward(self, state):
		x = F.relu(self.linear1(state))
		x = F.relu(self.linear2(x))
		x = self.linear3(x)
		return x

class SoftQNetwork(nn.Module):

	def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):
		super(SoftQNetwork, self).__init__()

		self.linear1 = nn.Linear(num_inputs + num_actions, hidden_size)
		self.linear2 = nn.Linear(hidden_size, hidden_size)
		self.linear3 = nn.Linear(hidden_size, 1)

		self.linear3.weight.data.uniform_(-init_w, init_w)
		self.linear3.bias.data.uniform_(-init_w, init_w)

	def forward(self, state, action):
		x = torch.cat([state, action], 1)
		x = F.relu(self.linear1(x))
		x = F.relu(self.linear2(x))
		x = self.linear3(x)
		return x

class PolicyNetwork(nn.Module):

	def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3, log_std_min=-20, log_std_max=2):
		super(PolicyNetwork, self).__init__()

		self.log_std_min = log_std_min
		self.log_std_max = log_std_max

		self.linear1 = nn.Linear(num_inputs, hidden_size)
		self.linear2 = nn.Linear(hidden_size, hidden_size)

		self.mean_linear = nn.Linear(hidden_size, num_actions)
		self.mean_linear.weight.data.uniform_(-init_w, init_w)
		self.mean_linear.bias.data.uniform_(-init_w, init_w)

		self.log_std_linear = nn.Linear(hidden_size, num_actions)
		self.log_std_linear.weight.data.uniform_(-init_w, init_w)
		self.log_std_linear.bias.data.uniform_(-init_w, init_w)

	def forward(self, state):

		x = F.relu(self.linear1(state))
		x = F.relu(self.linear2(x))

		mean    = self.mean_linear(x)
		log_std = self.log_std_linear(x)
		log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)

		return mean, log_std

	def evaluate(self, state, epsilon=1e-6):

		mean, log_std = self.forward(state)
		std = log_std.exp()

		normal = Normal(0, 1)
		z      = normal.sample()
		action = torch.tanh(mean+ std*z.to(device))
		log_prob = Normal(mean, std).log_prob(mean+ std*z.to(device)) - torch.log(1 - action.pow(2) + epsilon)
		return action, log_prob, z, mean, log_std

	def get_action(self, state):

		state = torch.FloatTensor(state).unsqueeze(0).to(device)
		mean, log_std = self.forward(state)
		std = log_std.exp()

		normal = Normal(0, 1)
		z      = normal.sample().to(device)
		action = torch.tanh(mean + std*z)

		action  = action.cpu()#.detach().cpu().numpy()
		return action[0]

def update(batch_size,gamma=0.99,soft_tau=1e-2):

	state, action, reward, next_state, done = replay_buffer.sample(batch_size)

	state      = torch.FloatTensor(state).to(device)
	next_state = torch.FloatTensor(next_state).to(device)
	action     = torch.FloatTensor(action).to(device)
	reward     = torch.FloatTensor(reward).unsqueeze(1).to(device)
	done       = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(device)

	predicted_q_value1 = soft_q_net1(state, action)
	predicted_q_value2 = soft_q_net2(state, action)
	predicted_value    = value_net(state)
	new_action, log_prob, epsilon, mean, log_std = policy_net.evaluate(state)

# Training Q Function
	target_value = target_value_net(next_state)
	target_q_value = reward + (1 - done) * gamma * target_value
	q_value_loss1 = soft_q_criterion1(predicted_q_value1, target_q_value.detach())
	q_value_loss2 = soft_q_criterion2(predicted_q_value2, target_q_value.detach())

	soft_q_optimizer1.zero_grad()
	q_value_loss1.backward()
	soft_q_optimizer1.step()
	soft_q_optimizer2.zero_grad()
	q_value_loss2.backward()
	soft_q_optimizer2.step()
# Training Value Function
	predicted_new_q_value = torch.min(soft_q_net1(state, new_action),soft_q_net2(state, new_action))
	# IPython.embed()
	target_value_func = predicted_new_q_value - log_prob
	value_loss = value_criterion(predicted_value, target_value_func.detach())

	value_optimizer.zero_grad()
	value_loss.backward()
	value_optimizer.step()
# Training Policy Function
	policy_loss = (log_prob - predicted_new_q_value).mean()

	policy_optimizer.zero_grad()
	policy_loss.backward()
	policy_optimizer.step()

	for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):
		target_param.data.copy_(
			target_param.data * (1.0 - soft_tau) + param.data * soft_tau
		)

pen_1 = args.pen_1
pen_2 = args.pen_2
fl_name= args.flname
env = Point('name', 20, -1, 20, -1, np.array([20.0,20.0]), np.array([5.5,14.5]), np.array([14.5,5.5]), pen_1, pen_2)#gym.make(args.env_name)
action_dim = 2#env.action_space.shape[0]
state_dim  = 2#env.observation_space.shape[0]
hidden_dim = 256

value_net        = ValueNetwork(state_dim, hidden_dim).to(device)
target_value_net = ValueNetwork(state_dim, hidden_dim).to(device)

soft_q_net1 = SoftQNetwork(state_dim, action_dim, hidden_dim).to(device)
soft_q_net2 = SoftQNetwork(state_dim, action_dim, hidden_dim).to(device)
policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim).to(device)

for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):
	target_param.data.copy_(param.data)

value_criterion  = nn.MSELoss()
soft_q_criterion1 = nn.MSELoss()
soft_q_criterion2 = nn.MSELoss()

value_lr  = 3e-4
soft_q_lr = 3e-4
policy_lr = 3e-4

value_optimizer  = optim.Adam(value_net.parameters(), lr=value_lr)
soft_q_optimizer1 = optim.Adam(soft_q_net1.parameters(), lr=soft_q_lr)
soft_q_optimizer2 = optim.Adam(soft_q_net2.parameters(), lr=soft_q_lr)
policy_optimizer = optim.Adam(policy_net.parameters(), lr=policy_lr)

replay_buffer_size = 100000
replay_buffer = ReplayBuffer(replay_buffer_size)

max_frames  = 200000
max_steps   = 500
frame_idx   = 0
rewards     = []
batch_size  = 128
cnt=0
while frame_idx < max_frames:
	episode_reward = 0

	state = np.zeros(2)
	# starting state
	if np.random.rand()<0.5:
		state[0] = 20*np.random.rand()
		state[1] = 1*np.random.rand()
	else:
		state[0] = 1*np.random.rand()
		state[1] = 20*np.random.rand()

	env.set_position(state[0], state[1])
	for step in range(max_steps):
		if frame_idx >20000:
			action = policy_net.get_action(state).detach()
			action=np.clip(action,-1, 1)
			next_state, reward, done, _ = env.step(action.numpy())
		else:
			# explore
			action = np.random.rand(2)
			action=np.clip(action,-1, 1)
			next_state, reward, done, _ = env.step(action)

		replay_buffer.push(state, action, reward, next_state, done)

		state = next_state
		episode_reward += reward
		frame_idx += 1

		if len(replay_buffer) > batch_size:
			update(batch_size)

		if frame_idx % 1000 == 0:
			print("In Frame {} and episode {} reward is {}".format(frame_idx, step, episode_reward))

		if done:
			print(frame_idx,'Done')
			break

	rewards.append(episode_reward)

torch.save(policy_net.state_dict(), 'model/' + fl_name + str(pen_1) + '_' + str(pen_2) + '.pth')

IPython.embed()

def display_frames_as_gif(frames):

	"""
	Displays a list of frames as a gif, with controls
	"""

	patch = plt.imshow(frames[0])
	plt.axis('off')

	def animate(i):
		patch.set_data(frames[i])

	anim = animation.FuncAnimation(plt.gcf(), animate, frames = len(frames), interval=50)
	display(anim)


import sys
import nltk
nltk.download(['punkt', 'wordnet'])
import re
import numpy as np
import pandas as pd
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.tree import DecisionTreeClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report
from sqlalchemy import create_engine
import pickle

def load_data(database_filepath):
    """Load the file and return the X,Y data"""
    name = 'sqlite:///' + database_filepath
    engine = create_engine(name)
    df = pd.read_sql('select * from InsertTableName', engine)
    X = df['message']
    Y = df.drop(columns=['message','original','genre','id'])
    category_names = Y.columns

    return X,Y, category_names

def tokenize(text):
    """Transform the input text by tokenization and cleaing"""
    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens

def build_model():
    """Builds Pipeline with Grid Search model and classifier"""

    pipeline = Pipeline([
    ('vect', CountVectorizer(tokenizer=tokenize)),
    ('tfidf', TfidfTransformer()),
    ('clf', MultiOutputClassifier(RandomForestClassifier())),
    ])

    parameters = {'clf__estimator__max_depth': [10, 50, None],
              'clf__estimator__min_samples_leaf':[2, 5, 10]}

    cv = GridSearchCV(pipeline, param_grid=parameters)
    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    """Print the evaluation results of the model"""
    y_pred = model.predict(X_test)
    for i, col in enumerate(Y_test.columns):
        print (classification_report(Y_test[col], y_pred.astype(int)[:,i],
            target_names = category_names))

def save_model(model, model_filepath):
    """Save model"""
    pickle.dump(model, open(model_filepath, 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train) # if not working try X_train.as_matrix(),Y_train.as_matrix()

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()

"""Below is the model with the best parameters"""
"""pipeline = Pipeline([
    ('vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', MultiOutputClassifier(RandomForestClassifier(max_depth= 10, min_samples_leaf= 5))),
])
"""


from lxml import etree
import webob

from nova.api.openstack.compute.contrib import quota_classes
from nova.api.openstack import wsgi
from nova import test
from nova.tests.api.openstack import fakes

def quota_set(class_name):
    return {'quota_class_set': {'id': class_name, 'metadata_items': 128,
                                'ram': 51200, 'floating_ips': 10,
                                'fixed_ips': -1, 'instances': 10,
                                'injected_files': 5, 'cores': 20,
                                'injected_file_content_bytes': 10240,
                                'security_groups': 10,
                                'security_group_rules': 20, 'key_pairs': 100,
                                'injected_file_path_bytes': 255}}

class QuotaClassSetsTest(test.TestCase):

    def setUp(self):
        super(QuotaClassSetsTest, self).setUp()
        self.controller = quota_classes.QuotaClassSetsController()

    def test_format_quota_set(self):
        raw_quota_set = {
            'instances': 10,
            'cores': 20,
            'ram': 51200,
            'floating_ips': 10,
            'fixed_ips': -1,
            'metadata_items': 128,
            'injected_files': 5,
            'injected_file_path_bytes': 255,
            'injected_file_content_bytes': 10240,
            'security_groups': 10,
            'security_group_rules': 20,
            'key_pairs': 100,
            }

        quota_set = self.controller._format_quota_set('test_class',
                                                      raw_quota_set)
        qs = quota_set['quota_class_set']

        self.assertEqual(qs['id'], 'test_class')
        self.assertEqual(qs['instances'], 10)
        self.assertEqual(qs['cores'], 20)
        self.assertEqual(qs['ram'], 51200)
        self.assertEqual(qs['floating_ips'], 10)
        self.assertEqual(qs['fixed_ips'], -1)
        self.assertEqual(qs['metadata_items'], 128)
        self.assertEqual(qs['injected_files'], 5)
        self.assertEqual(qs['injected_file_path_bytes'], 255)
        self.assertEqual(qs['injected_file_content_bytes'], 10240)
        self.assertEqual(qs['security_groups'], 10)
        self.assertEqual(qs['security_group_rules'], 20)
        self.assertEqual(qs['key_pairs'], 100)

    def test_quotas_show_as_admin(self):
        req = fakes.HTTPRequest.blank(
            '/v2/fake4/os-quota-class-sets/test_class',
            use_admin_context=True)
        res_dict = self.controller.show(req, 'test_class')

        self.assertEqual(res_dict, quota_set('test_class'))

    def test_quotas_show_as_unauthorized_user(self):
        req = fakes.HTTPRequest.blank(
            '/v2/fake4/os-quota-class-sets/test_class')
        self.assertRaises(webob.exc.HTTPForbidden, self.controller.show,
                          req, 'test_class')

    def test_quotas_update_as_admin(self):
        body = {'quota_class_set': {'instances': 50, 'cores': 50,
                                    'ram': 51200, 'floating_ips': 10,
                                    'fixed_ips': -1, 'metadata_items': 128,
                                    'injected_files': 5,
                                    'injected_file_content_bytes': 10240,
                                    'injected_file_path_bytes': 255,
                                    'security_groups': 10,
                                    'security_group_rules': 20,
                                    'key_pairs': 100}}

        req = fakes.HTTPRequest.blank(
            '/v2/fake4/os-quota-class-sets/test_class',
            use_admin_context=True)
        res_dict = self.controller.update(req, 'test_class', body)

        self.assertEqual(res_dict, body)

    def test_quotas_update_as_user(self):
        body = {'quota_class_set': {'instances': 50, 'cores': 50,
                                    'ram': 51200, 'floating_ips': 10,
                                    'fixed_ips': -1, 'metadata_items': 128,
                                    'injected_files': 5,
                                    'injected_file_content_bytes': 10240,
                                    'security_groups': 10,
                                    'security_group_rules': 20,
                                    'key_pairs': 100,
                                    }}

        req = fakes.HTTPRequest.blank(
            '/v2/fake4/os-quota-class-sets/test_class')
        self.assertRaises(webob.exc.HTTPForbidden, self.controller.update,
                          req, 'test_class', body)

class QuotaTemplateXMLSerializerTest(test.TestCase):
    def setUp(self):
        super(QuotaTemplateXMLSerializerTest, self).setUp()
        self.serializer = quota_classes.QuotaClassTemplate()
        self.deserializer = wsgi.XMLDeserializer()

    def test_serializer(self):
        exemplar = dict(quota_class_set=dict(
                id='test_class',
                metadata_items=10,
                injected_file_path_bytes=255,
                injected_file_content_bytes=20,
                ram=50,
                floating_ips=60,
                fixed_ips=-1,
                instances=70,
                injected_files=80,
                security_groups=10,
                security_group_rules=20,
                key_pairs=100,
                cores=90))
        text = self.serializer.serialize(exemplar)

        tree = etree.fromstring(text)

        self.assertEqual('quota_class_set', tree.tag)
        self.assertEqual('test_class', tree.get('id'))
        self.assertEqual(len(exemplar['quota_class_set']) - 1, len(tree))
        for child in tree:
            self.assertTrue(child.tag in exemplar['quota_class_set'])
            self.assertEqual(int(child.text),
                             exemplar['quota_class_set'][child.tag])

    def test_deserializer(self):
        exemplar = dict(quota_class_set=dict(
                metadata_items='10',
                injected_file_content_bytes='20',
                ram='50',
                floating_ips='60',
                fixed_ips='-1',
                instances='70',
                injected_files='80',
                security_groups='10',
                security_group_rules='20',
                key_pairs='100',
                cores='90'))
        intext = ("<?xml version='1.0' encoding='UTF-8'?>\n"
                  '<quota_class_set>'
                  '<metadata_items>10</metadata_items>'
                  '<injected_file_content_bytes>20'
                  '</injected_file_content_bytes>'
                  '<ram>50</ram>'
                  '<floating_ips>60</floating_ips>'
                  '<fixed_ips>-1</fixed_ips>'
                  '<instances>70</instances>'
                  '<injected_files>80</injected_files>'
                  '<cores>90</cores>'
                  '<security_groups>10</security_groups>'
                  '<security_group_rules>20</security_group_rules>'
                  '<key_pairs>100</key_pairs>'
                  '</quota_class_set>')

        result = self.deserializer.deserialize(intext)['body']
        self.assertEqual(result, exemplar)


import lreg
import numpy as np
import time
import matplotlib.pyplot as plt
plt.style.use("dark_background")

def naive_pure_python(x,y):
    x = x.reshape(-1,1)
    y = y.reshape(-1,1)
    ones = np.ones(x.shape)
    A = np.hstack([x, ones])
    At = A.T
    AtA = At@A
    AtA_1 = np.linalg.inv(AtA)
    b = AtA_1@(At.dot(y))
    return b

def test_linear_reg(fun, x, y):
    t = time.time()
    b = fun(x,y)
    took = time.time()-t
    print(f"{fun.__name__} took {took}")
    b_str = np.array2string(b.reshape(-1), precision=5)
    print(f"{fun.__name__} fit result: {b_str}")
    return took

def test_linalg_solv(x,y):
    t = time.time()
    x = x.reshape(-1,1)
    y = y.reshape(-1,1)
    ones = np.ones(x.shape)
    A = np.hstack([x, ones])
    np.linalg.solve(A.T@A,A.T@y)
    took = time.time()-t
    print(f"np.linalg.solve took {took}")
    return took

def main():
    naive = lreg.naive

    a=3
    b=1.2
    NUM_POINTS = int(1e7)

    x = np.linspace(0,1,NUM_POINTS)
    y = a*x+ b + np.random.normal(size=NUM_POINTS)
    x = x.reshape(1,-1)
    y = y.reshape(1,-1)

    lreg_dt = test_linear_reg(naive, x.reshape(-1), y.reshape(-1))
    pp_dt = test_linear_reg(naive_pure_python, x, y)
    test_linalg_solv(x,y)

    results = []
    t = np.linspace(10,int(1e6), 100)
    for NUMPOINTS in t :
        x = np.linspace(0,1,int(NUMPOINTS))
        y = a*x+ b + np.random.normal(size=int(NUMPOINTS))
        x = x.reshape(1,-1)
        y = y.reshape(1,-1)

        lreg_dt = test_linear_reg(naive, x.reshape(-1), y.reshape(-1))
        pp_dt = test_linear_reg(naive_pure_python, x, y)
        np_ls = test_linalg_solv(x,y)
        results.append([lreg_dt, pp_dt, np_ls])

    results = np.array(results)
    plt.plot(t, results[:,0], label="naive")
    plt.plot(t, results[:,1], label="naive pure python")
    plt.plot(t, results[:,2], label="np.linalg.solve")
    plt.legend()
    plt.show()

if __name__ == "__main__":
    main()


import logging, math, torch
import torch.optim as optim
from torch import optim
from torch.optim.lr_scheduler import LambdaLR

logger = logging.getLogger(__name__)

class ConstantLRSchedule(LambdaLR):
    """ Constant learning rate schedule.
    """
    def __init__(self, optimizer, last_epoch=-1):
        super(ConstantLRSchedule, self).__init__(optimizer, lambda _: 1.0, last_epoch=last_epoch)

class WarmupConstantSchedule(LambdaLR):
    """ Linear warmup and then constant.
        Linearly increases learning rate schedule from 0 to 1 over `warmup_steps` training steps.
        Keeps learning rate schedule equal to 1. after warmup_steps.
    """
    def __init__(self, optimizer, warmup_steps, last_epoch=-1):
        self.warmup_steps = warmup_steps
        super(WarmupConstantSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1.0, self.warmup_steps))
        return 1.

class WarmupLinearSchedule(LambdaLR):
    """ Linear warmup and then linear decay.
        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
        Linearly decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps.
    """

    def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1):
        self.warmup_steps = warmup_steps
        self.t_total = t_total
        super(WarmupLinearSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1, self.warmup_steps))
        return max(0.0, float(self.t_total - step) / float(max(1.0, self.t_total - self.warmup_steps)))

class WarmupCosineSchedule(LambdaLR):
    """ Linear warmup and then cosine decay.
        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
        Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.
        If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup.
    """
    def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):
        self.warmup_steps = warmup_steps
        self.t_total = t_total
        self.cycles = cycles
        super(WarmupCosineSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1.0, self.warmup_steps))
        # progress after warmup
        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))
        return max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress)))

class WarmupCosineWithHardRestartsSchedule(LambdaLR):
    """ Linear warmup and then cosine cycles with hard restarts.
        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.
        If `cycles` (default=1.) is different from default, learning rate follows `cycles` times a cosine decaying
        learning rate (with hard restarts).
    """

    def __init__(self, optimizer, warmup_steps, t_total, cycles=1., last_epoch=-1):
        self.warmup_steps = warmup_steps
        self.t_total = t_total
        self.cycles = cycles
        super(WarmupCosineWithHardRestartsSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)

    def lr_lambda(self, step):
        if step < self.warmup_steps:
            return float(step) / float(max(1, self.warmup_steps))
        # progress after warmup
        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))
        if progress >= 1.0:
            return 0.0
        return max(0.0, 0.5 * (1. + math.cos(math.pi * ((float(self.cycles) * progress) % 1.0))))

def defineLRScheduler(args, optimizer, length_train_dataloader):
    warmup_steps = int(length_train_dataloader * 0.15)

    # 파이토치 기본제공 learning rate scheduler
    if args.lr_scheduler.lower() == 'steplr':
        scheduler = optim.plr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
    elif args.lr_scheduler.lower() == 'multisteplr':
        '''
        - milestones : 해당 epoch마다 lr을 *gamma배만큼 decay
        - gamma : Multiplicative factor of learning rate decay. Default: 0.1.
        '''
        scheduler = optim.plr_scheduler.MultiStepLR(optimizer=optimizer, milestones=[2, 5, 10, 15, 30], gamma=0.5)
    elif args.lr_scheduler.lower() == 'reducelronplateau':
        scheduler = optim.plr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='min', factor=0.5, patience=3, )
    elif args.lr_scheduler.lower() == 'cosineschedule':
        scheduler = optim.plr_scheduler.CosineAnnealingLR(optimizer=optimizer, T_max=10, eta_min=0)

    # 파이토치 기본제공 learning rate의 수정버전
    elif args.lr_scheduler.lower() == 'warmupcosineschedule':
        # warmup_steps동안 0~1까지 warmup 후 1~0까지 t_total-warmup_steps의 step동안 cosine을 그리며 decay
        scheduler = WarmupCosineSchedule(optimizer=optimizer, warmup_steps=warmup_steps, t_total=length_train_dataloader, cycles=0.5, last_epoch=-1)
        print('Warm-up during 0 to ', warmup_steps, 'steps, Cosine decay during ', warmup_steps, ' to ', length_train_dataloader, 'steps.')
    elif args.scheduler == 'WarmupCosineWithHardRestartsSchedule':
        scheduler = WarmupCosineWithHardRestartsSchedule(optimizer=optimizer, warmup_steps=warmup_steps, t_total=length_train_dataloader, cycles=1.0, last_epoch=-1)
    elif args.scheduler == 'WarmupLinearSchedule': # Constant learning rate scheduling.
        scheduler = ConstantLRSchedule(optimizer, last_epoch=-1)
    else:
        print("Invalid args input.")
        raise ValueError('Invalid scheduler')

    print('optimizer : ', type(optimizer), ' lr scheduler : ', type(scheduler))
    return scheduler


import numpy as np

from numba import jit

def fdconv(y1,y2,yfd1,yfd2):
    """Perform the convolution with the Fermi Dirac function"""
    n = len(y1) # number of points
    nc = (n-1)//2 # central site
    out = np.array([0.0 for i in range(n)])
    return fdconv_jit(y1,y2,yfd1,yfd2,out)

@jit(nopython=True)
def fdconv_jit(y1,y2,yfd1,yfd2,out):
    n = len(y1) # number of points
    nc = (n-1)//2 # central site
    for i in range(n):
      for j in range(-nc,2*n-nc):
          wp = j-i+nc # distance to zero frequency
          if wp<0: yi = y2[0] ; di = yfd2[0]
          elif wp>=n: yi = y2[n-1] ; di = yfd2[n-1]
          else: yi = y2[wp] ; di = yfd2[wp]
          if j>=n: yj = y1[n-1] ; dj = yfd1[n-1]
          elif j<0: yj = y1[0] ; dj = yfd1[0]
          else: yj = y1[j] ; dj = yfd1[j]
          out[i] = out[i] + yj*yi*(-dj+di)
    return out

@jit
def conv(y1,y2):
    """Perform the convolution with the Fermi Dirac function"""
    n = len(y1) # number of points
    nc = (n-1)//2 # central site
    out = [0.0 for i in range(n)]
    for i in range(n):
      for j in range(-nc,2*n-nc):
          wp = j-i+nc # distance to zero frequency
#          if not 0<=wp<n: continue
#          if not 0<=(j+wp)<n: continue
          if wp<0: yi = y2[0]
          elif wp>=n: yi = y2[n-1]
          else: yi = y2[wp]
          if j>=n: yj = y1[n-1]
          elif j<0: yj = y1[0]
          else: yj = y1[j]
          out[i] = out[i] + yj*yi
    return np.array(out)

def fdconvolution(x,y1,y2,fd1=None,fd2=None):
    """Fermi Dirac convolution"""
    if fd1 is None and fd2 is None: # no function given
        return np.convolve(y1,y2,mode="same")/len(y1)
        f1 = interpolate(x,y1) # interpolate
        f2 = interpolate(x,y2) # interpolate
        dx = max(x) - min(x)
        xs = np.linspace(min(x)-dx,max(x)+dx,len(x)*20) # as many points
        y3 = np.convolve(f1(xs),f2(xs),mode="same")
        return interpolate(xs,y3)(x)
    else: # assume that there is a Fermi Dirac distribution as input
        if callable(fd1): fd1x = fd1(x) # call function
        else: fd1x = fd1 # assume it is an array
        if callable(fd2): fd2x = fd2(x) # call function
        else: fd2x = fd2 # assume it is an array
        return fdconv(y1,y2,fd1x,fd2x)/len(x)

def plain_convolution(x,y1,y2):
    dx = (np.max(x) - np.min(x))/len(x)
    return np.convolve(y1,y2,mode="same")*dx


MAPS_API_KEY = 'your_key'
MAPS_API_URL = 'https://maps.googleapis.com/maps/api/place/nearbysearch/json'
from datetime import datetime
import time
import asyncio
import argparse
import aiohttp
import json

HOST = '127.0.0.1'
PORTS = {
    'Bernard': 12311,
    'Clark': 12312,
    'Jaquez': 12313,
    'Johnson': 12314,
    'Juzang': 12315
}
COMM = {
    'Bernard': ['Jaquez', 'Johnson', 'Juzang'],
    'Clark' : ['Jaquez', 'Juzang'],
    'Jaquez' : ['Bernard', 'Clark'],
    'Johnson': ['Bernard', 'Juzang'],
    'Juzang': ['Bernard', 'Clark', 'Johnson']
}

# Print to log file.
def print_log(msg):
    now = datetime.now()
    time_str = now.strftime("%m-%d-%Y %H:%M:%S")
    log.write("[ " + time_str + " ] " + arg.ID + ": " + msg + "\n")
    log.flush()

# Class that define how servers communicate with clients and other servers within the herd.
class ServerProtocol(asyncio.Protocol):
    # Dict to store client recent info
    clients_recent_loc = {}

    # Constructor
    def __init__(self, id, port):
        self.server_name = id
        self.port = port

    # Handle connection to client
    async def handle_connection(self, reader, writer):
        encoded_req = await reader.readline()
        recieved_time = time.time()
        decoded_req = encoded_req.decode()
        #If there's a request sent from client
        if (decoded_req.strip()):
            print_log(f"Request received - {decoded_req.strip()}")
            req = decoded_req.strip().split()
            at_req = decoded_req.strip()
            res = await self.handle_req(req, recieved_time, at_req)
            if res != None:
                if req[0] != "AT":
                    writer.write(res.encode())
                    await writer.drain()
                writer.close()
            else:
                writer.close()
        else:
            writer.close()

    # Generate response for IAMAT command
    async def handle_IAMAT(self, req, recieved_time):
        client_id = req[1]
        client_coordinate = req[2]
        client_time = req[3]
        time_diff = str(float(recieved_time) - float(client_time))
        if (time_diff[0] != '-'):
            time_diff = f"+{time_diff}"
        # Store rec loc for WHATSAT command
        ServerProtocol.clients_recent_loc[client_id] = [client_coordinate, client_time, time_diff, self.server_name]
        # response to IAMAT
        IAMAT_res = f"AT {self.server_name} {time_diff} {client_id} {client_coordinate} {client_time}\n"
        print_log(f"Response sent - {IAMAT_res.rstrip()}")
        # Flood and Propagate the server location to other servers
        await self.flood(IAMAT_res.strip())
        return IAMAT_res

    # Generate response for WHATSAT command
    async def handle_WHATSAT(self, req):
        client_id = req[1]
        rec_list = ServerProtocol.clients_recent_loc[req[1]]
        client_coordinate =  rec_list[0]
        client_time =  rec_list[1]
        time_diff =  rec_list[2]
        old_server =  rec_list[3]
        #convert rad to meters
        rad = str(int(float(req[2]) * 1000))
        up_bound = int(req[3])
        for i in range(1, len(client_coordinate)):
            if client_coordinate[i] in ["+", "-"]:
                lat = float(client_coordinate[:i])
                lng = float(client_coordinate[i:])
        QUERY_res = await self.query_places(lat, lng, rad, up_bound)
        # response to WHATSAT
        WHATSAT_res = f"AT {old_server} {time_diff} {client_id} {client_coordinate} {client_time}\n{QUERY_res}\n\n"
        print_log(f"Response sent - {WHATSAT_res.rstrip()}")
        return WHATSAT_res

    # Send Google HTTP request to GOOGLE PLACE using Google API
    # Inspiration: https://pypi.org/project/aiohttp/
    # Inspiration: https://www.w3schools.com/python/python_json.asp
    async def query_places(self, lat, lng, rad, up_bound):
        print_log(f"Querying Google Places For Nearby Locations")
        async with aiohttp.ClientSession() as session:
            gp_url=f"{MAPS_API_URL}?location={lat},{lng}&radius={rad}&key={MAPS_API_KEY}"
            async with session.get(gp_url) as response:
                query_res = await response.text()
            # Parse query_res
            obj = json.loads(query_res)
            # Limit search result
            obj["results"] = (obj["results"])[:up_bound]
            return json.dumps(obj, indent=4, sort_keys=True).rstrip()

    # Generate response for AT command
    async def handle_AT(self, req, at_req):
        # Parse AT Command
        main_server = req[1]
        time_diff = req[2]
        client_id = req[3]
        coordinate = req[4]
        client_time = float(req[5])
        # Check if client has already announced his or her location
        if client_id in ServerProtocol.clients_recent_loc.keys():
            # Update the location only if it is the most recent location by checking the time provided by the client
            if client_time > float(ServerProtocol.clients_recent_loc[client_id][1]):
                print_log("Updating existing client's location")
                ServerProtocol.clients_recent_loc[client_id] = [coordinate, client_time, time_diff, main_server]
                await self.flood(at_req)
            else:
                print_log(f"{client_id}'s location is up-to-date")
        else:
            print_log("Initialize new client's location")
            ServerProtocol.clients_recent_loc[client_id] = [coordinate, client_time, time_diff, main_server]
            await self.flood(at_req)

    # Flood the client's location to other running servers
    async def flood(self, at_req):
        client_id = (at_req.split())[3]
        for server in COMM[self.server_name]:
            try:
                reader, writer = await asyncio.open_connection(HOST, PORTS[server])
                print_log(f"Connected to {server}")
                writer.write(at_req.encode())
                await writer.drain()
                print_log(f"Updating {client_id}'s location to server {server}")
                print_log(f"{server} - {self.clients_recent_loc}")
                print_log(f"Connection Dropped with {server}")
                await writer.drain()
                writer.close()
            except:
                # Catch connection error exception if the server is not running
                print_log(f"Error: Cannot connect to server: {server}")

    # Handle resquests from client
    async def handle_req(self, req, recieved_time, at_req):
        # Validate IAMAT command
        if req[0] == "IAMAT":
            # Check if it is the command IAMAT
            if len(req) != 4:
                print_log("Error: IAMAT - Invalid arguments.")
                return
            # Check if the coordinate provided is in correct format
            coordinate = req[2]
            total_syms = (sum(map(lambda x: 1 if x in ['+', '-'] else 0, coordinate)))
            if total_syms == 2:
                # Extract the lng and lat from the given coord
                if coordinate[0] in ['+', '-']:
                    for i in range(1, len(coordinate)):
                        if coordinate[i] in ["+", "-"]:
                            # Check if lat and lng are valid
                            if abs(float(coordinate[:i])) > 90 or abs(float(coordinate[:i])) < -90:
                                print_log("Error: IAMAT - Latitude(-90 to +90) is out of range.")
                                return
                            if abs(float(coordinate[i:])) < -180 or abs(float(coordinate[i:])) > 180:
                                print_log("Error: IAMAT - Longitude(-180 to +180) is out of range.")
                                return
            else:
                print_log("Error: IAMAT -  Invalid coordination format.")
                return
            # Check if timestamp given is valid (float)
            try:
                isinstance(float(req[3]), float)
            except ValueError:
                print_log("Error: IAMAT - Time Stamp is not valid.")
                return
            return await self.handle_IAMAT(req, recieved_time)

        # Validate WHATSAT command
        elif req[0] == "WHATSAT":
            # Check if it is the query WHATSAT
            if len(req) != 4:
                print_log("Error: WHATSAT - Invalid arguments.")
                return
            # Check if the client ever announce his or her location
            try:
                ServerProtocol.clients_recent_loc[req[1]]
            except KeyError:
                print_log(f"Error: WHATSAT - {req[1]}'s location is unknown.")
                return
            try:
                rad = isinstance(float(req[2]), float)
                up_bound = isinstance(int(req[3]), int)
            except ValueError:
                print_log("Error: WHATSAT - Invalid Radius and Upper Bound Value.")
                return
            if float(req[2]) > 50 or float(req[2]) < 0:
                print_log("Error: WHATSAT - Radius provided is out of range.")
                return
            if int(req[3]) > 20 or int(req[3]) < 0:
                print_log("Error: WHATSAT - Upper Bound provided is out of range. ")
                return
            return await self.handle_WHATSAT(req)

        # Validate AT command
        # AT {SERVER_ID} {TIME_DIFF} {CLIENT_ID} {COORDINATE} {TIME_SENT}
        elif req[0] == "AT":
            # Check if it is the AT command
            if len(req) != 6:
                print_log("Error: AT - Invalid arguments.")
                return
            # Check if SERVER_ID is valid
            if req[1] not in PORTS.keys():
                print_log("Error: AT - Server_ID does not exist.")
                return
            # Check if the coordinate provided is in correct format
            coordinate = req[4]
            total_syms = (sum(map(lambda x: 1 if x in ['+', '-'] else 0, coordinate)))
            if total_syms == 2:
                # Extract the lng and lat from the given coord
                if coordinate[0] in ['+', '-']:
                    for i in range(1, len(coordinate)):
                        if coordinate[i] in ["+", "-"]:
                            # Check if lat and lng are valid
                            if abs(float(coordinate[:i])) > 90 or abs(float(coordinate[:i])) < -90:
                                print_log("Error: IAMAT - Latitude(-90 to +90) is out of range.")
                                return
                            if abs(float(coordinate[i:])) < -180 or abs(float(coordinate[i:])) > 180:
                                print_log("Error: IAMAT - Longitude(-180 to +180) is out of range.")
                                return
            # Check if timestamp given is valid
            try:
                isinstance(float(req[5]), float)
            except ValueError:
                print_log("Error: AT - Time Stamp is not valid.")
                return
            return await self.handle_AT(req, at_req)
        else:
            print_log(f"Error: \"{at_req.strip()}\" is an invalid command")
            return f"? {at_req}"

    # To start server
    async def serve(self):
        #starting server
        print_log("Server is starting...")
        server = await asyncio.start_server(self.handle_connection, HOST, self.port)
        #serving server
        async with server:
            await server.serve_forever()
        server.close()

if __name__ == "__main__":
    # Positional arg to parse for server name.
    parser = argparse.ArgumentParser()
    parser.add_argument("ID", type=str, help="Name of the server", choices=["Juzang", "Bernard", "Johnson", "Jaquez", "Clark"])
    arg = parser.parse_args()
    # Creating a log file
    log = open("./logs/" + arg.ID + "-server.log", mode='w')
    # Starting the server
    server = ServerProtocol(arg.ID, PORTS[arg.ID])
    try:
        asyncio.run(server.serve())
    except KeyboardInterrupt:
        print_log("Server is shutting down...")
        log.close()


import bpy, bmesh
from bpy import context as C
from mathutils import Vector

def newobj(bm, name):
    me = bpy.data.meshes.new(name)
    bm.to_mesh(me)
    ob = bpy.data.objects.new(name,me)
    C.scene.objects.link(ob)
    return ob

# https://blender.stackexchange.com/questions/32283/what-are-all-values-in-bound-box
def bounds(obj, local=False):
    local_coords = obj.bound_box[:]
    om = obj.matrix_world
    if not local:
        worldify = lambda p: om * Vector(p[:])
        coords = [worldify(p).to_tuple() for p in local_coords]
    else:
        coords = [p[:] for p in local_coords]
    rotated = zip(*coords[::-1])
    push_axis = []
    for (axis, _list) in zip('xyz', rotated):
        info = lambda: None
        info.max = max(_list)
        info.min = min(_list)
        info.distance = info.max - info.min
        push_axis.append(info)
    import collections
    originals = dict(zip(['x', 'y', 'z'], push_axis))
    o_details = collections.namedtuple('object_details', 'x y z')
    return o_details(**originals)

object_details = bounds(C.object, True)
bpy.ops.object.mode_set(mode='OBJECT')

cut_index = 0
step_size = 5

bisection_outer = bmesh.new()
bisection_outer.from_mesh(C.object.data)

# split the model into parts along y-axis, with steps of step_size in local coordinate space (this ignores the scale property, so make sure to apply it before)
for i in range(int(round(object_details.y.min)), int(round(object_details.y.max+step_size)), step_size):
        bisection_inner = bisection_outer.copy()
        bmesh.ops.bisect_plane(bisection_outer, geom=bisection_outer.verts[:]+bisection_outer.edges[:]+bisection_outer.faces[:], plane_co=(0,i,0), plane_no=(0,1,0), clear_inner=False)
        bmesh.ops.bisect_plane(bisection_inner, geom=bisection_inner.verts[:]+bisection_inner.edges[:]+bisection_inner.faces[:], plane_co=(0,i,0), plane_no=(0,1,0), clear_outer=True)
        newobj(bisection_inner, "bisect-"+str(cut_index))
        bisection_inner.free()  # free and prevent further access
        cut_index+=1

bisection_outer.free()  # free and prevent further access
bpy.data.objects.remove(C.object, True)


import math, random
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from sklearn import metrics
from sklearn import svm
from sklearn import tree
from sklearn import preprocessing
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import label_binarize
from sklearn.preprocessing import StandardScaler

from shapley.models.uci import *
from shapley.utils.shap_utils import *
from shapley.utils.utils import *

# remove new data to the train set and draw the picture
def plot_summarization(loo_pre_idx, tmc_pre_idx, g_values, knn_values, loo_knn_values, kmin, kmax, kinterval, x_train, y_train, x_test, y_test, HtoL=False):
    sns.set()
    plt.figure()
    # set the model deterministic
    seed = 0
    np.random.seed(seed)
    random.seed(seed)
    torch.manual_seed(seed)
    plt.xlabel('Fraction of train data removed (%)')
    plt.ylabel('Prediction accuracy (%)')

    tmc_accs = []
    knn_accs = [[] for _ in range(len(knn_values))]
    loo_knn_accs = [[] for _ in range(len(knn_values))]
    marks = ['o-', '*-', 's-', 'v-', '^-', '.-', '1-', '2-']
    colors = ['b', 'g', 'r', 'y', 'c', 'm', 'olive', 'pink']
    labels = [i for i in range(kmin, kmax, kinterval)]
    model = 'uci'
    count = int(len(x_train)/2)
    interval = int(count*0.02)
    x = np.arange(0, count, interval)/len(x_train)
    if(HtoL == True):
        print("removing data from Highest to Lowest!")
        knn_pre_idx = np.flip(knn_pre_idx, 1)
        tmc_pre_idx = np.flip(tmc_pre_idx, 0)
        loo_pre_idx = np.flip(loo_pre_idx, 0)
    else:
        print("removing data from Lowest to Highest!")
    # G Shapley
    print("G")
    g_accs = []
    idxs = np.argsort(g_values)
    keep_idxs = idxs.tolist()
    for j in range(0, count, interval):
        if len(keep_idxs) == len(x_train):
            x_train_keep, y_train_keep = x_train, y_train
        else:
            x_train_keep, y_train_keep = x_train[keep_idxs], y_train[keep_idxs]
        clf_g = return_model(model)
        clf_g.fit(x_train_keep, y_train_keep)
        acc = clf_g.score(x_test, y_test)
        print(len(keep_idxs), acc)
        g_accs.append(acc)
        keep_idxs = keep_idxs[interval:]
    print("G:", g_accs)
    plt.plot(x, g_accs, '-', label='G Shapley', color='olive')

    # TMC Shapley
    print("TMC")
    tmc_accs = []
    idxs = np.argsort(tmc_pre_idx)
    keep_idxs = idxs.tolist()
    for j in range(0, count, interval):
        if len(keep_idxs) == len(x_train):
            x_train_keep, y_train_keep = x_train, y_train
        else:
            x_train_keep, y_train_keep = x_train[keep_idxs], y_train[keep_idxs]
        clf_tmc = return_model(model)
        clf_tmc.fit(x_train_keep, y_train_keep)
        acc = clf_tmc.score(x_test, y_test)
        print(len(keep_idxs), acc)
        tmc_accs.append(acc)
        keep_idxs = keep_idxs[interval:]
    print("TMC:", tmc_accs)
    plt.plot(x, tmc_accs, '-', label='TMC Shapley', color='olive')

    # Loo Shapley
    print("Loo")
    loo_accs = []
    idxs = np.argsort(loo_pre_idx)
    keep_idxs = idxs.tolist()
    for j in range(0, count, interval):
        if len(keep_idxs) == len(x_train):
            x_train_keep, y_train_keep = x_train, y_train
        else:
            x_train_keep, y_train_keep = x_train[keep_idxs], y_train[keep_idxs]
        clf_loo = return_model(model)
        clf_loo.fit(x_train_keep, y_train_keep)
        acc = clf_loo.score(x_test, y_test)
        print(len(keep_idxs), acc)
        loo_accs.append(acc)
        keep_idxs = keep_idxs[interval:]
    print("LOO: ", loo_accs)
    plt.plot(x, loo_accs, '^-', label='LOO', color='pink')

    # Knn Shapley
    print("KNN")
    for i in range(len(knn_values)):
        idxs = np.argsort(knn_values[i])
        keep_idxs = idxs.tolist()
        x = np.arange(0, count, interval)/len(x_train)

        for j in tqdm_notebook(range(0, count, interval), total=int(count/interval), leave=False):
            if len(keep_idxs) == len(x_train):
                x_train_keep, y_train_keep = x_train, y_train
            else:
                x_train_keep, y_train_keep = x_train[keep_idxs], y_train[keep_idxs]
            clf_knn = return_model(model)
            clf_knn.fit(x_train_keep, y_train_keep)
            acc = clf_knn.score(x_test, y_test)
            acc_t = clf_knn.score(x_train_keep, y_train_keep)

            knn_accs[i].append(acc)
            keep_idxs = keep_idxs[interval:]
            clf_knn = 0
    for i in range(0, len(knn_values)):
        print(x, knn_accs[i])
        plt.plot(x, knn_accs[i], '-', label="Knn Shapley, K="+str(labels[i]), color=colors[i])

    # LOO Knn Shapley
    print("LOO KNN")
    for i in range(len(loo_knn_values)):
        idxs = np.argsort(loo_knn_values[i])
        keep_idxs = idxs.tolist()
        x = np.arange(0, count, interval)/len(x_train)

        for j in tqdm_notebook(range(0, count, interval), total=int(count/interval), leave=False):
            if len(keep_idxs) == len(x_train):
                x_train_keep, y_train_keep = x_train, y_train
            else:
                x_train_keep, y_train_keep = x_train[keep_idxs], y_train[keep_idxs]
            clf_knn = return_model(model)
            clf_knn.fit(x_train_keep, y_train_keep)
            acc = clf_knn.score(x_test, y_test)
            loo_knn_accs[i].append(acc)
            keep_idxs = keep_idxs[interval:]

    for i in range(0, len(knn_values)):
        print(x, loo_knn_accs[i])
        plt.plot(x, loo_knn_accs[i], '-', label="LOO Knn Shapley, K="+str(labels[i]), color=colors[i+2])

    # random solution
    times = 5
    all_rand_accs = []
    print("random")
    for time in range(times):
        print(time)
        random_accs = []
        keep_idxs = np.arange(0, len(x_train))
        random.shuffle(keep_idxs)
        for j in range(0, count, interval):
            if len(keep_idxs) == len(x_train):
                x_train_keep, y_train_keep = x_train, y_train
            else:
                x_train_keep, y_train_keep = x_train[keep_idxs], y_train[keep_idxs]
            clf_random = return_model(model)
            clf_random.fit(x_train_keep, y_train_keep)
            acc = clf_random.score(x_test, y_test)
            random_accs.append(acc)
            keep_idxs = keep_idxs[interval:]
        all_rand_accs.append(random_accs)
    all_rand_accs = np.mean(all_rand_accs, 0)
    print("random:", all_rand_accs.tolist())
    plt.plot(x, all_rand_accs, '-', label='random', color='red')
    plt.legend()
    plt.tight_layout()
    if(HtoL == True):
        plt.savefig('knnH-L.png')
    else:
        plt.savefig('knnL-H.png')
    return

column_names = ['age', 'workclass', 'fnlwgt', 'education', 'educational-num','marital-status', 'occupation',
                'relationship', 'race', 'gender','capital-gain', 'capital-loss', 'hours-per-week', 'native-country',
                'income']

train = pd.read_csv('./data/uci/data.train', sep=",\s", header=None, names = column_names, engine = 'python')
test = pd.read_csv('./data/uci/data.test', sep=",\s", header=None, names = column_names, engine = 'python')
test['income'].replace(regex=True,inplace=True,to_replace=r'\.',value=r'')

adult = pd.concat([test,train])
adult.reset_index(inplace = True, drop = True)
for col in set(adult.columns) - set(adult.describe().columns):
    adult[col] = adult[col].astype('category')

# removing ? and Nan
adult = adult.dropna(how='any')
adult = adult[(adult.astype(str) != '?').all(axis=1)]

# Data Prep
adult_data = adult.drop(columns = ['income'])
adult_label = adult.income

le = preprocessing.LabelEncoder()
le.fit(adult_label)
adult_label = le.transform(adult_label)

adult_cat_1hot = pd.get_dummies(adult_data.select_dtypes('category'))
adult_non_cat = adult_data.select_dtypes(exclude = 'category')
adult_data_1hot = pd.concat([adult_non_cat, adult_cat_1hot], axis=1, join='inner')

idx0 = np.where(adult_label == 0)[0]
idx1 = np.where(adult_label == 1)[0]
print(len(idx0),len(idx1))

adult_data_1hot = adult_data_1hot.to_numpy()

train_size = 500
test_size = 250
heldout_size = 500

# print(adult_data_1hot(idx0[:train_size]))
train_data = np.concatenate((adult_data_1hot[idx0[:train_size]], adult_data_1hot[idx1[:train_size]]), 0)
train_label = np.concatenate((adult_label[idx0[:train_size]], adult_label[idx1[:train_size]]), axis=0)
test_data = np.concatenate((adult_data_1hot[idx0[train_size:train_size+test_size]], adult_data_1hot[idx1[train_size:train_size+test_size]]),0)
test_label = np.concatenate((adult_label[idx0[train_size:train_size+test_size]], adult_label[idx1[train_size:train_size+test_size]]), 0)
heldout_data = np.concatenate((adult_data_1hot[idx0[train_size+test_size:train_size+test_size+heldout_size]], adult_data_1hot[idx1[train_size+test_size:train_size+test_size+heldout_size]]), 0)
heldout_label = np.concatenate((adult_label[idx0[train_size+test_size:train_size+test_size+heldout_size]], adult_label[idx1[train_size+test_size:train_size+test_size+heldout_size]]), 0)

print(train_data.shape, heldout_data.shape, test_data.shape)

# Normalization
scaler = StandardScaler()

# Fitting only on training data
scaler.fit(train_data)
train_data = scaler.transform(train_data)

# Applying same transformation to test data
test_data = scaler.transform(test_data)
heldout_data = scaler.transform(heldout_data)

model = "uci"
clf = return_model(model)
# xx = 1000
clf.fit(train_data, train_label)
# pre_t = clf.predict(test_data)
directory = './temp_ds_uci'

test_acc = clf.score(heldout_data, heldout_label)
num_test = test_data.shape[0]

#data preparation
batch_size = 1024
epochs = 400

x_train = torch.from_numpy(train_data).contiguous().view(-1, 254)
y_train = torch.from_numpy(train_label).view(-1,).long()
print("train_size:", x_train.shape)
x_test = torch.from_numpy(test_data).contiguous().view(-1, 254)
y_test = torch.from_numpy(test_label).view(-1,).long()
print("test_size:", x_test.shape)
x_heldout = torch.from_numpy(heldout_data).contiguous().view(-1, 254)
y_heldout = torch.from_numpy(heldout_label).view(-1,).long()
print("heldout_size:", x_heldout.shape)

device = torch.device('cuda')
uci = UCI().to(device)
optimizer = optim.Adam(uci.parameters(), lr=0.001)
criterion = nn.CrossEntropyLoss()

train(uci, device, x_train, y_train, batch_size, optimizer, criterion, epochs)
accuracy, avg_loss = evaluate(uci, device, x_train, y_train, batch_size, criterion)
print(f'[Train] Accuracy: {100 * accuracy:5.2f}%, loss: {avg_loss:7.4f}')
accuracy, avg_loss = evaluate(uci, device, x_heldout, y_heldout, batch_size, criterion)
print(f'[Test] Accuracy: {100 * accuracy:5.2f}%, loss: {avg_loss:7.4f}')

deep_f_train = []
deep_f_test = []
targets = []
for X, y in batch(x_train, y_train, batch_size):
    X = X.to(device).float()
    fc3, y_pre = uci(X)
    deep_f_train.append(fc3.view(fc3.size(0), -1).cpu().detach().numpy())

for X, y in batch(x_test, y_test, batch_size):
    X = X.to(device).float()
    fc3, y_pre = uci(X)
    deep_f_test.append(fc3.view(fc3.size(0), -1).cpu().detach().numpy())

deep_f_train = np.concatenate(deep_f_train) # deep features are not normalized
deep_f_test = np.concatenate(deep_f_test) # deep features are not normalized
print(deep_f_train.shape, deep_f_test.shape)

kmin = 5
kmax = 6
kinterval = 5
fc1_knn_values = [[] for _ in range(math.ceil((kmax-kmin)/kinterval))] # deep features
loo_fc1_knn_values = [[] for _ in range(math.ceil((kmax-kmin)/kinterval))] # deep features

# t = 10000
for i, k in enumerate(range(kmin, kmax, kinterval)):
    print("neighbour number:", k)
    loo_fc1_knn_values[i],*_ = loo_knn_shapley(k, deep_f_train, deep_f_test, y_train, y_test)
    fc1_knn_values[i],*_ = old_knn_shapley(k, deep_f_train, deep_f_test, y_train, y_test)

plot_summarization(loo_values, tmc_values, g_values, fc1_knn_values, loo_fc1_knn_values, kmin, kmax, kinterval,
          x_train, y_train, x_heldout, y_heldout, False)


"""base module for Importing python files and manipulating the data structures before integrating into  Object"""

import pkgutil
import os
from datetime import datetime
from importlib.machinery import SourceFileLoader
from myfunctions.listmods import to_list
import importlib
from importlib.abc import Loader
import glob

class ImportRepository:
    """Main class that contains importing relating functions and methods"""

    def __init__(self):
        self.generic_dictionary = {}

    def load_execute_modules(
            self,
            executed_function,
            input_package,
            *kwargs,
            given_module=None):
        """Load the Variables from a package and all contained submodules and classes,
        Define a specific package or A module (file) from the package
        """
        input_package = to_list(input_package)

        for package in input_package:
            detected_modules = self.detect_submodules(package)
            if given_module is None:
                for module_name in detected_modules:
                    full_module_name = package + '.' + module_name
                    self.generic_dictionary.update(
                        self._execute_function(
                            executed_function,
                            full_module_name,
                            module_name,
                            *kwargs))
            else:
                full_module_name = input_package[0] + '.' + given_module
                self.generic_dictionary.update(
                    self._execute_function(
                        executed_function,
                        full_module_name,
                        given_module,
                        *kwargs))
            return self.generic_dictionary

    @staticmethod
    def _execute_function(functionname, *args):
        return functionname(*args)

    @staticmethod
    def rpo(object_name):
        """Ignores an object that is an internal or private variable and any functions"""
        if not object_name.startswith('__') and not object_name.startswith(
                '_') and not callable(object_name):
            return object_name
        return None

    @staticmethod
    def path_finder(object_path, start_path=None, walk=True):
        """Smart path finder that searches from a start path to find nearest match and continues to search up to the root directory"""
        start_time = datetime.now()
        user_base, user_ext = os.path.splitext(object_path)
        unicode_path_base = user_base.replace('/', '\\')
        full_name = None
        if os.path.exists(object_path) is True:
            return os.path.abspath(object_path)
        if start_path:
            dir_path = start_path
        else:
            dir_path = os.path.dirname(os.path.realpath(__file__))
        split_paths = dir_path.split('\\')
        directory = split_paths[0]
        for i in range((len(split_paths)), 0, -1):
            crawler = split_paths[:i]
            joiner = '\\'.join(crawler)
            if walk is False:
                joiner = dir_path
            for root, dirs, files in os.walk(joiner):
                if user_ext is None or user_ext == '':
                    for name in dirs:
                        full_name = os.path.join(root, name)
                        if unicode_path_base == name:
                            if full_name:
                                return str(os.path.realpath(full_name))
                else:
                    for name in files:
                        (base, ext) = os.path.splitext(
                            name)  # split base and extension
                        if ext == user_ext:  # check the extension
                            if base in unicode_path_base:
                                full_name = os.path.join(root, name)
                                if unicode_path_base in full_name:
                                    if full_name:
                                        return str(os.path.realpath(full_name))
        timediff = (datetime.now() - start_time)
        time_s = (timediff.seconds + ((timediff.microseconds) / (1000000)))
        msg = (
            f'The system was searched recursively up to the {directory} drive, yet no file or folder path containing {object_path} was found. Check your provided file path and extension.\n (Processing time: {time_s} seconds) ')
        raise ValueError(msg)

    def filename(self, path):
        return os.path.basename(os.path.splitext(path)[0])

    def locate_package(self, packagename, walk=True, init=True):
        """This method verifies the existence of a package and returns the full path
        arguments:
        filename: The suspected name of the file being searched for
            ex: "path_manipulation.py"
        walk: walk is a boolean argument. A value of "True" will allow the file finder to search
              recursively on the current drive.
         """
        initstring = r'\__init__.py'
        if not isinstance(walk, bool):
            raise TypeError(
                'walk is a boolean argument, must be either True or False')
        if init is True:
            packagename = packagename + initstring
        return self.path_finder(packagename, walk=walk)

    def detect_submodules(self, package, init=True):
        """Import all submodules of a module, recursively, including subpackages"""
        package_path = self.locate_package(package, walk=True, init=init)
        found_package = SourceFileLoader(
            package, package_path).load_module(package)
        results = []
        for loader, name, is_pkg in pkgutil.walk_packages(
                found_package.__path__):
            results.append(name)
        return results

    def locate_python_subfiles(self, dir_path):
        dir_path = self.path_finder(dir_path)
        return [
            os.path.join(
                dir_path,
                f) for f in os.listdir(dir_path) if os.path.isfile(
                os.path.join(
                    dir_path,
                    f)) and os.path.splitext(f)[1] == '.py']

    def import_class(self, module_path, class_name=None):
        try:
            assert os.path.isfile(module_path) is True
        except AttributeError:
            raise ImportError(
                "Module file does not exist, ensure full path has been given")
        module_name = self.filename(module_path)
        if not class_name:
            class_name = module_name
        spec = importlib.util.spec_from_file_location(module_name, module_path)
        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)
        try:
            klass = getattr(module, class_name)
            return klass, class_name
        except AttributeError as abe:
            raise ImportError(
                f"A module named {module_name}.py was found but it did not have a class named {class_name}") from abe

    def find_files(self, path, patterns):
        filelist = []
        for pattern in patterns:
            filelist = filelist + \
                glob.glob(f"{path}/**/{pattern}", recursive=True)
        return filelist


import logging

from collections import defaultdict
from os.path import abspath
from typing import Any, Callable, Optional

from zelos.api.memory_api import MemoryApi
from zelos.api.regs_api import RegsApi
from zelos.breakpoints import BreakState
from zelos.config_gen import generate_config, generate_config_from_cmdline
from zelos.engine import Engine
from zelos.hooks import HookInfo, HookType
from zelos.plugin import ParsedBinary, Plugins
from zelos.processes import Process
from zelos.threads import Thread

class Zelos:
    """
    Class that provides access to the core APIs. These core APIs
    are event hooking, debugging, memory access, register access, and
    emulation context.

    Args:
        filename: Specifies the name of the file to emulate.
        cmdline_args: Arguments that are passed to the emulated binary.
        flags: Parameters for zelos. To see the list of all flags, refer
            to :ref:`flag-label`

    Example:
        .. code-block:: python

            from zelos import Zelos

            # initialize zelos with binary name, 2 cmdline args, and
            # turn on the inst feed to print instructions.
            z = Zelos(
                "binary_to_emulate"
                "ARG1",
                "ARG2",
                inst=True,
            )
    """

    def __init__(self, filename, *cmdline_args, **flags):
        config = generate_config(filename, *cmdline_args, **flags)
        self._setup(config)

    def _setup(self, config):
        self.config = config
        self._regs = RegsApi(self)
        self._memory = MemoryApi(self)

        self._breakpoints = {}
        self._watchpoints = defaultdict(dict)

        # If you need to access data that is not exposed through the api
        # yet, access the internal_engine representation at your own
        # risk.
        e = Engine(config=config, api=self)
        self.plugins = Plugins(self, ["plugins"])
        self.plugins.initialize()
        self.internal_engine = e  # Used to inform what type this is.
        self.internal_engine.plugins = self.plugins

        # After setup allow MEMORY.INTERNAL_* hooks to run.
        e.hook_manager._enable_internal_memory_hooks()

    # **** Memory API ****
    @property
    def memory(self):
        """
        Returns the :py:class:`~zelos.api.memory_api.MemoryApi` object.
        """
        return self._memory

    # **** Registers API ****
    @property
    def regs(self):
        """
        Returns the :py:class:`~zelos.api.regs_api.RegsApi` object.
        """
        return self._regs

    @property
    def logger(self):
        """
        Returns a logger that will behave in accordance to the --log
        config/cmdline option.
        """
        return logging.getLogger(__name__)

    # **** Begin Hook API ****
    def hook_memory(
        self,
        hook_type: HookType.MEMORY,
        callback: Callable[["Zelos", int, int, int, int], Any],
        mem_low: Optional[int] = None,
        mem_high: Optional[int] = None,
        name: Optional[str] = None,
        end_condition: Optional[Callable[[], bool]] = None,
    ) -> HookInfo:
        """
        Registers a hook on memory. Executes callback every time the
        specified event happens in memory.

        The hook will only trigger when the event occurs at an address
        between mem_low and mem_high, if either of them are specified.

        The hook will continue to trigger until the end_condition
        specified evaluates to True.

        Args:
            hook_type: Specifies the event in memory that should trigger
                the callback to be executed. Options can be found in
                :py:class:`zelos.HookType.MEMORY`
            callback: The code that should be executed when the
                specified event occurs. The function should accept the
                following inputs: (zelos, access, address, size, value).
                The return value of "callback" is ignored.
            mem_low: If specified, only executes callback if the
                event occurs at an address greater than or equal to
                this.
            mem_high: If specified, only executes callback if the
                event occurs at an address less than or equal to this.
            name: An identifier for this hook. Used for debugging.
            end_condition: If specified, executes after the callback. If
                the function returns True, this hook is deleted.

        Returns:
            Information regarding the hook.

        Example:
            .. code-block:: python

                from zelos import Zelos, HookType

                # Print every write to memory
                def memory_hook(zelos, access, address, size, value):
                    print(value)

                z = Zelos("binary_to_emulate")
                z.hook_memory(
                    HookType.MEMORY.WRITE,
                    memory_hook
                )
                z.start()
        """

        return self.internal_engine.hook_manager.register_mem_hook(
            hook_type,
            callback,
            mem_low=mem_low,
            mem_high=mem_high,
            name=name,
            end_condition=end_condition,
        )

    def hook_execution(
        self,
        hook_type: HookType.EXEC,
        callback: Callable[["Zelos", int, int], Any],
        ip_low: Optional[int] = None,
        ip_high: Optional[int] = None,
        name: Optional[str] = None,
        end_condition: Optional[Callable[[], bool]] = None,
    ) -> HookInfo:
        """
        Registers a hook that executes when code is executed. This is
        either for every instruction that is executed, or every block.

        The hook will only trigger when the event occurs at an address
        between ip_low and ip_high, if either of them are specified.

        The hook will continue to trigger until the end_condition
        specified evaluates to True.

        Args:
            hook_type: Specifies whether the callback should be
                triggered every instruction, or every block. Options can
                be found in :py:class:`zelos.HookType.EXEC`
            callback: The code that should be executed when the
                specified event occurs. The function should accept the
                following inputs: (zelos, address, size).
                The return value of "callback" is ignored.
            ip_low: If specified, only executes callback if the
                event occurs at an address greater than or equal to
                this.
            ip_high: If specified, only executes callback if the
                event occurs at an address less than or equal to this.
            name: An identifier for this hook. Used for debugging.
            end_condition: If specified, executes after the callback. If
                the function returns True, this hook is deleted.

        Returns:
            Information regarding the hook.

        Example:
            .. code-block:: python

                from zelos import Zelos, HookType

                # Print the first address of every block
                def exec_hook(zelos, address, size):
                    print(address)

                z = Zelos("binary_to_emulate")
                z.hook_execution(
                    HookType.EXEC.BLOCK, exec_hook
                )
                z.start()
        """
        return self.internal_engine.hook_manager.register_exec_hook(
            hook_type,
            callback,
            ip_low=ip_low,
            ip_high=ip_high,
            name=name,
            end_condition=end_condition,
        )

    def hook_close(self, callback: Callable[[], Any]) -> HookInfo:
        """
        Registers a callback that is called when
        :py:meth:`zelos.Engine.close()` is called.

        Args:
            callback: Called when zelos is closed. Does not take any
                arguments. The return value of `callback` is ignored

        Example:
            .. code-block:: python

                from zelos import Zelos

                # Close a file you are using with zelos
                file = open("testfile", "r")
                def close_cleanup():
                    file.close()

                z = Zelos("binary_to_emulate")
                z.hook_close(close_cleanup)
                z.start()

                # Hooks are run at this point
                z.close()
        """
        return self.internal_engine.hook_manager.register_close_hook(callback)

    def hook_syscalls(
        self,
        syscall_hook_type: HookType.SYSCALL,
        callback: Callable[["Zelos", str, "Args", int], Any],
        name: str = None,
        syscall_name: str = None,
    ) -> HookInfo:
        """
        Registers a callback that is called when a syscall is invoked.

        Args:
            syscall_hook_type: Decides when the hook should be triggered
                in relation to the execution of the syscall. Options can
                be found in :py:class:`zelos.HookType.SYSCALL`
            callback: The code that should be executed when the
                specified event occurs. The function should accept the
                following inputs:
                (zelos, syscall_name, args, return_value)
                The return value of "callback" is ignored.
            syscall_name: Restricts calls of the hook to syscalls that
                match the given name.
            name: An identifier for this hook. Used for debugging.

        Example:
            .. code-block:: python

                from zelos import Zelos, HookType

                # Keep track of the syscall return values
                syscall_return_values = []
                def syscall_hook(zelos, sys_name, args, ret_val):
                    syscall_return_values.append((sys_name, ret_val))

                z = Zelos("binary_to_emulate")
                z.hook_syscalls(
                    HookType.SYSCALL.AFTER, syscall_hook
                )
                z.start()

        """
        return self.internal_engine.hook_manager.register_syscall_hook(
            syscall_hook_type, callback, name=name, syscall_name=syscall_name
        )

    def hook_zml(
        self, zml_string: str, callback: Callable[[], Any]
    ) -> HookInfo:
        """
        Register a callback that triggers when a given ZML string is
        satisfied. For more information on ZML, view
        :py:class:`zelos.zml.ZmlParser`.

        Args:
            zml_string: Specifies condition to call callback.
            callback: The code that should be executed when the specified
                event occurs.
        """
        return self.internal_engine.hook_manager.register_zml_hook(
            zml_string, callback
        )

    def delete_hook(self, hook_info: HookInfo) -> None:
        self.internal_engine.hook_manager.delete_hook(hook_info)

    # **** Begin Debugging API ****

    def start(self, timeout: float = 0) -> Optional[BreakState]:
        """
        Begin emulation. When called for the first time, begins
        execution at the binary entry point. If the emulation is
        stopped (for example, after calling :py:meth:`stop()`) this
        will resume execution from the current IP.

        Args:
            timeout: Stops execution after `timeout` seconds.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                # Start execution from the entry point
                z.start()

        """
        return self.internal_engine.start(timeout=timeout)

    def step(self, count=1) -> None:
        """
        Begin emulation, stopping after executing `count` instructions.

        Args:
            count: Maximum number of instructions to execute before
                stopping
        """
        return self.internal_engine.step(count=count)

    def next(self, count=1) -> None:
        """
        Begin emulation, executing `count` instructions not including
        instructions inside function calls.

        Args:
            count: Maximum number of instructions to execute before
                stopping.
        """
        return self.internal_engine.step_over(count=count)

    def stop(self, reason: str = "plugin"):
        """
        Stop the Zelos run loop. After a call to
        :py:meth:`stop()`, execution can be resumed from the
        current IP with a call to :py:meth:`start()`.

        Args:
            reason: An optional identifier that specifies a reason for
                stopping execution. Upon calling stop, the reason will
                be printed to stdout after Zelos exits the run loop.
                This is useful for debugging when log level is set to
                'debug' or 'spam'.
        """
        self.internal_engine.scheduler.stop(reason)

    def close(self):
        """
        Closes Zelos and runs cleanup functions.
        """
        self.internal_engine.close()

    def end_thread(self):
        """
        End the current thread. Marks current thread as successfully
        completed and swaps to the next available thread, if one exists.
        """
        self.internal_engine.thread_manager.complete_current_thread()

    def swap_thread(self, reason: str = "thread swap"):
        """
        Swap the running thread with the next scheduled thread.

        Args:
            reason: An optional identifier that specifies a reason for
                swapping threads. Upon calling swap_thread, the reason
                will be printed to stdout after Zelos has successfully
                swapped to the next scheduled thread. This is useful
                for debugging when log level is set to 'spam'.
        """
        self.internal_engine.scheduler.stop_and_exec(
            reason, self.internal_engine.processes.schedule_next
        )

    def set_breakpoint(self, address: int, temporary: bool = False):
        """
        Set a breakpoint at a particular address.

        Args:
            address: Target address of breakpoint.
            temporary: Determines whether or not the breakpoint is
                temporary. A temporary breakpoint will be automatically
                removed after use.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_breakpoint(0xdeadbeef)

                z.start()

        """
        self.internal_engine.breakpoints.set_breakpoint(
            address, temporary=temporary
        )

    def remove_breakpoint(self, address: int):
        """
        Remove a previously set breakpoint.

        Args:
            address: Target address of breakpoint to remove.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_breakpoint(0xdeadbeef)

                z.remove_breakpoint(0xdeadbeef)

                z.start()

        """
        self.internal_engine.breakpoints.remove_breakpoint(address)

    def set_syscall_breakpoint(self, syscall_name: str):
        """
        Set a breakpoint at all syscalls of a specified name.

        Args:
            syscall_name: Target syscall set breakpoint at.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_syscall_breakpoint("write")

                z.start()

        """
        self.internal_engine.kernel.set_breakpoint(syscall_name)

    def remove_syscall_breakpoint(self, syscall_name: str):
        """
        Remove a previously set syscall breakpoint specified by name.

        Args:
            syscall_name: Target syscall to remove breakpoints from.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_syscall_breakpoint("write")

                z.remove_syscall_breakpoint("write")

                z.start()

        """
        self.internal_engine.kernel.remove_breakpoint(syscall_name)

    def set_watchpoint(
        self, address: int, read: bool, write: bool, temporary: bool = False
    ):
        """
        Set a watchpoint on a particular memory address.

        Args:
            address: Target address of watchpoint.
            read: Determines whether to watch for reads to the target
                memory address.
            write: Determines whether to watch for writes to the target
                memory address.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                # Break at any read or write to memory address 0xdeadbeef
                z.set_watchpoint(0xdeadbeef, True, True)

                # Break only at writes to memory address 0xfeedf00d
                z.set_watchpoint(0xfeedf00d, False, True)

                # Break only at reads to memory address 0xb0bad00d
                z.set_watchpoint(0xb0bad00d, True, False)

                z.start()

        """

        def hook(zelos, access, address, size, value):
            zelos.stop("watchpoint")

        if read:
            read_hook_info = self.hook_memory(
                HookType.MEMORY.READ,
                hook,
                name=f"read_watchpoint_{address:x}",
                mem_low=address,
                mem_high=address,
                end_condition=lambda: temporary,
            )
            self._watchpoints[address]["read"] = read_hook_info
        if write:
            write_hook_info = self.hook_memory(
                HookType.MEMORY.WRITE,
                hook,
                name=f"write_watchpoint_{address:x}",
                mem_low=address,
                mem_high=address,
                end_condition=lambda: temporary,
            )
            self._watchpoints[address]["write"] = write_hook_info

        return True

    def remove_watchpoint(self, address: int):
        """
        Remove a previously set watchpoint.

        Args:
            address: Target address of watchpoint to remove.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.set_watchpoint(0xdeadbeef, True, True)

                z.remove_watchpoint(0xdeadbeef)

                z.start()

        """
        for hook_info in self._watchpoints[address].values():
            self.internal_engine.hook_manager.delete_hook(hook_info)
        del self._watchpoints[address]

        return True

    # **** Begin Context API ****

    @property
    def date(self):
        """
        Returns the date string used internally during emulation. The
        date string format is `YYYY-MM-DD`.

        :getter: Returns the date string in YYYY-MM-DD format.
        :setter: Sets the date string. Input must be YYYY-MM-DD format.
        :type: str
        """
        return self.internal_engine.date

    @date.setter
    def date(self, date_str: str):
        """
        Set the date to be used for emulation. This affects the linux
        system calls sys_time, sys_gettimeofday, and sys_clock_gettime.

        Args:
            date_str: A string of the form `YYYY-MM-DD` specifying the
                target date.

        Example:
            .. code-block:: python

                from zelos import Zelos

                z = Zelos("binary_to_emulate")

                z.date = "2020-03-04"

                z.start()

        """
        self.internal_engine.date = date_str

    @property
    def process(self) -> Process:
        """
        Returns the currently active process.

        :type: Process

        """
        return self.internal_engine.current_process

    @property
    def thread(self) -> Thread:
        """
        Returns the currently active thread.

        :type: Thread

        """
        return self.internal_engine.current_process.current_thread

    @property
    def main_binary(self) -> Optional[ParsedBinary]:
        """
        Returns the parsed main binary, if it exists, otherwise returns
        None.
        Note that the "main" binary denotes the binary that is loaded by
        Zelos during emulation, not necessarily the binary that
        is specified as input (the "target" binary). When the specified
        input binary ("target") is statically linked, it is also the
        "main" binary. However, when the specified input binary
        ("target") is dynamically linked, the "main" binary instead
        refers to the dynamic linker/loader.

        :type: :py:class:`zelos.plugin.ParsedBinary`

        """
        return self.internal_engine.main_module

    @property
    def main_binary_path(self) -> Optional[str]:
        """
        Returns the absolute path to the main binary, if it exists, otherwise
        returns None.
        Note that the "main" binary denotes the binary that is loaded by
        Zelos during emulation, not necessarily the binary that
        is specified as input (the "target" binary). When the specified
        input binary ("target") is statically linked, it is also the "main"
        binary. However, when the specified input binary ("target") is
        dynamically linked, the "main" binary instead refers to the dynamic
        linker/loader.

        :type: str

        """
        if self.internal_engine.main_module_name:
            return abspath(self.internal_engine.main_module_name)
        return None

    @property
    def target_binary_path(self) -> Optional[str]:
        """
        Returns the absolute path to the target binary, if it exists,
        otherwise returns None. Note that the "target" binary denotes
        the binary specified as input to Zelos.

        :type: str

        """
        if self.internal_engine.target_binary_path:
            return abspath(self.internal_engine.target_binary_path)
        return None

class ZelosCmdline(Zelos):
    """
    A workaround for allowing `Zelos` to be initialized with a
    commandline string while also allowing the main `Zelos` to maintain
    its simple/intuitive constructor based on commandline arguments and
    flags.
    """

    def __init__(self, cmdline_args):
        config = generate_config_from_cmdline(cmdline_args)
        self._setup(config)