# Developed by MD-Nazmus Samin Sunbeam
import os
import time

import gym
import numpy as np
import pyautogui
from gym import Env, spaces

class ComputerEnv(gym.Env):
    """Computer Environment that follows the gym template."""
    metadata = {'render.modes': ['human']}

    def __init__(self):
        super(ComputerEnv, self).__init__()
        # define observation space
        (width, height) = pyautogui.size()
        self.observation_shape = (width, height, 3)
        self.observation_space = spaces.Box(low=0, high=255, shape=self.observation_shape, dtype=np.uint8)
        # define action space
        self.action_space = spaces.Discrete(44)

    def step(self, action):
        # Flag that marks the termination of an episode
        done = False

        # Assert that it is a valid action
        assert self.action_space.contains(action), "Invalid Action"
        t = 2
        d = 50
        x_mouse, y_mouse = pyautogui.position()
        # apply actions with mouse and keyboard
        # move horizontally right
        if action == 0:
            x_mouse = d
            pyautogui.move(x_mouse, 0, t)
        # move vertically down
        elif action == 1:
            y_mouse = d
            pyautogui.move(0, y_mouse, t)
        # move horizontally left
        elif action == 2:
            x_mouse = d
            pyautogui.move(x_mouse, 0, t)
        # move vertically down
        elif action == 3:
            y_mouse = d
            pyautogui.move(0, y_mouse, t)
        # move diagonally southeast
        if action == 4:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # move diagonally southwest
        elif action == 5:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # move diagonally northeast
        elif action == 6:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # move diagonally northwest
        elif action == 7:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # right click
        elif action == 8:
            pyautogui.click(button='right', clicks=1, interval=0.25)
        # left click
        elif action == 9:
            pyautogui.click(button='left', clicks=1, interval=0.25)
        # press a key
        elif action == 10:
            pyautogui.press('a')
        # press b key
        elif action == 11:
            pyautogui.press('b')
        # press c key
        elif action == 12:
            pyautogui.press('c')
        # press d key
        elif action == 13:
            pyautogui.press('d')
        # press e key
        elif action == 14:
            pyautogui.press('e')
        # press f key
        elif action == 15:
            pyautogui.press('f')
        # press g key
        elif action == 16:
            pyautogui.press('g')
        # press h key
        elif action == 17:
            pyautogui.press('h')
        # press j key
        elif action == 18:
            pyautogui.press('j')
        # press k key
        elif action == 19:
            pyautogui.press('k')
        # press l key
        elif action == 20:
            pyautogui.press('l')
        # press m key
        elif action == 21:
            pyautogui.press('m')
        # press n key
        elif action == 22:
            pyautogui.press('n')
        # press o key
        elif action == 23:
            pyautogui.press('o')
        # press p key
        elif action == 24:
            pyautogui.press('p')
        # press q key
        elif action == 25:
            pyautogui.press('q')
        # press r key
        elif action == 26:
            pyautogui.press('r')
        # press s key
        elif action == 27:
            pyautogui.press('s')
        # press t key
        elif action == 28:
            pyautogui.press('t')
        # press u key
        elif action == 29:
            pyautogui.press('u')
        # press v key
        elif action == 30:
            pyautogui.press('v')
        # press x key
        elif action == 31:
            pyautogui.press('x')
        # press y key
        elif action == 32:
            pyautogui.press('y')
        # press z key
        elif action == 33:
            pyautogui.press('z')
        # press 0 key
        elif action == 34:
            pyautogui.press('0')
        # press 1 key
        elif action == 35:
            pyautogui.press('1')
        # press 2 key
        elif action == 36:
            pyautogui.press('2')
        # press 3 key
        elif action == 37:
            pyautogui.press('3')
        # press 4 key
        elif action == 38:
            pyautogui.press('4')
        # press 5 key
        elif action == 39:
            pyautogui.press('5')
        # press 6 key
        elif action == 40:
            pyautogui.press('6')
        # press 7 key
        elif action == 41:
            pyautogui.press('7')
        # press 8 key
        elif action == 42:
            pyautogui.press('8')
        # press 9 key
        elif action == 43:
            pyautogui.press('9')

        obs = pyautogui.screenshot()

        return obs

    def reset(self):
        os.system("killall -9 'Safari'")

    def render(self, mode='human', close=False):
        raise NotImplementedError


import os
import functools
import time
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

from sklearn.base import BaseEstimator, TransformerMixin

from constants import PLOTS_DIRECTORY, DIAGNOSTICS_DIRECTORY, FEATURES_TO_DROP, TARGET

def _plot_correlation_matrix(df):
    """
    Plots a correlation matrix for all numeric features in a dataframe, excluding those that will be dropped for
    modeling, per constants.py. Saves a png of the plot in the nested plots directory, identified in constant.py.
    :param df: pandas dataframe
    """
    df = df.select_dtypes(exclude=[object])
    corr = df.corr()
    mask = np.triu(np.ones_like(corr, dtype=np.bool))
    cmap = sns.diverging_palette(220, 10, as_cmap=True)
    sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0, square=True, linewidths=.5, cbar_kws={"shrink": .5})
    plt.title('Correlation Matrix')
    plt.tight_layout()
    plt.savefig(os.path.join(DIAGNOSTICS_DIRECTORY, PLOTS_DIRECTORY, 'correlation_matrix_.png'))
    plt.clf()

def _plot_average_by_category(df, fill_na_value='unknown'):
    """
    Produces a bar plot of the average value of the target, defined in constants.py, for each category in the dataframe.
    The plots are saved as png files in the nested plots directory, identified in constants.py. One file per categorical
    column is produced.
    :param df: pandas dataframe
    :param fill_na_value: the value for which to fill categorical nulls
    """
    features = list(df.select_dtypes(include=[object]))
    for feature in features:
        if feature not in FEATURES_TO_DROP:
            df.fillna({feature: fill_na_value}, inplace=True)
            grouped_df = pd.DataFrame(df.groupby(feature)[TARGET].mean())
            grouped_df.columns = ['mean']
            grouped_df.reset_index(inplace=True)
            sns.barplot(x=feature, y='mean', data=grouped_df)
            plt.title('Class Average for ' + feature)
            plt.xticks(rotation=90)
            plt.savefig(os.path.join(DIAGNOSTICS_DIRECTORY, PLOTS_DIRECTORY, 'class_average_for_' + feature + '.png'))
            plt.clf()

def _plot_histogram(df, column):
    """
    Plots a histogram and saves the result as a png file in the nested plots directory, identified in constants.py.
    :param df: pandas dataframe
    :param column: the column for which to make the histogram
    """
    sns.distplot(df[column], kde=False)
    plt.xlabel(column)
    plt.ylabel('density')
    plt.title('Histogram for {}'.format(column))
    plt.savefig(os.path.join(DIAGNOSTICS_DIRECTORY, PLOTS_DIRECTORY, 'histogram_plot_for_' + column + '.png'))
    plt.clf()

def make_diagnostic_plots(df):
    """
    Runs plotting functions in one call to produce a correlation matrix, average value of the target by category,
    and a histogram of the target.
    :param df: pandas dataframe
    """
    _plot_correlation_matrix(df)
    _plot_average_by_category(df)
    _plot_histogram(df, TARGET)

def timer(func):
    """
    Prints the time in seconds it took to run a function. To be used as a decorator.
    """
    @functools.wraps(func)
    def wrapper_timer(*args, **kwargs):
        start_time = time.perf_counter()
        value = func(*args, **kwargs)
        end_time = time.perf_counter()
        run_time = end_time - start_time
        print(f"finished {func.__name__!r} in {run_time:.4f} secs")
        return value
    return wrapper_timer

def _create_directories_helper(directory, parent):
    """
    Creates a directory if it does not exist.
    :param directory: directory to create
    :param parent: parent directory of the directory argument, which will also be created if necessary
    """
    if not os.path.exists(os.path.join(parent, directory)):
        os.makedirs(os.path.join(parent, directory))

def create_directories(directories_list, parent=os.getcwd()):
    """
    Calls _create_directories_helper to create an arbitrary number of directories
    :param directories_list: list of directories to create
    :param parent: the parent directory for each directory in directories_list
    """
    _create_directories_helper(parent, os.getcwd())
    if parent in directories_list:
        directories_list.remove(parent)
    for directory in directories_list:
        _create_directories_helper(directory, parent)

def drop_features(df, features_drop_list):
    """
    Drops columns from a pandas dataframe.
    :param df: pandas dataframe
    :param features_drop_list: list of features to drop
    :return: pandas dataframe
    """
    return df.drop(features_drop_list, 1)

def get_num_and_cat_feature_names(df):
    """
    Produces lists of numeric and categorical feature names from a dataframe.
    :param df: pandas dataframe
    :return: tuple of two lists, the first containing the names of the numeric features and the second containing the
    names of the categorical features
    """
    numeric_features = list(df.select_dtypes(exclude=[object]))
    categorical_features = list(df.select_dtypes(include=[object]))

    for feature in FEATURES_TO_DROP:
        numeric_features = [f for f in numeric_features if f != feature]
        categorical_features = [f for f in categorical_features if f != feature]
    return numeric_features, categorical_features

class FeaturesToDict(BaseEstimator, TransformerMixin):
    """
    Converts dataframe, or numpy array, into a dictionary oriented by records. This is a necessary pre-processing step
    for DictVectorizer().
    """
    def __int__(self):
        pass

    def fit(self, X, Y=None):
        return self

    def transform(self, X, Y=None):
        if isinstance(X, np.ndarray):
            X = pd.DataFrame(X)
        X = X.to_dict(orient='records')
        return X


import re
from collections import namedtuple
from .headers import DeviceHeader
from .user_agent import UserAgnet
from .constants import MOBILE_HEADERS, TABLET_DEVICES, get_extended_rule, get_mobile_rule

class Detection(object):

    # Mobile detection type.
    DETECTION_TYPE_MOBILE = 'mobile'

    # Extended detection type.
    DETECTION_TYPE_EXTENDED = 'extended'

    def __init__(self, request):
        self.device_header = DeviceHeader(request.META)
        self.user_agent = UserAgnet(self.device_header.http_headers, self.device_header.cloud_front_headers)

    def get_user_agent(self):
        """
        Retrieve the User-Agent name
        """
        return self.user_agent.name

    def is_mobile(self, user_agent=None, http_headers=None):
        """
        Check if the device is mobile.
        Returns true if any type of mobile device detected, including special ones
        """
        if http_headers:
            self.device_header.change(http_headers)
        if user_agent:
            self.user_agent.change(user_agent)

        # Check specifically for cloudfront headers if the useragent is 'Amazon CloudFront'
        if self.user_agent.name == 'Amazon CloudFront':
            if "HTTP_CLOUDFRONT_IS_MOBILE_VIEWER" in self.device_header.cloud_front_headers and self.device_header.cloud_front_headers[
                'HTTP_CLOUDFRONT_IS_MOBILE_VIEWER'] == 'true':
                return True

        self.set_detection_type(self.DETECTION_TYPE_MOBILE)

        if self.check_http_headers_for_mobile():
            return True
        else:
            return self.match_detection_rules()

    def is_tablet(self, user_agent=None, http_headers=None):
        """
        Check if the device is a tablet.
        Return true if any type of tablet device is detected.
        """

        # Check specifically for cloudfront headers if the useragent == 'Amazon CloudFront'
        if self.user_agent == 'Amazon CloudFront':
            if "HTTP_CLOUDFRONT_IS_TABLET_VIEWER" in self.device_header.cloud_front_headers and \
                    self.device_header.cloud_front_headers['HTTP_CLOUDFRONT_IS_TABLET_VIEWER'] == 'true':
                return True
        self.set_detection_type(self.DETECTION_TYPE_MOBILE)
        for _regex in TABLET_DEVICES:
            if self.match(_regex, user_agent):
                return True

    def set_detection_type(self, detection_type):
        """
        Set the detection type. Must be one of self.DETECTION_TYPE_MOBILE or
        self.DETECTION_TYPE_EXTENDED. Otherwise, nothing is set.
        """
        if detection_type is None:
            detection_type = self.DETECTION_TYPE_MOBILE

        if detection_type != self.DETECTION_TYPE_MOBILE and detection_type != self.DETECTION_TYPE_EXTENDED:
            return None

        self.detection_type = detection_type

    def check_http_headers_for_mobile(self):
        """
        Check the HTTP headers for signs of mobile.
        This is the fastest mobile check possible; it's used
        inside is_mobile() method.
        """
        http_headers = self.device_header.http_headers
        for mobile_header in MOBILE_HEADERS:
            match_type = MOBILE_HEADERS[mobile_header]
            if mobile_header in http_headers:
                if 'matches' in match_type and isinstance(match_type['matches'], list):
                    for _match in match_type['matches']:
                        if http_headers[mobile_header].find(_match) != -1:
                            return True
                    return False

                return False

        return False

    def match_detection_rules(self, user_agent=None):
        # Begin general search.
        rules = self.get_rules()

        for _regex in rules:
            if not _regex:
                continue
            if self.match(_regex, user_agent):
                return True
        else:
            return False

    def get_rules(self):
        """
        Retrieve the current set of rules.
        """
        if self.detection_type == self.DETECTION_TYPE_EXTENDED:
            return get_extended_rule()
        else:
            return get_mobile_rule()

    def match(self, match, user_agents=None):
        """
        Some detection rules are relative (not standard),
        because of the diversity of devices, vendors and
        their conventions in representing the User-Agent or
        the HTTP headers.
        This method will be used to check custom regexes against
        the User-Agent string.
        """

        ## if user agents is empty, return false by default
        if not user_agents:
            return False

        matches = re.findall(re.compile(match), user_agents if user_agents else self.user_agent.name)
        if matches and len(matches) > 0:
            self.matching_regex = match
            self.matches_array = matches
            return True
        else:
            return False

    def get_device_type(self):
        pass

def detect(request):
    detector = Detection(request)
    result = {
        'is_mobile': detector.is_mobile(),
        'is_tablet': detector.is_mobile(),
        'user_agent': detector.user_agent.name
    }
    object_name = namedtuple("ObjectName", result.keys())(*result.values())
    return object_name


from cmq_rpc import RpcServer, RpcClient, DataPacker
from gui_misc import field2variable, variable2field
import base
import logging
import datetime
import sys
import time
import json

class ProxyRpcServer(RpcServer):
    def __init__(self, rep_address, pub_address, config = {}, tday = datetime.date.today()):
        super(ProxyRpcServer, self).__init__(rep_address, pub_address)
        self.scur_day = tday
        self.name = config.get('name', 'test_agent')
        agent_class = config.get('agent_class', 'agent.Agent')
        cls_str = agent_class.split('.')
        agent_cls = getattr(__import__(str(cls_str[0])), str(cls_str[1]))
        self.agent = agent_cls(config = config, tday = tday)
        self.restart()
        self.register(self.restart)
        self.register(self.get_agent_params)
        self.register(self.set_agent_params)
        self.register(self.set_gateway_params)
        self.register(self.get_strat_params)
        self.register(self.set_strat_params)
        self.register(self.run_agent_func)
        self.register(self.run_gateway_func)
        self.register(self.run_strat_func)
        self.register(self.get_hist_data)
        self.register(self.exit_agent)

    def restart(self):
        if self.agent != None:
            self.scur_day = self.agent.scur_day
        self.agent.restart()

    def get_agent_params(self, fields):
        res = {}
        for f in fields:
            field_list = f.split('.')
            field = field_list[0]
            if field not in res:
                res[field] = {}
            if field == 'Positions':
                positions = {}
                for gway in self.agent.gateways:
                    gateway = self.agent.gateways[gway]
                    positions[gway] = {}
                for inst in gateway.positions:
                    pos = gateway.positions[inst]
                    positions[gway][inst] = {'currlong' : pos.curr_pos.long,
                                       'currshort': pos.curr_pos.short,
                                       'locklong' : pos.locked_pos.long,
                                       'lockshort': pos.locked_pos.short,
                                       'ydaylong':  pos.pos_yday.long,
                                       'ydayshort': pos.pos_yday.short}
                res[field] = positions
            elif field == 'Risk':
                risk_list = field_list[1:]
                sum_risk, risk_dict = self.agent.risk_by_strats(risk_list)
                res[field] = {'total': sum_risk, 'strats': risk_dict }
            elif field == 'Account':
                gateway = self.agent.gateways[field_list[1]]
                res[field][gateway.gatewayName] = dict([(variable2field(var), gateway.account_info[var]) for var in gateway.account_info])
            elif field ==' Orderstats':
                gateway = self.agent.gateways[field_list[1]]
                res[field][gateway.gatewayName] = dict([(variable2field(var), gateway.order_stats[var]) for var in gateway.order_stats])
            elif field == 'Orders':
                order_list = []
                gateway = self.agent.gateways[field_list[1]]
                for o in gateway.id2order.values():
                    inst = o.position.instrument.name
                    order_list.append([o.order_ref, o.sys_id, inst, o.diretion, o.volume, o.filled_volume,  o.limit_price, o.status])
                res[field][gateway.gatewayName] = order_list
            elif field == 'Trades':
                trade_list = []
                for etrade in self.agent.etrades:
                    insts = ' '.join(etrade.instIDs)
                    volumes = ' '.join([str(i) for i in etrade.volumes])
                    filled_vol = ' '.join([str(i) for i in etrade.filled_vol])
                    filled_price = ' '.join([str(i) for i in etrade.filled_price])
                    trade_list.append([etrade.id, insts, volumes, filled_vol, filled_price, etrade.limit_price, etrade.valid_time,
                                  etrade.strategy, etrade.book, etrade.status])
                res[field] = trade_list
            elif field == 'Insts':
                if len(field_list) > 1:
                    insts = field_list[1:]
                else:
                    insts = self.agent.instruments.keys()
                inst_dict = {}
                for inst in insts:
                    inst_obj = self.agent.instruments[inst]
                    inst_dict[inst] = {'Name': inst, 'Price': inst_obj.price, 'MidPrice': inst_obj.mid_price,
                                 'BidPrice': inst_obj.bid_price1, 'BidVol': inst_obj.bid_vol1,
                                 'AskPrice': inst_obj.ask_price1, 'AskVol': inst_obj.ask_vol1,
                                 'PrevClose': inst_obj.prev_close, 'MarginRate': inst_obj.marginrate,
                                 'Updated': inst_obj.last_update, 'Traded': inst_obj.last_traded,
                                 'UpLimit': inst_obj.up_limit, 'DownLimit': inst_obj.down_limit,
                                 'Volume': inst_obj.volume, 'OI': inst_obj.open_interest }
                    if 'Option' in inst_obj.__class__.__name__:
                        inst_dict[inst]['PV'] = inst_obj.pv
                        inst_dict[inst]['Delta'] = inst_obj.delta
                        inst_dict[inst]['Gamma'] = inst_obj.gamma
                        inst_dict[inst]['Vega']  = inst_obj.vega
                        inst_dict[inst]['Theta'] = inst_obj.theta
                        inst_dict[inst]['Otype'] = inst_obj.otype
                        inst_dict[inst]['Strike'] = inst_obj.strike
                        inst_dict[inst]['Underlying'] = inst_obj.underlying
                        inst_dict[inst]['RiskPrice'] = inst_obj.risk_price
                        inst_dict[inst]['RiskUpdated'] = inst_obj.risk_updated
                res[field] = inst_dict
            elif field == 'Volgrids':
                prod = field_list[1]
                vg_param = {}
                if prod in self.agent.volgrids:
                    vg = self.agent.volgrids[prod]
                    for expiry in vg.volnode:
                        vg_param[expiry] = {}
                        vg_param[expiry]['Fwd'] = vg.fwd[expiry]
                        vg_param[expiry]['Updated'] = vg.last_update[expiry]
                        vg_param[expiry]['T2expiry'] = vg.t2expiry[expiry]
                        vg_param[expiry]['Under'] = vg.underlier[expiry]
                        vg_param[expiry]['Df'] = vg.df[expiry]
                        vol_param = vg.volparam[expiry]
                        vg_param[expiry]['Atm'] = vol_param[0]
                        vg_param[expiry]['V90'] = vol_param[1]
                        vg_param[expiry]['V75'] = vol_param[2]
                        vg_param[expiry]['V25'] = vol_param[3]
                        vg_param[expiry]['V10'] = vol_param[4]
                res[field] = {prod: vg_param}
            else:
                var = field2variable(field)
                res[field] = getattr(self.agent, var)
        return res

    def set_agent_params(self, params):
        for field in params:
            var = field2variable(field)
            value = params[field]
            setattr(self.agent, var, value)

    def set_gateway_params(self, gway, params):
        gateway = self.agent.gateways[gway]
        for field in params:
            var = field2variable(field)
            value = params[field]
            setattr(gateway, var, value)

    def get_strat_params(self, strat_name, fields):
        params = {}
        strat = self.agent.strategies[strat_name]
        for field in fields:
            var = field2variable(field)
            params[field] = getattr(strat, var)
        return params

    def set_strat_params(self, strat_name, params):
        strat = self.agent.strategies[strat_name]
        for field in params:
            var = field2variable(field)
            value = params[field]
            setattr(strat, var, value)

    def run_strat_func(self, strat_name, func_name):
        strat = self.agent.strategies[strat_name]
        getattr(strat, func_name)()

    def run_agent_func(self, func_name, params):
        return getattr(self.agent, func_name)(*params)

    def run_gateway_func(self, gway, func_name, params):
        gateway = self.agent.gateways[gway]
        return getattr(gateway, func_name)(*params)

    def get_hist_data(self, inst, freq, nlen = 20):
        data = []
        if freq in ['D', 'd']:
            if inst in self.agent.day_data:
                data = self.agent.day_data[inst].data[-nlen:]
        elif ('m' in freq) or ('M' in freq):
            f = int(freq[:-1])
            if (inst in self.agent.min_data) and (f in self.agent.min_data[inst]):
                data = self.agent.min_data[inst][f].data[-nlen:]
        return data

    def exit_agent(self):
        if self.agent != None:
            self.agent.exit()

def run_proxy_server(config_file, tday):
    with open(config_file, 'r') as infile:
        config = json.load(infile)
    name = config.get('name', 'test_agent')
    base.config_logging(name + "/" + name + ".log", level=logging.DEBUG,
                        format='%(name)s:%(funcName)s:%(lineno)d:%(asctime)s %(levelname)s %(message)s',
                        to_console=True,
                        console_level=logging.INFO)
    scur_day = datetime.datetime.strptime(tday, '%Y%m%d').date()
    rep_address = config.get('rep_address', 'tcp://*:10010')
    pub_address = config.get('pub_address', 'tcp://*:10020')
    proxy_app = ProxyRpcServer(rep_address, pub_address, config, scur_day)
    proxy_app.start()
    try:
        while 1:
            time.sleep(1)
    except KeyboardInterrupt:
        proxy_app.exit_agent()

def run_proxy_client(config_file, tday):
    pass

if __name__=="__main__":
    args = sys.argv[1:]
    app_name = args[0]
    params = (args[1], args[2],)
    getattr(sys.modules[__name__], app_name)(*params)


import os
import sys
import tensorflow as tf
import argparse
import train
import inference
import dataset_generator

class  ConfigInference(object):

    if __name__ == '__main__':
        print("parser")
        parser = argparse.ArgumentParser(
            description='Classification Network')

        parser.add_argument('--mode',
                            default='training',
                            help='training or Inference')
        parser.add_argument('--dataset_name', # "New Dataset Generation"으로 생성한 Dataset 폴더 이름.
                            default='TEST',
                            type=str)
        parser.add_argument('--pj_name', # 동일 Data set 하
                            default='128_train_set_test_2',
                            type=str)
        parser.add_argument('--cutmix',
                            default=False,
                            type=bool)
        #=============== Args For New Dataset Generation ================
        # Data set folder(--data_path) => Train folder, Validation folder, Dataset_Train.txt, Dataset_Valid.txt 생성
        # -> --dataset_name
        parser.add_argument('--generate_dataset',
                            default=None)
        parser.add_argument('--data_path',  # Data set folder (이미지가 저장된 폴더)
                            default="D:\\Dataset\\[Project] HKC H5 SAMPLE TEST IMAGE\\Dataset_v1\\",
                            type=str)
        parser.add_argument('--val_split',
                            default=0.25,
                            type=float)
        #==================================================================
        args = parser.parse_args()

        if args.generate_dataset != None:
            g_dataset = dataset_generator.Generate_dataset(args.data_path, args.dataset_name, args.val_split)
            g_dataset.Makedir()
            g_dataset.generate_dataset()

        # Configurations
        if args.mode == 'training':
            model = train.ModelTraining(
                mode=6,
                crop_size=224,
                batch_size=8,
                epoch=200,
                checkpoint_continue=0,
                learningRate=0.001,
                learningRateDecayFactor=0.9,
                args=args);

            model.setDataAugmentationRotation(False, False, False, False)
            model.setDataAugmentationGaussianBlur(False, 5, 0.5, 1.5)
            model.setDataAugmentationGaussianNoise(False, 0, 1)
            #model.setConfig(2)
            model.train()


# -*- coding: UTF-8 -*-

from random import getrandbits
from Crypto import Random
from Crypto.PublicKey.pubkey import bignum, getPrime

_g = 2

def GetBigPrime(bits = 300):
    "获取大素数"
    return bignum(getPrime(bits-1, Random.new().read))

class DiffieHellman:
    "p大素数,g底数,公钥可以公开，私钥_randnum要保密"
    def __init__(self, prime, bits = 300):
        self.__prime = prime
        while True:
            r = getrandbits(bits)
            if r < self.__prime:
                self.__randnum = r
                break
        self.pubkey = pow(_g, self.__randnum, self.__prime)

    def getPubkey(self):
        "获取公钥"
        return self.pubkey

    def getKey(self,otherpubkey):
        "获取最终的协商密钥"
        return pow(otherpubkey,self.__randnum, self.__prime)

if __name__ == '__main__':
    p = GetBigPrime()
    d1 = DiffieHellman(p)
    d2 = DiffieHellman(p)


# -*- coding: utf-8 -*-
from time import sleep
import pygame_widgets
from pygame_widgets.button import Button
import pygame
import os
import sys
import sqlite3

FPS = 30

# Переменные
moves = 0  # Шаги
level = 0  # Начальный уровень
places = {}  # Ключ = координаты Значение = 0/1(бутылка на месте/нет)
tile_width = tile_height = 50  # Размер клетки
clock = pygame.time.Clock()
con = sqlite3.connect('sokoban_db.sqlite')
cur = con.cursor()
moveLeft, moveRight, moveUp, moveDown = False, False, False, False
running = True
moving = False
end = False

# Группы спрайтов
all_sprites = pygame.sprite.Group()
tiles_group = pygame.sprite.Group()
player_group = pygame.sprite.Group()
bottles_group = pygame.sprite.Group()

# Загрузка изображений
def load_image(name):
    fullname = os.path.join('data', name)
    image = pygame.image.load(fullname)
    return image

# Изображения
tile_images = {  # Клеток
    'wall': load_image('brickwall.png'),
    'empty': load_image('floor.png'),
    'place': load_image('place.png')
}
player_image = load_image('worker.png')  # Игрока
bottle_image = load_image('bottle.png')  # Бутылки

pygame.init()  # Инициализация
pygame.display.set_caption('Кладовщик')  # Название
size = width, height = 1000, 750  # Размер окна
screen = pygame.display.set_mode(size)
icon = pygame.image.load('data/icon.png')  # Иконка игры
pygame.display.set_icon(icon)
pygame.mixer.music.load('data/sounds/music.mp3')  # Музыка
pygame.mixer.music.play(-1)
pygame.mixer.music.set_volume(0.025)

# Загрузка уровней
def load_level(filename):
    filename = "data/levels/" + filename
    with open(filename, 'r') as mapFile:
        level_map = [line.strip() for line in mapFile]
    max_width = max(map(len, level_map))
    return list(map(lambda x: x.ljust(max_width, '.'), level_map))

# Список уровней
levels = [load_level('level1.txt'),
          load_level('level2.txt'),
          load_level('level3.txt'),
          load_level('level4.txt'),
          load_level('level5.txt')]

# Выход из игры
def terminate():
    pygame.quit()
    sys.exit()

# Клетка
class Tile(pygame.sprite.Sprite):
    def __init__(self, tile_type, pos_x, pos_y):
        super().__init__(tiles_group, all_sprites)
        self.image = tile_images[tile_type]
        self.rect = self.image.get_rect().move(
            tile_width * pos_x, tile_height * pos_y)

# Игрок
class Player(pygame.sprite.Sprite):
    def __init__(self, pos_x, pos_y):
        super().__init__(player_group, all_sprites)
        self.image = player_image
        self.x = pos_x
        self.y = pos_y
        self.rect = self.image.get_rect().move(
            tile_width * self.x, tile_height * self.y)

    # Перемещение игрока
    def move(self, x, y):
        self.x = x
        self.y = y
        self.rect = self.image.get_rect().move(
            tile_width * self.x, tile_height * self.y)

# Бутылка
class Bottle(pygame.sprite.Sprite):
    def __init__(self, pos_x, pos_y):
        super().__init__(bottles_group, all_sprites)
        self.image = bottle_image
        self.x = pos_x
        self.y = pos_y
        self.rect = self.image.get_rect().move(
            tile_width * self.x, tile_height * self.y)

    # Перемещение бутылки
    def move(self, x, y):
        self.x = x
        self.y = y
        self.rect = self.image.get_rect().move(
            tile_width * self.x, tile_height * self.y)

    # Тоже перемещение бутылки
    def update(self, x, y):
        self.move(x, y)

        # Проверка на месте
        if cur_level[y][x] == '&':
            places[f'{x} {y}'] = 1

# Генерация уровня
def generate_level(lvl):
    new_player, x, y = None, None, None
    for y in range(len(lvl)):
        for x in range(len(lvl[y])):
            if lvl[y][x] == '.':
                Tile('empty', x, y)
            elif lvl[y][x] == '#':
                Tile('wall', x, y)
            elif lvl[y][x] == '@':
                Tile('empty', x, y)
                new_player = Player(x, y)
            elif lvl[y][x] == '$':
                Tile('empty', x, y)
                Bottle(x, y)
            elif lvl[y][x] == '&':
                places[f'{x} {y}'] = 0
                Tile('place', x, y)
    return new_player, x, y

font = pygame.font.Font(None, 50)
cur_level = levels[level]  # Текущий уровень в списке
player, level_x, level_y = generate_level(cur_level)  # Генерация
all_sprites.draw(screen)

# Звуки
btl_move_sound = pygame.mixer.Sound("data/sounds/bottle_move.mp3")
nxt_lvl_sound = pygame.mixer.Sound("data/sounds/next_lvl.mp3")
plyr_move_sound = pygame.mixer.Sound("data/sounds/player_move.mp3")
btl_move_sound.set_volume(0.15)
nxt_lvl_sound.set_volume(0.2)
plyr_move_sound.set_volume(0.12)

# Концовка игры
def end_of_the_game():
    res_button.hide()
    end_background = pygame.transform.scale(load_image('end.jpg'), (width, height))
    screen.blit(end_background, (0, 0))
    screen.blit(font.render(f'Вы прошли все уровни!', True,
                            'black'), (300, 300))
    plyr_move_sound.set_volume(0)
    btl_move_sound.set_volume(0)
    for i in all_sprites:
        i.kill()

# Функция перехода на следцющий уровень по кнопке
def next_level():
    # Глобал :(
    global places, end, level, cur_level, moves, \
        player, level_x, level_y, all_sprites
    nxt_lvl_sound.play()
    button.hide()
    places = {}
    end = False
    level += 1
    cur_level = levels[level]
    moves = 0
    for sprite in all_sprites:
        sprite.kill()
    player, level_x, level_y = generate_level(cur_level)
    all_sprites.draw(screen)

# Рестарт
def restart_level():
    # Глобал :(
    global places, end, level, cur_level, moves, \
        player, level_x, level_y, all_sprites
    places = {}
    button.hide()
    end = False
    cur_level = levels[level]
    moves = 0
    for sprite in all_sprites:
        sprite.kill()
    player, level_x, level_y = generate_level(cur_level)
    all_sprites.draw(screen)

# Кнопка перехода дальше
button = Button(
    screen,
    840, 5, 150, 40, text='Дальше', fontSize=40,
    inactiveColour='yellow', hoverColour='yellow', onClick=next_level)
button.hide()  # Прячем кнопку

# Кнопка рестарта
# Любая ошибка в уровне = рестарт
res_button = Button(
    screen,
    240, 5, 150, 40, text='Рестарт', fontSize=40,
    inactiveColour='yellow', hoverColour='yellow',
    pressedColour='green', onClick=restart_level)

# Выход в главное меню
def go_main_menu():
    res_button.hide()
    button.hide()
    go_menu_button.hide()
    main_menu()

go_menu_button = Button(
    screen,
    10, 705, 200, 40, text='Главное меню', fontSize=40,
    inactiveColour='yellow', hoverColour='yellow',
    pressedColour='green', onClick=go_main_menu)

# Главное меню
def main_menu():
    global level
    background = load_image('fon.jpg')
    screen.blit(background, (0, 0))
    res_button.hide()
    go_menu_button.hide()
    # Кнопка "продолжить"
    cont_button = Button(screen, 410, 200, 300, 80,
                         text='Продолжить уже начатую', fontSize=33,
                         inactiveColour='yellow', hoverColour='yellow')
    cont_button.hide()
    # Кнопка "начать"
    start_button = Button(screen, 100, 200, 300, 80,
                          text='Начать новую игру', fontSize=45,
                          inactiveColour='yellow', hoverColour='yellow')
    # Кнопка "выход"
    exit_button = Button(screen, 100, 300, 300, 80,
                         text='Выход из игры', fontSize=55,
                         inactiveColour='yellow', hoverColour='yellow')
    # Проверка на уже пройденные уровни
    result = cur.execute("""SELECT id_lvl FROM score WHERE moves = 0""")\
        .fetchall()
    if 0 < len(result) < 5:
        cont_button.show()

    while True:
        events = pygame.event.get()
        for event in events:
            if event.type == pygame.QUIT:
                terminate()
            elif event.type == pygame.MOUSEBUTTONDOWN:
                if 100 <= event.pos[0] <= 400 and \
                        200 <= event.pos[1] <= 280:  # Нажатие на кнопку
                    start_button.hide()
                    exit_button.hide()
                    cont_button.hide()
                    btl_move_sound.set_volume(0.15)
                    plyr_move_sound.set_volume(0.1)
                    level = 0
                    cur.execute("""UPDATE score SET moves = 0
                    WHERE moves > 0""")
                    con.commit()
                    restart_level()
                    return
                elif (410 <= event.pos[0] <= 710 and
                        200 <= event.pos[1] <= 280) and \
                        cont_button.isVisible():  # Нажатие на кнопку 2
                    start_button.hide()
                    exit_button.hide()
                    cont_button.hide()
                    level = result[0][0] - 1
                    restart_level()
                    return
                elif 100 <= event.pos[0] <= 400 and \
                        300 <= event.pos[1] <= 380:  # Нажатие на кнопку 3
                    terminate()
        pygame_widgets.update(events)
        pygame.display.flip()
        clock.tick(FPS)

main_menu()  # Отображние заставки

# Цикл игры
while running:
    events = pygame.event.get()
    for event in events:
        go_menu_button.show()
        res_button.show()
        # Выход из игры(тоже в начале!?)
        if event.type == pygame.QUIT:
            terminate()

        # Управление
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                moveRight = False
                moveLeft = True
            if event.key == pygame.K_RIGHT:
                moveLeft = False
                moveRight = True
            if event.key == pygame.K_UP:
                moveDown = False
                moveUp = True
            if event.key == pygame.K_DOWN:
                moveUp = False
                moveDown = True
        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT:
                moveLeft = False
            if event.key == pygame.K_RIGHT:
                moveRight = False
            if event.key == pygame.K_UP:
                moveUp = False
            if event.key == pygame.K_DOWN:
                moveDown = False
    # Тут очень сложно
    # Итак: передвижение игрока, передвижение бутылки, изменение счеткика,
    # проверка на стенку, проверка на бутылку, проверка на 2 бутылки рядом
    # проверка на месте, проверка на 2 места рядом
    if moveLeft:
        # не стена,не стена,игрок?
        if cur_level[player.y][player.x - 1] != '#' and \
                cur_level[player.y][player.x - 2] != '#':
            # игрок,не стена,не стена
            player.move(player.x - 2, player.y)
            # игрок+бутылка,не стена,не стена?
            if pygame.sprite.spritecollideany(player, bottles_group):
                # бутылка,игрок,не стена
                player.move(player.x + 1, player.y)
                # бутылка,игрок+бутылка,не стена?
                if pygame.sprite.spritecollideany(player, bottles_group):
                    # бутылка,бутылка,игрок(значит стоит на месте)
                    player.move(player.x + 1, player.y)
                # бутылка,игрок,не стена
                else:
                    # счетчик ходов + 1
                    moves += 1
                    plyr_move_sound.play()
            # не бутылка+игрок,не стена,не стена
            else:
                # бутылка,бутылка,игрок(значит стоит на месте)
                player.move(player.x + 2, player.y)
                # налево
                moving = True
        else:
            # налево
            moving = True
        # Движение
        if moving:
            # не стена, игрок?
            if cur_level[player.y][player.x - 1] != '#':
                # игрок,не стена
                player.move(player.x - 1, player.y)
                # игрок+бутылка,не стена?
                if pygame.sprite.spritecollideany(player, bottles_group):
                    #  не стена,игрок+бутылка,не стена?
                    if cur_level[player.y][player.x - 1] != '#':
                        # бутылка,игрок,не стена
                        pygame.sprite.spritecollideany(player,
                                                       bottles_group)\
                            .update(player.x - 1, player.y)
                        # место,игрок?
                        if cur_level[player.y][player.x - 1] == '&':
                            # место = 1(бутылка на месте)
                            places[f'{player.x - 1} {player.y}'] = 1
                            # место+игрок?
                            if cur_level[player.y][player.x] == '&':
                                # место+игрок = 0;место = 1,игрок
                                places[f'{player.x} {player.y}'] = 0
                                places[f'{player.x - 1} {player.y}'] = 1
                        # не место,игрок
                        else:
                            # место+игрок?
                            if cur_level[player.y][player.x] == '&':
                                # место+игрок = 0
                                places[f'{player.x} {player.y}'] = 0
                        # счетчик ходов + 1
                        moves += 1
                        btl_move_sound.play()
                    # стена,игрок+бутылка,не стена
                    else:
                        # стена,бутылка,игрок(стоим на месте)
                        player.move(player.x + 1, player.y)
                else:
                    # счетчик ходов + 1
                    plyr_move_sound.play()
                    moves += 1
        # задержка между ходами
        sleep(0.2)
        # нет движению!
        moving = False

    # и так по аналогии со всеми
    if moveRight:
        if cur_level[player.y][player.x + 1] != '#' and \
                cur_level[player.y][player.x + 2] != '#':
            player.move(player.x + 2, player.y)
            if pygame.sprite.spritecollideany(player, bottles_group):
                player.move(player.x - 1, player.y)
                if pygame.sprite.spritecollideany(player, bottles_group):
                    player.move(player.x - 1, player.y)
                else:
                    moves += 1
                    plyr_move_sound.play()
            else:
                player.move(player.x - 2, player.y)
                moving = True
        else:
            moving = True
        if moving:
            if cur_level[player.y][player.x + 1] != '#':
                player.move(player.x + 1, player.y)
                if pygame.sprite.spritecollideany(player, bottles_group):
                    if cur_level[player.y][player.x + 1] != '#':
                        pygame.sprite.spritecollideany(player,
                                                       bottles_group).\
                            update(player.x + 1, player.y)
                        if cur_level[player.y][player.x + 1] == '&':
                            places[f'{player.x + 1} {player.y}'] = 1
                            if cur_level[player.y][player.x] == '&':
                                places[f'{player.x} {player.y}'] = 0
                                places[f'{player.x + 1} {player.y}'] = 1
                        else:
                            if cur_level[player.y][player.x] == '&':
                                places[f'{player.x} {player.y}'] = 0
                        moves += 1
                        btl_move_sound.play()
                    else:
                        player.move(player.x - 1, player.y)
                else:
                    moves += 1
                    plyr_move_sound.play()
        sleep(0.2)
        moving = False

    if moveUp:
        if cur_level[player.y - 1][player.x] != '#' and \
                cur_level[player.y - 2][player.x] != '#':
            player.move(player.x, player.y - 2)
            if pygame.sprite.spritecollideany(player, bottles_group):
                player.move(player.x, player.y + 1)
                if pygame.sprite.spritecollideany(player, bottles_group):
                    player.move(player.x, player.y + 1)
                else:
                    moves += 1
                    plyr_move_sound.play()
            else:
                player.move(player.x, player.y + 2)
                moving = True
        else:
            moving = True
        if moving:
            if cur_level[player.y - 1][player.x] != '#':
                player.move(player.x, player.y - 1)
                if pygame.sprite.spritecollideany(player, bottles_group):
                    if cur_level[player.y - 1][player.x] != '#':
                        pygame.sprite.spritecollideany(player,
                                                       bottles_group)\
                            .update(player.x, player.y - 1)
                        if cur_level[player.y - 1][player.x] == '&':
                            places[f'{player.x} {player.y - 1}'] = 1
                            if cur_level[player.y][player.x] == '&':
                                places[f'{player.x} {player.y}'] = 0
                                places[f'{player.x} {player.y - 1}'] = 1
                        else:
                            if cur_level[player.y][player.x] == '&':
                                places[f'{player.x} {player.y}'] = 0
                        moves += 1
                        btl_move_sound.play()
                    else:
                        player.move(player.x, player.y + 1)
                else:
                    moves += 1
                    plyr_move_sound.play()
        sleep(0.2)
        moving = False

    if moveDown:
        if cur_level[player.y + 1][player.x] != '#' and \
                cur_level[player.y + 2][player.x] != '#':
            player.move(player.x, player.y + 2)
            if pygame.sprite.spritecollideany(player, bottles_group):
                player.move(player.x, player.y - 1)
                if pygame.sprite.spritecollideany(player, bottles_group):
                    player.move(player.x, player.y - 1)
                else:
                    moves += 1
                    plyr_move_sound.play()
            else:
                player.move(player.x, player.y - 2)
                moving = True
        else:
            moving = True
        if moving:
            if cur_level[player.y + 1][player.x] != '#':
                player.move(player.x, player.y + 1)
                if pygame.sprite.spritecollideany(player, bottles_group):
                    if cur_level[player.y + 1][player.x] != '#':
                        pygame.sprite.spritecollideany(player,
                                                       bottles_group).\
                            update(player.x, player.y + 1)
                        if cur_level[player.y + 1][player.x] == '&':
                            places[f'{player.x} {player.y + 1}'] = 1
                            if cur_level[player.y][player.x] == '&':
                                places[f'{player.x} {player.y}'] = 0
                                places[f'{player.x} {player.y + 1}'] = 1
                        else:
                            if cur_level[player.y][player.x] == '&':
                                places[f'{player.x} {player.y}'] = 0
                        moves += 1
                        btl_move_sound.play()
                    else:
                        player.move(player.x, player.y - 1)
                else:
                    moves += 1
                    plyr_move_sound.play()
        sleep(0.2)
        moving = False
    # Ураа! Вся основная механика есть
    # Рисуем спрайты и счетчик:
    clock.tick(FPS)
    screen.fill((0, 0, 0))
    all_sprites.draw(screen)
    bottles_group.draw(screen)
    player_group.draw(screen)
    screen.blit(font.render(f'Шагов: {moves}', True, 'black'), (10, 10))

    # Проверка все ли бутылки на месте
    if all(places.values()):
        if level < len(levels) - 1:  # Еще есть уровни:
            end = True
        else:  # Был последний уровень:
            cur.execute(f"""UPDATE score SET moves = {moves}
                                            WHERE id_lvl = {level + 1}""")
            con.commit()
            end_of_the_game()

    # Конец уровня
    if end:
        screen.blit(font.render('Уровень пройден!', True, 'white'),
                    (450, 10))
        cur.execute(f"""UPDATE score SET moves = {moves}
                                WHERE id_lvl = {level + 1}""")
        con.commit()
        button.show()
    # Смена кадра
    pygame_widgets.update(events)
    pygame.display.flip()
pygame.quit()


import os
import pandas as pd
import yfinance as yf
import datetime as dt

try:
    from Features import create_features
except:
    from Database.Features import create_features

import traceback

class DATABASE:

    def __init__(self):
        pass

    def update_data(self, ticker):
        try:
            ticker_ = yf.Ticker(ticker)
            df = ticker_.history(period="10y")

            # Creating Features and Target variable
            df = create_features(df)

            file_path = "C://Users/Rachel/Desktop/FP/Database/Datasets/" + ticker + '.csv'
            df.to_csv(file_path)

            return 1

        except:
            print("Error in creating Initial Data for {}".format(ticker))
            print(traceback.print_exc())
            return 0

    def get_current_day_data(self, ticker):
        try:
            update_status = self.update_data(ticker)

            if update_status == 1:
                file_path = "C://Users/Rachel/Desktop/FP/Database/Datasets/" + ticker + '.csv'
                df = pd.read_csv(file_path)
                current_df = df.tail(1)
                return current_df

        except:
            print("Error in getting current day data for {}".format(ticker))
            print(traceback.print_exc())
            return -2

"""
database = DATABASE()
#update_status = database.update_data("AAPL")
#print("update_status: ", update_status)
current_day_data = database.get_current_day_data("AAPL")
# Checking if Scaled Data working
import pickle
import joblib
ct1 = joblib.load("C://Users/Rachel/Desktop/FP/Model/Scaler/AAPL")
technical_indicators = ["volume", "SMA_10", "SMA_20", "SMA_50", "SMA_100", "SMA_200", "EMA_10", "EMA_20", "EMA_50", "EMA_100", "EMA_200", "ATR", "ADX", "CCI", "ROC", "RSI", "Williams%R", "SO%K"]
current_day_data = current_day_data[technical_indicators].copy()
scaled_processed = ct1.transform(current_day_data)
print("scaled_processed: ",scaled_processed)
model_file_path = "C://Users/Rachel/Desktop/FP/Model/Model_Pickle/" + "AAPL" + '.pkl'
model = pickle.load(open(model_file_path, 'rb'))
prediction = model.predict(scaled_processed)
print("prediction: ", prediction)
"""


import argparse
import os
import re
import subprocess
import tempfile

import at28c256
import bootstrap

class Label:
    def __init__(self, label):
        self.label = label

    def __str__(self):
        return f'Label[{self.label}]'

class LabelRef:
    def __init__(self, label):
        self.label = label

    def __str__(self):
        return f'LabelRef[{self.label}]'

class Memory:
    def __init__(self, ref):
        self.ref = ref

    def __str__(self):
        return f'Memory[{self.ref}]'

class Register:
    def __init__(self, reg):
        reg = reg.upper()
        if not reg in ['A', 'B', 'C', 'DL', 'DH', 'D', 'SEG']:
            raise Exception(f'Invalid Register Specified {reg}')
        self.reg = reg
        self.reg_offset = {'A': 0, 'B': 1, 'C': 2, 'DL': 3, 'DH': 4, 'D': 5, 'SEG': 6}[reg]

    def __str__(self):
        return f'Register[{self.reg}]'

class Value:
    def __init__(self, val):
        self.val = val

    def __str__(self):
        return f'Value[{self.val}]'

def assemble(tokens):
    for i in range(0, len(tokens)):
        mneumonic, params = tokens[i][0], tokens[i][1:]

        if type(mneumonic) == Label:
            continue
        elif type(mneumonic) != str:
            raise Exception(f'Unknown Mneumonic {mneumonic}')

        mneumonic = mneumonic.upper()

        translation = []

        if mneumonic == 'HLT':
            translation.append(0x01)
        elif mneumonic == 'NOP':
            translation.append(0x00)
        elif mneumonic == 'OUT':
            translation.append(0x43)

        elif mneumonic == 'LD':
            if len(params) != 2:
                raise Exception(f'Syntax: LD SRC DST')
            src, dst = params

            if type(dst) != Register:
                raise Exception(f'LD DST Register Expected {dst}')

            if type(src) == Value:
                translation.append(0x09 + dst.reg_offset)
                translation.append(src.val)
            elif type(src) == Memory:
                src = src.ref
                if type(src) != Value:
                    raise Exception(f'LD SRC Memory Requires Fixed Address {src}')
                translation.append(0x02 + dst.reg_offset)
                translation.append(src.val)
        elif mneumonic == 'MV':
            if len(params) != 2:
                raise Exception(f'Syntax: MV SRC DST')
            src, dst = params

            if type(src) != Register:
                raise Exception(f'MV SRC Register Expected')
            elif type(dst) != Register:
                raise Exception(f'MV DST Register Expected')

            translation.append(0x25 + 7 * src.reg_offset + dst.reg_offset)

        elif mneumonic == 'ST':
            if len(params) != 2:
                raise Exception(f'Syntax: ST SRC DST')
            src, dst = params

            if type(src) != Register:
                raise Exception(f'ST SRC Register Expected')
            elif type(dst) is not Memory:
                raise Exception(f'ST DST Memory Expected')

            dst = dst.ref
            if type(dst) not in [Register, Value]:
                raise Exception(f'ST DST Memory *Register/*Value')

            if type(dst) == Value:
                translation.append(0x10 + src.reg_offset)
                translation.append(dst.val)
            else:
                if dst.reg not in ['C', 'D']:
                    raise Exception(f'ST DST Register Must Be C or D')
                if dst.reg == 'C':
                    translation.append(0x17 + src.reg_offset)
                else:
                    translation.append(0x1e + src.reg_offset)

        elif mneumonic == 'ADD':
            translation.append(0x41)
        elif mneumonic == 'SUB':
            translation.append(0x42)

        elif mneumonic == 'JMP':
            if len(params) != 1:
                raise Exception(f'Syntax: JMP TARGET')
            target = params[0]
            if type(target) not in [LabelRef, Memory, Register]:
                raise Exception('JMP TARGET Memory/Label/Register Expected')

            if type(target) == Register:
                translation.append(0x45 + target.reg_offset)
            else:
                translation.append(0x44)
                if type(target) == Memory:
                    translation.append(target.addr)
                else:
                    translation.append(target)

        elif mneumonic in ['JC', 'JZ']:
            if len(params) != 1:
                raise Exception(f'Syntax: JC/JZ TARGET')
            target = params[0]
            if type(target) not in [LabelRef, Memory]:
                raise Exception('JMP TARGET Memory/Label Expected')
            translation.append({'JC': 0x4c, 'JZ': 0x4d}[mneumonic])
            if type(target) == Memory:
                translation.append(target.addr)
            else:
                translation.append(target)

        else:
            raise Exception(f'Unknown Mneumonic {mneumonic}')

        tokens[i] = translation

    return tokens

def link(object_code):
    labels = {}
    machine_code = []
    offset = 0
    for i in range(0, len(object_code)):
        if len(object_code[i]) == 1 and type(object_code[i][0]) == Label:
            labels[object_code[i][0].label] = offset
            continue
        machine_code.append(object_code[i])
        offset += len(object_code[i])
    for i in range(0, len(machine_code)):
        for j in range(0, len(machine_code[i])):
            if type(machine_code[i][j]) == LabelRef:
                if not machine_code[i][j].label in labels:
                    raise Exception(f'Unmatched Label Reference {machine_code[i][j].label}')
                machine_code[i][j] = labels[machine_code[i][j].label]

    return machine_code

def read_lines(source_file):
    with open(source_file, 'r') as f:
        file_lines = f.readlines()

    for i in range(0, len(file_lines)):
        l = file_lines[i]
        l = l.strip()
        l = l.split('#', maxsplit=1)[0]
        file_lines[i] = l.strip()

    file_lines = [l for l in file_lines if len(l)]
    return file_lines

def parse(file_lines):
    tokens = []

    def _translate(t):
        if t.startswith('*'):
            t = _translate(t[1:])
            if type(t) not in [Register, Value]:
                raise Exception(f'Memory Syntax Error: *0x10')
            t = Memory(t)
        elif t.isdigit():
            t = Value(int(t))
        elif t.startswith('0x') and re.match('[0-9a-fA-F]+', t[2:]):
            t = Value(int(t, 16))
        elif re.match('^[a-zA-Z]{1}[a-zA-Z0-9\-_]+:$', t):
            t = Label(t[:-1])
        elif t.startswith('R'):
            t = Register(t[1:])
        elif j > 0 and re.match('^[a-zA-Z]{1}[a-zA-Z0-9\-_]+$', t):
            t = LabelRef(t)
        return t

    for i in range(0, len(file_lines)):
        l_tokens = file_lines[i].split(' ')
        for j in range(0, len(l_tokens)):
            l_tokens[j] = _translate(l_tokens[j])

        tokens.append(l_tokens)

    return tokens

def print_assembly(file_lines, machine_lines, machine_code):
    print()

    j = 0
    k = 0
    for i in range(0, len(file_lines)):
        if re.match('^[a-zA-Z]{1}[a-zA-Z0-9\-_]+:$', file_lines[i]):
            print(file_lines[i])
        else:
            machine_line = ' '.join([f'{v:02x}' for v in machine_lines[j]])
            pad = (5 - len(machine_line)) * ' '
            print(f'  [ {k:04x} ]  {machine_line}{pad}    {file_lines[i]}')
            k += len(machine_lines[j])
            j += 1

    print()
    at28c256.hexdump(machine_code)
    print()

def run(args):
    source_file = args.file
    bin_file = os.path.splitext(args.file)[0] + '.bin'

    file_lines = read_lines(source_file)
    tokens = parse(file_lines)
    object_lines = assemble(tokens)
    machine_lines = link(object_lines)
    machine_code = b''.join([bytes(b) for b in machine_lines])

    print_assembly(file_lines, machine_lines, machine_code)

    if args.emulator:
        t = tempfile.NamedTemporaryFile('wb', delete=False)
        t.write(machine_code)
        t.close()
        try:
            p = subprocess.run([args.emulator, t.name])
        finally:
            os.unlink(t.name)
    if args.out:
        with open(args.out, 'wb') as f:
            f.write(machine_code)

    if args.port:
        with bootstrap.Bootstrap(args.port) as b:
            b.write_range(0, machine_code)
        print()

    if args.romh:
        machine_code_hex = ['0x%02x' % b for b in machine_code]
        with open(args.romh, 'w') as f:
            print('const uint8_t INIT_PROGRAM[] PROGMEM = {', file=f)
            for i in range(0, len(machine_code_hex), 16):
                print('    ', ', '.join(machine_code_hex[i:i+16]), ',' if i + 16 < len(machine_code_hex) else '', sep='', file=f)
            print('};', file=f)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Assembler.')
    parser.add_argument('--emulator', dest='emulator', default='', type=str)
    parser.add_argument('--out', dest='out', default='', type=str)
    parser.add_argument('--program', dest='port', default='', type=str)
    parser.add_argument('--romh', dest='romh', default='', type=str)
    parser.add_argument('file', type=str)
    args = parser.parse_args()

    if not args.file:
        raise Exception(f'Invalid File {args.file}')

    args.file = os.path.abspath(args.file)
    if not os.path.exists(args.file):
        raise Exception(f'Cannot File File {args.file}')

    run(args)


# wav, マイク, (gps:未実装)の配信クライアント
# wavファイルのみの配信と、wav+マイクの配信に対応

from pyproj import Transformer
import matplotlib.pyplot as plt
import logging
from subprocess import TimeoutExpired
import math
import numpy as np
import socket
from threading import Thread

from ENV import DISABLE_HIT_JUDGE, MAX_HOST, DEBUG as DEBUG_MODE, DEVICE_TYPE, NETWORK_ADDRESS, HOST_ADDRESS_START, MAX_X, MIN_X, MAX_Y, MIN_Y

DUMMY_BYTE_TYPE = np.float64

def getMyIp():
    host_addr = ""
    for i in range(HOST_ADDRESS_START, HOST_ADDRESS_START+MAX_HOST):
        try:
            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
                sock.bind((NETWORK_ADDRESS+str(i), 12345))
                sock.listen(5)
                host_addr = NETWORK_ADDRESS+str(i)
                sock.close()
            break
        except Exception:
            continue
    return host_addr

class MasterServer(Thread):

    def __init__(self, server_host, server_port):
        Thread.__init__(self)
        self.SERVER_HOST = server_host
        self.SERVER_PORT = int(server_port)
        self.daemon = True
        self.name = "MasterServer"
        self.last_message = "No incoming connection"
        self.socks: set[socket.socket] = set()
        self.recieves = dict()  # key:str "IPアドレス:ポート" value: lat, lonのタプル

    def getSocketList(self):
        return [':'.join([str(i) for i in s.getpeername()]) for s in self.socks]

    def run(self):
        # サーバーソケット生成
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_sock:
            server_sock.bind((self.SERVER_HOST, self.SERVER_PORT))
            # server_sock.settimeout(5)
            server_sock.listen(16)
            print(
                f"Listen Server listening on {self.SERVER_HOST}:{self.SERVER_PORT}")
            # クライアントと接続
            while True:
                client_sock, addr = server_sock.accept()
                hbuf, sbuf = socket.getnameinfo(
                    addr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV)
                client_sock.settimeout(5)  # 5秒でタイムアウト
                print(f"accept:{hbuf}:{sbuf}")
                t = Thread(target=self.recv, args=[
                           client_sock, hbuf, sbuf], daemon=True, name="recv")
                t.start()

    def recv(self, client_sock: socket.socket, ip: str, port: str):
        with client_sock:
            # 他スレッドから読まれることを考慮して、最初にrevieves辞書を更新しておく
            client_addr = ':'.join([str(i)
                                    for i in client_sock.getpeername()])
            self.recieves[client_addr] = {
                "lat": -2, "lon": -2, "course": 90}
            self.socks.add(client_sock)
            try:
                # クライアントからオーディオプロパティを受信
                settings_list = client_sock.recv(
                    256).decode('utf-8').split(",")
                CHANNEL = int(settings_list[0])
                FORMAT_BIT = int(settings_list[1])
                # RATE = int(settings_list[2])
                FRAMES = int(settings_list[3])
                DUMMY_FORMAT_BIT = int(settings_list[4])
                DUMMY_NUMBER_COUNT = int(settings_list[5])
                frame_length = CHANNEL * FORMAT_BIT // 8 * \
                    FRAMES  # フレームの長さ(バイト)
                dummy_length = DUMMY_FORMAT_BIT // 8 * \
                    DUMMY_NUMBER_COUNT  # ダミーデータの長さ(バイト)
                data_length = frame_length + dummy_length

                print(settings_list)

                # メインループ
                data = bytes()

                while True:
                    # クライアントから音データを受信
                    # なぜかクライアントがCHUNKの4倍量を送ってくるので合わせる。

                    data += (client_sock.recv(data_length))

                    # 切断処理
                    if not data:
                        self.socks.remove(client_sock)
                        break
                    if len(data) < data_length:  # データが必要量に達していなければなにもしない
                        continue
                    chunk = data[:data_length]  # 使用チャンク分だけ取り出す
                    data = data[data_length:]  # 今回使わないデータだけ残す
                    dummy_bytes = chunk[0:dummy_length]
                    dummy_arr = np.frombuffer(dummy_bytes, np.float64)
                    lat, lon, course = dummy_arr

                    self.recieves[client_addr] = {
                        "lat": lat, "lon": lon, "course": course}
                    print(client_addr, self.recieves[client_addr])

            except UnicodeDecodeError:
                self.socks.remove(client_sock)
                msg = f"DecodeError with {ip}:{port}, connection reset"
                print(msg)
                self.last_message = msg
            except TimeoutError:
                self.socks.remove(client_sock)
                msg = f"Connection with {ip}:{port} was timeout1"
                print(msg)
                self.last_message = msg
            except ConnectionResetError:
                self.socks.remove(client_sock)
                msg = f"Connection with {ip}:{port} was reset by peer"
                print(msg)
                self.last_message = msg
            except TimeoutExpired:
                self.socks.remove(client_sock)
                msg = f"Connection with {ip}:{port} was timeout2"
                print(msg)
                self.last_message = msg
            except OSError as e:
                self.socks.remove(client_sock)
                if e.errno == 113 or e.errno == None:
                    msg = f"Connection with {ip}:{port} was timeout3"
                    print(msg)
                    self.last_message = msg
                else:
                    msg = f"Unhandled Exception on SLS {e.errno}"
                    print(msg)
                    self.last_message = msg
            except Exception as e:
                self.socks.remove(client_sock)
                msg = f"Unhandled Exception on SLS {e}"
                print(msg)
                self.last_message = msg

    def course_convert(self, course: float):
        """NMEAのcouse(進行方向)を、真東0°、真北90°の角度に変換する"""
        return (course*-1+90+360) % 360

    def hit_sector(self, target_x: float, target_y: float, start_angle: float, end_angle: float, radius: float):
        """真東を0°、真北を90°とする"""
        if start_angle > end_angle:
            start_angle, end_angle = end_angle, start_angle
        dx = target_x - self.gps.lon  # X = longtitude 経度
        dy = target_y - self.gps.lat  # Y = latitude 緯度
        sx = math.cos(math.radians(start_angle))
        sy = math.sin(math.radians(start_angle))
        ex = math.cos(math.radians(end_angle))
        ey = math.sin(math.radians(end_angle))

        # 円の内外判定
        if dx ** 2 + dy ** 2 > radius ** 2:
            return False

        # 扇型の角度が180を超えているか
        if sx * ey - ex * sy > 0:
            # 開始角に対して左にあるか
            if sx * dy - dx * sy < 0:
                return False
            # 終了角に対して右にあるか
            if ex * dy - dx * ey > 0:
                return False
            # 扇型の内部にあることがわかった
            return True
        else:
            # 開始角に対して左にあるか
            if sx * dy - dx * sy >= 0:
                return True
            # 終了角に対して右にあるか
            if ex * dy - dx * ey <= 0:
                return True
            # 扇型の外部にあることがわかった
            return False

def main():
    try:
        # host_addr = getMyIp()
        host_addr = "192.168.86.21"
        ms = MasterServer(host_addr, 12345)
        ms.start()
        fig = plt.figure()
        ax = fig.add_subplot(111)
        # 6676 静岡県
        toSquare = Transformer.from_crs("epsg:4326", "epsg:6676")
        while True:
            # ax.set_xlim([135, 145])
            # ax.set_ylim([32.5, 37.5])
            sockets = ms.getSocketList()
            annotates = []
            for socket in sockets:
                lat = ms.recieves[socket]["lat"]
                lon = ms.recieves[socket]["lon"]
                course = ms.recieves[socket]["course"]
                rad = math.radians(ms.course_convert(course))
                y, x = toSquare.transform(lat, lon)
                target_y = lat + math.sin(rad)
                target_x = lon + math.cos(rad)
                host_addr = socket.split(":")[0].split(".")[-1]
                # ap = ax.plot(x, y, 'o', color="red", label=host_addr)
                ax.scatter(x, y, marker='o', label=host_addr)
                print(x, y)
                # ap = ax.plot(lon, lat, 'o', color="red", label=host_addr)
                # an = ax.annotate(host_addr, xy=(target_x, target_y), xytext=(lon, lat),
                #                  arrowprops=dict(shrink=0, width=1, headwidth=8,
                #                                  headlength=10, connectionstyle='arc3',
                #                                  facecolor='gray', edgecolor='gray')
                #                  )
                # an = ax.arrow(lon, lat, target_x-lon, target_y-lat, width=0.1)
                an = ax.arrow(x, y, math.cos(rad), math.sin(rad))
                # annotates.append(an)
            ax.set_xlim(auto=True)
            ax.set_ylim(auto=True)
            ax.legend()
            plt.pause(1)
            for annnotate in annotates:
                annnotate.remove()
            annotates.clear()
            ax.clear()
            continue
    except KeyboardInterrupt:
        print("Exit.")
        exit(0)

if __name__ == '__main__':
    main()


# import json
import sys
import os
import re
import gensim
import nltk
import main as MAIN
# import numpy as np
import xlsxwriter
from openpyxl import load_workbook
from keras.preprocessing import sequence
import csv
import xlrd
from gensim.models import Word2Vec
import applying_ML_algorithms
from datetime import datetime
import time as time_module
import Streamer
import time
import glob
import queue

convert_to_vec_file_exist = 0  # flag: prevent recreation of convert_to_vec file

STOPWORDS = nltk.corpus.stopwords.words('english')

emoticons_str = r"""
		(?:
			[:=;] # Eyes
			[oO\-]? # Nose (optional)
			[D\)\]\(\]/\\OpP] # Mouth
		)"""

regex_str = [
    emoticons_str,
    r'<[^>]+>',  # HTML tags
    r'(?:@[\w_]+)',  # @-mentions
    r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)",  # hash-tags
    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+',  # URLs
    r'(?:(?:\d+,?)+(?:\.?\d+)?)',  # numbers
    r"(?:[a-z][a-z'\-_]+[a-z])",  # words with - and '
    r'(?:[\w_]+)'  # other words
    # r'(?:\S)'  # anything else
]

number_str = r'(?:(?:\d+,?)+(?:\.?\d+)?)'
tokens_re = re.compile(r'(' + '|'.join(regex_str) + ')', re.VERBOSE | re.IGNORECASE)
emoticon_re = re.compile(r'^' + emoticons_str + '$', re.VERBOSE | re.IGNORECASE)
number_re = re.compile(r'^' + number_str + '$', re.VERBOSE | re.IGNORECASE)

header = []

def tokenize(s):
    return tokens_re.findall(s)

def preprocess(s):
    tokens = tokenize(s)
    # tokens = [token.lower() if emoticon_re.search(token) == False and token not in STOPWORDS else  for token in tokens]
    tokens = map(lambda token: token.lower(),
                 filter(lambda token: emoticon_re.search(token) is None
                                      and token not in STOPWORDS
                                      and token.find('http') == -1
                                      and number_re.search(token) is None
                        , tokens))

    return tokens

def load_google_word2vec_model():
    path = os.path.join(MAIN.project_temp_files_path, "GoogleNews-vectors-negative300.bin")
    google_model = gensim.models.KeyedVectors.load_word2vec_format(path, binary=True)
    return google_model

def how_many(str, char):
    count = 0
    for i in range(0, len(str)-1):
        if str[i] == char:
            count = count + 1
    return count

def convert_train_to_vec(train_path, google_model):
    # convert train file to vectors file and save it
    print("\nCONVERT_TO_VEC: converting train to vec")
    max_review_length = 300
    header = []
    train_to_vec_path = os.path.join(MAIN.project_temp_files_path, "train_to_vectors.csv")
    header.append('id')
    header.append('text')
    for k in range(1, 301):
        # str = 'v{}'.format(i-1)
        # header.append(locals()[str])
        my_str = "v%s" % (k)
        header.append(my_str)
    header.append('RT')

    feature_file = open(train_to_vec_path, 'w', encoding='utf-8')  # output file here
    feature_file.writelines(','.join(map(str, header)) + '\n')

    with open(train_path, 'r', encoding='ISO-8859-1') as f:
        reader = csv.reader(f)
        next(reader)
        for line in reader:
            id = line[0]
            text = line[1]  # [id,text]

            data_dict = line
            sentence = preprocess(data_dict[1])  # takes the text
            sentence_vector = []
            for word in sentence:
                try:
                    tmp_vec = google_model.word_vec(word).tolist()  # np.array([1,1,1,1]).tolist()
                    # tmp_vec = model.wv.word_vec(word).tolist()
                except:
                    e = sys.exc_info()[0]
                    print("CONVERT_TO_VEC: our model did not learn the word", word, "
Error: %s
" % e)
                    tmp_vec = []

                sentence_vector += tmp_vec

            dummy_list = []
            # dummy_list.append(data_dict[0])
            # dummy_list.append(data_dict[1])
            # dummy_list.append(data_dict[2])
            # dummy_list.append(data_dict[3])
            dummy_list.append(sentence_vector)
            sentence_vector = dummy_list
            # print("1")
            # print(sentence_vector)
            sentence_vector = sequence.pad_sequences(sentence_vector,
                                                     maxlen=max_review_length,
                                                     padding='post', truncating='post',
                                                     dtype='float32')
            sentence_vector = sentence_vector[0]
            text = '"' + re.sub('"', '', text) + '"'
            feature_file.writelines(
                id + ' ,' + text + ' ,' + ','.join(map(str, sentence_vector)) + ' ,' + line[2] + '\n')

def convert_my_tweets_to_vec(inpath, google_model):
    # convert tweet file to vectors file and save it
    print('\nCONVERT_TO_VEC: converting my tweets to vectors')

    max_review_length = 300
    header = []
    my_tweets_to_vec_path = os.path.join(MAIN.project_temp_files_path, "my_tweets_to_vectors.csv")
    header.append('id')
    header.append('text')
    header.append('geotag')
    header.append('time')
    for k in range(1, 301):
        my_str = "v%s" % k
        header.append(my_str)
    feature_file = open(my_tweets_to_vec_path, 'w', encoding='utf-8')  # output file here
    feature_file.writelines(','.join(map(str, header)) + '\n')

    #with open(inpath, 'r', encoding='ISO-8859-1') as f:
    wb = xlrd.open_workbook(inpath)
    sheet = wb.sheet_by_index(0)
    # sheet = wb.active
    maxRow = sheet.nrows
    for line in range(1, maxRow):
        id = '"' + str(sheet.cell_value(line, 0)) + '"'
        text = '"' + re.sub('"', '', str(sheet.cell_value(line, 1))) + '"'
        geotag = '"' + str(sheet.cell_value(line, 2)) + '"'
        time = '"' + str(sheet.cell_value(line, 3)) + '"'

        sentence = preprocess(text)  # takes the text
        sentence_vector = []
        for word in sentence:

            try:
                tmp_vec = google_model.word_vec(word).tolist()  # np.array([1,1,1,1]).tolist()
                # tmp_vec = model.wv.word_vec(word).tolist()
            except:
                e = sys.exc_info()[0]
                print("CONVERT_TO_VEC: our model did not learn the word", word, "
Error: %s
" % e)
                tmp_vec = []

            sentence_vector += tmp_vec
        dummy_list = []
        # dummy_list.append(data_dict[0])
        # dummy_list.append(data_dict[1])
        # dummy_list.append(data_dict[2])
        # dummy_list.append(data_dict[3])
        dummy_list.append(sentence_vector)
        sentence_vector = dummy_list
        sentence_vector = sequence.pad_sequences(sentence_vector,
                                                 maxlen=max_review_length,
                                                 padding='post', truncating='post',
                                                 dtype='float32')
        sentence_vector = sentence_vector[0]
        feature_file.writelines(
            id + ' ,' + text + ' ,' + geotag + ' ,' + time + ' ,' + ','.join(map(str, sentence_vector)) + '\n')
    feature_file.close()

def main1(my_tweets_path):  # file path = the MY_TWEETS file to convert
    google_model = load_google_word2vec_model()

    train_path = os.path.join(MAIN.project_temp_files_path, "train_new.csv")

    global convert_to_vec_file_exist
    if not convert_to_vec_file_exist:  # flag: train_to_vec already created before
        convert_train_to_vec(train_path, google_model)
        convert_to_vec_file_exist = 1

    convert_my_tweets_to_vec(my_tweets_path, google_model)

    applying_ML_algorithms.main1()

def fill_and_print_queue(fill=1):  # if fill = 1, the function fill the Queue according to folder 'Queue'

    queue_files_path = os.path.join(MAIN.queue_path, '*.xlsx')
    base = 40  # the width of the Queue to print
    first = 1

    for n in range(0, 2):
        i = 0
        print("|", end="")
        while i < int(2*base) - 2:
            i += 1
            print(" ", end="")
        print("|")

    if base % 2 != 0:
        base1 = int(base)
        base2 = int(base)
    else:
        base1 = int(base) - 1
        base2 = int(base) - 1
    i = 0

    print("|", end="")
    while i < base1 - 3:  # - "QUE
        i += 1
        print("-", end="")
    print("Queue", end="")
    i = 0
    while i < base2 - 2:  # - "UE
        i += 1
        print("-", end="")
    print("|")

    for n in range(0, 2):
        print("|", end="")
        i = 0
        while i < int(2*base) - 2:
            i += 1
            print(" ", end="")
        print("|")

    for file in glob.glob(queue_files_path):
        file_name = os.path.split(file)[1]
        if fill == 1:  # fill the Queue
            MAIN.Queue_pointer.put(file_name)  # add new file to queue, for convert_to_vec
        print('|', end="")
        len_ = int(len(file_name))
        if (len_ % 2) != 0:
            len1 = base - int(len_ / 2)
            len2 = base - (int(len_ / 2) + 1)
        else:
            len1 = len2 = base - int(len_ / 2)
        for i in range(1, len1):
            print(" ", end="")
        print(file_name, end="")
        for i in range(1, len2):
            print(" ", end="")
        if first == 1:
            print('|', end="")
            print("<---- HEAD, FIFO")
            first = 0
        else:
            print('|')
    for i in range(0, 2*base):
        print("-", end="")

def main():
    # check the queue every hour
    global convert_to_vec_file_exist
    convert_to_vec_file_exist = 0
    sleep_time = 3600  # time in sec, until the next Queue check
    fill_and_print_queue(1)  # fill Queue with files from "physical folder- 'Queue'"
    while True:
        empty = MAIN.Queue_pointer.empty()  # check if queue empty
        while not empty:  # while THERE IS READY FILES in queue (queue fills up by Streamer)
            poped_file_path = os.path.join(MAIN.project_temp_files_path, 'Queue')  # the file path to be analyzed
            pop = MAIN.Queue_pointer.get()  # pop new file_name to analyze from Queue
            print('\nCONVERT_TO_VEC: POP QUEUE TO ANALYSE, FILE NAME: ', pop)
            poped_file_path = os.path.join(poped_file_path, pop)  # the file path to be analyzed
            main1(poped_file_path)
            # --------delete file from physical folder----------
            os.remove(poped_file_path)
            # -----------------
            print("CONVERT_TO_VEC: Finish FILE: ", pop)
            fill_and_print_queue(0)  # print without filling Queue
            empty = MAIN.Queue_pointer.empty()  # check if queue empty
        if empty:
            print("\nCONVERT_TO_VEC: TIME IS: " + str(datetime.now().hour) + ":" + str(datetime.now().minute) + " 0 files in Queue")
            print("\n\n\n\n\n\n\n\n\n\n going to sleep for ", sleep_time, "sec")
            time.sleep(sleep_time)

if __name__ == '__main__':
    main()


import abc
from typing import Any, Generator, Iterable, List, NamedTuple, Optional, Union
import unittest

from nmigen import *
from nmigen.sim import *

from nmigen_nexys.core import edge
from nmigen_nexys.core import util
from nmigen_nexys.test import test_util

Event = Any  # TODO: Maybe a protocol type?

class TimestampedEvent(NamedTuple):
    cycle: int
    event: Event

class EdgeEvent(NamedTuple):
    """Simulation event: GPIO toggled."""
    signal: Signal
    direction: str  # TODO(python-3.8): Literal['rose', 'fell']

class DelayConstraint(object):

    def __init__(self, *, seconds: Optional[float] = None,
                 cycles: Optional[int] = None):
        super().__init__()
        if seconds is not None:
            assert cycles is None
            self.cycles = int(seconds * util.SIMULATION_CLOCK_FREQUENCY)
        else:
            assert cycles is not None
            self.cycles = cycles

class MinDelay(DelayConstraint):
    pass

class MaxDelay(DelayConstraint):
    pass

EventConstraints = Iterable[Union[Event, DelayConstraint]]

class EventSeries(object):

    def __init__(self, cycle_counter: Signal):
        super().__init__()
        self._cycle_counter = cycle_counter
        self._events: List[TimestampedEvent] = []

    def add(self, event: Event) -> Generator[None, Signal, int]:
        cycle = yield self._cycle_counter
        self._events.append(TimestampedEvent(cycle, event))

    def ShowEvents(self):
        if not self._events:
            print('No simulation events captured')
            return
        dbg_table = []
        for e1, e2 in zip([None] + self._events[:-1], self._events):
            diff = f'({e2.cycle - e1.cycle:+})' if e1 else ' '
            dbg_table.append((str(e2.cycle), diff, repr(e2.event)))
        widths = [max(len(row[col]) for row in dbg_table) for col in range(2)]
        dbg_out = ['Simulation events:']
        for ts, diff, disp in dbg_table:
            dbg_out.append(f'  {ts:>{widths[0]}} {diff:>{widths[1]}} {disp}')
        print('\n'.join(dbg_out))

    def ValidateConstraints(self, tc: unittest.TestCase,
                            expected: EventConstraints):
        actual_iter = iter(self._events)
        last_ts = 0
        delay_constraints = []
        for constraint in expected:
            if isinstance(constraint, DelayConstraint):
                delay_constraints.append(constraint)
                continue
            event = next(actual_iter)
            # tc.assertIsInstance(constraint, Event)
            tc.assertEqual(event.event, constraint)
            if delay_constraints:
                total = sum(c.cycles for c in delay_constraints)
                if all(isinstance(c, MinDelay) for c in delay_constraints):
                    tc.assertGreaterEqual(event.cycle - last_ts, total)
                elif all(isinstance(c, MinDelay) for c in delay_constraints):
                    tc.assertGreaterEqual(event.cycle - last_ts, total)
                else:
                    tc.fail('Invalid mix of delay constraints')
                delay_constraints = []
            last_ts = event.cycle

class Monitor(abc.ABC):

    def __init__(self):
        super().__init__()
        self._event_series = None

    def emit(self, event: Event) -> Generator[None, Signal, int]:
        yield from self._event_series.add(event)

    @abc.abstractmethod
    def process(self) -> test_util.CoroutineProcess[None]:
        pass

    def attach(self, sim: Simulator, event_series: EventSeries):
        self._event_series = event_series
        sim.add_sync_process(self.process)

class EdgeMonitor(Monitor):

    def __init__(self, detectors: Iterable[edge.Detector]):
        super().__init__()
        self._detectors = detectors

    def process(self) -> test_util.CoroutineProcess[None]:
        yield Passive()
        while True:
            yield
            for detector in self._detectors:
                if (yield detector.rose):
                    yield from self.emit(EdgeEvent(detector.input, 'rose'))
                if (yield detector.fell):
                    yield from self.emit(EdgeEvent(detector.input, 'fell'))


import json
from src.services.task_service import TaskService
from src.core_module import core

def resp_ajax_tasks_search(environ, response):
    if response is not None:
        qs = environ['QUERY_STRING']
        # core.dd(qs)
        search_str = qs.split("=")[1].replace("%20", " ")
        # core.dd(search_str)
        if qs is not None:
            headers = [('Content-Type', 'application/json')]
            status = '200 OK'
            response(status, headers)

            task_service = TaskService()
            tasks = task_service.get_filtered_tasks(search_str)
            ls = []
            for row in tasks:
                dict_el = {
                    'taskid': row[0],
                    'taskname': row[1],
                    'enddate': row[2],
                    'isregular': row[3],
                    'isclosed': row[4]
                }
                ls.append(dict_el)

            response = json.dumps(ls)  # converts dict(or list of dicts) to string
            return [response.encode()]
    return None


# Generic
import unittest

# Libs
from flask_testing import TestCase

# Own
from flask import current_app as app

class TestArtificialSongs(TestCase):
    def create_app(self):
        app.config.from_object('config.default')
        app.config.from_object('config.testing')
        return app

class TestArtificialSongEndPoints(TestArtificialSongs):
    def test_artificial_song_with_seed_get(self):
        self.create_app()
        seed = 1234
        words = 100
        artist = 1
        response = self.client.get('/api/v1/artificial_songs/generate/en/seed/{}/words/{}/artist/{}/'.format(seed, words, artist))
        print('\n\nResponse (test_artificial_song_with_seed_get): ' + str(response.status_code))
        self.assertEqual(response.status_code, 200)

    def test_artificial_song_no_seed_get(self):
        self.create_app()
        words = 100
        artist = 1
        response = self.client.get('/api/v1/artificial_songs/generate/en/words/{}/artist/{}/'.format(words, artist))
        print('\n\nResponse (test_artificial_song_no_seed_get): ' + str(response.status_code))
        self.assertEqual(response.status_code, 200)

    def test_all_artificial_songs_get(self):
        response = self.client.get('/api/v1/artificial_songs/')
        print('\n\nResponse (test_all_artificial_songs_get): ' + str(response.status_code))
        self.assertEqual(response.status_code, 200)

    def test_one_artificial_songs_get(self):
        artificial_song_id = 1
        response = self.client.get('/api/v1/artificial_songs/{}'.format(artificial_song_id))
        print('\n\nResponse: ' + str(response.status_code))
        self.assertEqual(response.status_code, 200)

if __name__ == '__main__':
    unittest.main()


# Implementation for
# Comparator-learning Combinatorial Functions (JMLR, 2016)
# @author: Yuhao Wan

import numpy as np
import pandas as pd
from functools import cmp_to_key
import math
import copy
from sklearn.linear_model import perceptron
from sklearn.metrics import accuracy_score

class pairwise_oracle():
  # Implementation of pairwise comparison oracle

    def __init__(self, preference_list):
        """ taken in an ordered list as given preference
        the list is in ascending order (so later in the list means more preferred)
        assuming strict preference for now """
        self.preference = np.array(preference_list)

    def compare(self, s1, s2):
        """ output -1 or 1 according to the preference list
        # s1 > s2 --> 1; s1 < s2 --> -1 """
        i, j = np.where(self.preference == s1), np.where(self.preference == s2)
        return 1 if i > j else -1

    def batch_compare(self, s_table):
        """ Input dim: k x 2 matrix, each row is (s1, s2)
        Return a vector of length k according to the preference list """
        vectorCompare = np.vectorize(self.compare)
        return vectorCompare(s_table.T[0], s_table.T[1])

class comparison_learner():
  # Implementation of a comparison learning function

    def __init__(self, S, n, oracle, epsilon, delta):
        """initialize with
        # a training sample S, ground set size n
        # an oracle
        # initial dictionary R[(i, j)] = [w, theta]
        # hyper-parameters epsilon, delta"""

        self.S = S
        self.n = n
        self.oracle = oracle
        self.R = {}
        self.epsilon = epsilon
        self.delta = delta
        self.m = self.calculate_m(epsilon, delta)

    def train(self):
        """main function, updates dictionary R with weights"""

        # step 1
        S1 = np.random.choice(self.S, self.m, replace = False)
        S2 = np.setdiff1d(self.S, S1)

        # step 2
        S1_sorted = self.sort_1d(S1)

        # step 3
        S2_dict = self.sort_2d(S1_sorted, S2)

        # step 4 & 5
        R = self.get_weights(S1_sorted, S2_dict)

        # step 6
        self.R = self.minimalize(R)

    def predict(self, s1, s2):
        """Input s1, s2. Returns comparison result. (s1 > s2 --> 1; s1 < s2 --> -1)"""

        chi_s1 = np.append(num_to_ind(s1, self.n), 1) # add intercept term
        chi_s2 = np.append(num_to_ind(s2, self.n), 1)

        for (a, b) in self.R:
            weight = self.R[(a, b)]
            if np.dot(weight, chi_s1) < 0 and np.dot(weight, chi_s2) > 0:
                return -1
        return 1

    def batch_predict(self, test_sample):
        """Input dimension: K x 2. Returns a vector of length K comparison result"""
        return [self.predict(row[0], row[1]) for row in test_sample]

    def test_accuracy(self, test_sample, test_truth):
        """Input vector of length k. Returns accuracy rate"""
        predict_results = self.batch_predict(test_sample)
        accuracy = sum(np.equal(predict_results, test_truth)) / len(test_truth)
        return accuracy

    def calculate_m(self, epsilon, delta):
        """Input hyperparameters. Returns m"""
        return int((2/epsilon)*math.log(1/(epsilon*delta)))

    def sort_1d(self, S1):
        """Sorts S1 by value function"""
        return sorted(S1, key=cmp_to_key(self.oracle.compare))

    def sort_2d(self, S1, S2):
        """Sorts S2 2-dimensionally"""
        S2_dict = {}
        for i in range(self.m):
            for j in range(i+1, self.m):
                S2_dict[(i, j)] = []
                for s in S2:
                    # add into S2_dict if s < s_i or s > s_j
                    if self.oracle.compare(s, S1[i]) == -1 or self.oracle.compare(s, S1[j]) == 1:
                        S2_dict[(i, j)].append(s)
        return S2_dict

    def minimalize(self, R):
        """Returns a minimalized dictionary R to contain only minimal pairs"""
        R_copy = copy.deepcopy(R)
        for (a, b) in R_copy:
            for (c, d) in R_copy:
                if (a, b)!= (c, d) and a <= c and b >= d and (a, b) in R:
                    del R[(a, b)]
        return R

    def get_weights(self, S1, S2_dict):
        """Returns a dictionary R where key is (i, j) and value is the weights [w_ij, theta_ij]"""
        R = {}
        for i in range(self.m):
            for j in range(i+1, self.m):
                S_ij = S2_dict[(i, j)]
                labels = [-1 if self.oracle.compare(s, S1[i]) == -1 else 1 for s in S_ij]
                S_ij = set_to_matrix(S_ij, self.n)
                net = perceptron.Perceptron(max_iter=100,
                                            fit_intercept=True,
                                            eta0=0.002)
                if len(np.unique(labels)) == 2: # fit if having two unique labels
                    net.fit(S_ij, labels)
                    if net.score(S_ij, labels) == 1: # add to R if no training error
                        w = net.coef_[0]
                        theta = net.intercept_[0]
                        R[(i, j)] = np.append(w, theta)
        return R

# Helper functions
def num_to_ind(k, n):
    """Input number. Returns a list of binary with length n."""
    length = '0' + str(n) + 'b'
    return np.array([int(x) for x in format(k, length)])

def ind_to_num(ind_vec):
    """Input a list of binary with length n. Returns a number."""
    return np.array(int(''.join(map(str, list(ind_vec))), 2))

def set_to_matrix(num_list, n):
    """Input set of numbers. Returns a matrix where each row is a indicator vector."""
    return np.array([num_to_ind(x, n) for x in num_list])

def run_test(n, epsilon, delta, S_size, pref_list, test_size):
    """Input learning model hyper-parameters and test parameters. Return accuracy rate."""
    # build pair-comparison oracle
    oracle = pairwise_oracle(pref_list)

    # build comparison learner
    S = np.random.choice(pref_list, S_size, replace = False)
    myLearner = comparison_learner(S, n, oracle, epsilon, delta)

    # train
    myLearner.train()

    # test
    select_sample = np.random.choice(pref_list, 2*test_size, replace = False)
    test_sample = [select_sample[0:test_size], select_sample[test_size:2*test_size]]
    test_sample = np.rot90(test_sample) # k x 2 dimension
    test_truth  = oracle.batch_compare(test_sample)
    accuracy    = myLearner.test_accuracy(test_sample, test_truth)

    return accuracy

if __name__ == "__main__":
    run_test(n=10, epsilon=0.2, delta=0.2, S_size=200,
        pref_list=range(1024), test_size=100)