import os
import re
import time

from git import Repo, Git, InvalidGitRepositoryError, GitError, PushInfo
from halo import Halo

from ml_git import log
from ml_git.config import get_metadata_path
from ml_git.constants import METADATA_MANAGER_CLASS_NAME, HEAD_1, RGX_ADDED_FILES, RGX_DELETED_FILES, RGX_SIZE_FILES, \
    RGX_AMOUNT_FILES, TAG, AUTHOR, EMAIL, DATE, MESSAGE, ADDED, SIZE, AMOUNT, DELETED
from ml_git.manifest import Manifest
from ml_git.utils import get_root_path, ensure_path_exists, yaml_load, RootPathException, get_yaml_str

class MetadataRepo(object):

    def __init__(self, git, path):
        try:
            root_path = get_root_path()
            self.__path = os.path.join(root_path, path)
            self.__git = git
            ensure_path_exists(self.__path)
        except RootPathException as e:
            log.error(e, class_name=METADATA_MANAGER_CLASS_NAME)
            raise e
        except Exception as e:
            if str(e) == '\'Metadata\' object has no attribute \'_MetadataRepo__git\'':
                log.error('You are not in an initialized ml-git repository.', class_name=METADATA_MANAGER_CLASS_NAME)
            else:
                log.error(e, class_name=METADATA_MANAGER_CLASS_NAME)
            return

    def init(self):
        try:
            log.info('Metadata init [%s] @ [%s]' % (self.__git, self.__path), class_name=METADATA_MANAGER_CLASS_NAME)
            Repo.clone_from(self.__git, self.__path)
        except GitError as g:
            if 'fatal: repository \'\' does not exist' in g.stderr:
                raise GitError('Unable to find remote repository. Add the remote first.')
            elif 'Repository not found' in g.stderr:
                raise GitError('Unable to find ' + self.__git + '. Check the remote repository used.')
            elif 'already exists and is not an empty directory' in g.stderr:
                raise GitError('The path [%s] already exists and is not an empty directory.' % self.__path)
            elif 'Authentication failed' in g.stderr:
                raise GitError('Authentication failed for git remote')
            else:
                raise GitError(g.stderr)
            return

    def remote_set_url(self, repotype, mlgit_remote):
        try:
            if self.check_exists():
                repo = Repo(self.__path)
                repo.remote().set_url(new_url=mlgit_remote)
        except InvalidGitRepositoryError as e:
            log.error(e, class_name=METADATA_MANAGER_CLASS_NAME)
            raise e

    def check_exists(self):
        log.debug('Metadata check existence [%s] @ [%s]' % (self.__git, self.__path),
                  class_name=METADATA_MANAGER_CLASS_NAME)
        try:
            Repo(self.__path)
        except Exception:
            return False
        return True

    def checkout(self, sha):
        repo = Git(self.__path)
        repo.checkout(sha)

    def update(self):
        log.info('Pull [%s]' % self.__path, class_name=METADATA_MANAGER_CLASS_NAME)
        repo = Repo(self.__path)
        o = repo.remotes.origin
        o.pull('--tags')

    def commit(self, file, msg):
        log.info('Commit repo[%s] --- file[%s]' % (self.__path, file), class_name=METADATA_MANAGER_CLASS_NAME)
        repo = Repo(self.__path)
        repo.index.add([file])
        return repo.index.commit(msg)

    def tag_add(self, tag):
        repo = Repo(self.__path)
        return repo.create_tag(tag, message='Automatic tag "{0}"'.format(tag))

    @Halo(text='Pushing metadata to the git repository', spinner='dots')
    def push(self):
        log.debug('Push [%s]' % self.__path, class_name=METADATA_MANAGER_CLASS_NAME)
        repo = Repo(self.__path)
        try:
            for i in repo.remotes.origin.push(tags=True):
                if (i.flags & PushInfo.ERROR) == PushInfo.ERROR:
                    raise RuntimeError('Error on push metadata to git repository. Please update your mlgit project!')

            for i in repo.remotes.origin.push():
                if (i.flags & PushInfo.ERROR) == PushInfo.ERROR:
                    raise RuntimeError('Error on push metadata to git repository. Please update your mlgit project!')
        except GitError as e:
            err = e.stderr
            match = re.search("stderr: 'fatal:((?:.|\\s)*)'", err)
            if match:
                err = match.group(1)
                raise GitError(err)

    def fetch(self):
        try:
            log.debug(' fetch [%s]' % self.__path, class_name=METADATA_MANAGER_CLASS_NAME)
            repo = Repo(self.__path)
            repo.remotes.origin.fetch()
        except GitError as e:
            err = e.stderr
            match = re.search('stderr: \'fatal:(.*)\'', err)
            if match:
                err = match.group(1)
                log.error(err, class_name=METADATA_MANAGER_CLASS_NAME)
            else:
                log.error(err, class_name=METADATA_MANAGER_CLASS_NAME)
            return False

    def list_tags(self, spec, full_info=False):
        tags = []
        try:
            repo = Repo(self.__path)
            r_tags = repo.tags if full_info else repo.git.tag(sort='creatordate').split('\n')
            for tag in r_tags:
                if f'__{spec}__' in str(tag):
                    tags.append(tag)

        except Exception:
            log.error('Invalid ml-git repository!', class_name=METADATA_MANAGER_CLASS_NAME)
        return tags

    def delete_tag(self, tag):
        pass

    def _usrtag_exists(self, usrtag):
        repo = Repo(self.__path)
        sutag = usrtag._get()
        for tag in repo.tags:
            stag = str(tag)
            if sutag in stag:
                return True
        return False

    def _tag_exists(self, tag):
        tags = []
        repo = Repo(self.__path)
        if tag in repo.tags:
            tags.append(tag)
        model_tag = '__'.join(tag.split('__')[-3:])
        for r_tag in repo.tags:
            if model_tag == str(r_tag):
                tags.append(str(r_tag))
        return tags

    def __realname(self, path, root=None):
        if root is not None:
            path = os.path.join(root, path)
        result = os.path.basename(path)
        return result

    def list(self, title='ML Datasets'):
        metadata_path = self.__path

        prefix = 0
        if metadata_path != '/':
            if metadata_path.endswith('/'):
                metadata_path = metadata_path[:-1]
            prefix = len(metadata_path)

        output = title + '\n'
        for root, dirs, files in os.walk(metadata_path):
            if root == metadata_path:
                continue
            if '.git' in root:
                continue

            level = root[prefix:].count(os.sep)
            indent = subindent = ''
            if level > 0:
                indent = '|   ' * (level - 1) + '|-- '
            subindent = '|   ' * (level) + '|-- '
            output += '{}{}\n'.format(indent, self.__realname(root))
            # print dir only if symbolic link; otherwise, will be printed as root
            for d in dirs:
                if os.path.islink(os.path.join(root, d)):
                    output += '{}{}\n'.format(subindent, self.__realname(d, root=root))
        if output != (title + '\n'):
            print(output)
        else:
            log.error('You don\'t have any entity being managed.')
            # for f in files:
            # if 'README' in f: continue
            # if 'MANIFEST.yaml' in root: continue # TODO : check within the ML entity metadat for manifest files
            # print('{}{}'.format(subindent, self.__realname(f, root=root)))

    @staticmethod
    def metadata_print(metadata_file, spec_name):
        md = yaml_load(metadata_file)

        sections = ['dataset', 'model', 'labels']
        for section in sections:
            if section in ['model', 'dataset', 'labels']:
                try:
                    md[section]  # 'hack' to ensure we don't print something useless
                    # 'dataset' not present in 'model' and vice versa
                    print('-- %s : %s --' % (section, spec_name))
                except Exception:
                    continue
            elif section not in ['model', 'dataset', 'labels']:
                print('-- %s --' % (section))
            try:
                print(get_yaml_str(md[section]))
            except Exception:
                continue

    def sha_from_tag(self, tag):
        try:
            r = Repo(self.__path)
            return r.git.rev_list(tag).split('\n', 1)[0]
        except Exception:
            return None

    def git_user_config(self):
        r = Repo(self.__path)
        reader = r.config_reader()
        config = {}
        types = ['email', 'name']
        for type in types:
            try:
                field = reader.get_value('user', type)
                config[type] = field
            except Exception:
                config[type] = None
        return config

    def metadata_spec_from_name(self, specname):
        specs = []
        for root, dirs, files in os.walk(self.__path):
            if '.git' in root:
                continue
            if specname in root:
                specs.append(os.path.join(root, specname + '.spec'))
        return specs

    def show(self, spec):
        specs = self.metadata_spec_from_name(spec)

        for specpath in specs:
            self.metadata_print(specpath, spec)

    def get(self, categories, model_name, file=None):
        if file is None:
            full_path = os.path.join(self.__path, os.sep.join(categories), model_name, model_name)
        else:
            full_path = os.path.join(self.__path, os.sep.join(categories), model_name, file)
        log.info('Metadata GET %s' % full_path, class_name=METADATA_MANAGER_CLASS_NAME)
        if os.path.exists(full_path):
            return yaml_load(full_path)
        return None

    def reset(self):
        repo = Repo(self.__path)
        # get current tag reference
        tag = self.get_current_tag()
        # reset
        try:
            repo.head.reset(HEAD_1, index=True, working_tree=True, paths=None)
        except GitError as g:
            if 'Failed to resolve \'HEAD~1\' as a valid revision.' in g.stderr:
                log.error('There is no commit to go back. Do at least two commits.',
                          class_name=METADATA_MANAGER_CLASS_NAME)
            raise g
        # delete the associated tag
        repo.delete_tag(tag)

    def get_metadata_manifest(self, path):
        if os.path.isfile(path):
            return Manifest(path)
        return None

    def remove_deleted_files_meta_manifest(self, manifest, deleted_files):
        if manifest is not None:
            for file in deleted_files:
                manifest.rm_file(file)
            manifest.save()

    def get_current_tag(self):
        repo = Repo(self.__path)
        tag = next((tag for tag in repo.tags if tag.commit == repo.head.commit), None)
        return tag

    def __sort_tag_by_date(self, elem):
        return elem.commit.authored_date

    def get_log_info(self, spec, fullstat=False):
        tags = self.list_tags(spec, True)
        formatted = ''
        if len(tags) == 0:
            raise RuntimeError('No log found for entity [%s]' % spec)

        tags.sort(key=self.__sort_tag_by_date)

        for tag in tags:
            formatted += '\n' + self.get_formatted_log_info(tag, fullstat)

        return formatted

    def get_formatted_log_info(self, tag, fullstat):
        commit = tag.commit
        info = ''
        info += '\n{}: {}'.format(TAG, str(tag))
        info += '\n{}: {}'.format(AUTHOR, commit.author.name)
        info += '\n{}: {}'.format(EMAIL, commit.author.email)
        info += '\n{}: {}'.format(DATE, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(commit.authored_date)))
        info += '\n{}: {}'.format(MESSAGE, commit.message)

        if fullstat:
            added, deleted, size, amount = self.get_ref_diff(tag)
            if len(added) > 0:
                added_list = list(added)
                info += '\n\n{} [{}]:\n\t{}'.format(ADDED, len(added_list), '\n\t'.join(added_list))
            if len(deleted) > 0:
                deleted_list = list(deleted)
                info += '\n\n{} [{}]:\n\t{}'.format(DELETED, len(deleted_list), '\n\t'.join(deleted_list))
            if len(size) > 0:
                info += '\n\n{}: {}'.format(SIZE, '\n\t'.join(size))
            if len(amount) > 0:
                info += '\n\n{}: {}'.format(AMOUNT, '\n\t'.join(amount))

        return info

    def get_ref_diff(self, tag):
        repo = Repo(self.__path)
        commit = tag.commit
        parents = tag.commit.parents
        added_files = []
        deleted_files = []
        size_files = []
        amount_files = []
        if len(parents) > 0:
            diff = repo.git.diff(str(parents[0]), str(commit))
            added_files = re.findall(RGX_ADDED_FILES, diff)
            deleted_files = re.findall(RGX_DELETED_FILES, diff)
            size_files = re.findall(RGX_SIZE_FILES, diff)
            amount_files = re.findall(RGX_AMOUNT_FILES, diff)

        return added_files, deleted_files, size_files, amount_files

class MetadataManager(MetadataRepo):
    def __init__(self, config, type='model'):
        self.path = get_metadata_path(config, type)
        self.git = config[type]['git']

        super(MetadataManager, self).__init__(self.git, self.path)

class MetadataObject(object):
    def __init__(self):
        pass

# TODO signed tag
# try:
#            self.repo.create_tag(self.config['tag'],
#                verify=True,
#                ref=None)
#            print('okay')
#        except:
#            print('not okay')

if __name__ == '__main__':
    r = MetadataRepo('git@github.com:example/your-mlgit-datasets.git', 'ml-git/datasets/')
    # tag = 'vision-computing__images__cifar-10__1'
    # sha = '0e4649ad0b5fa48875cdfc2ea43366dc06b3584e'
    # #r.checkout(sha)
    # #r.checkout('master')


from gene_finder.pipeline import Pipeline
import yaml
import os, sys

import os
def get_all_file_paths(path_to_dir):
    """
    Takes a path to a single directory and returns a list of the
    paths to each file in the directory.
    """
    fasta_files = os.listdir(path_to_dir)
    full_paths = [os.path.join(path_to_dir, f) for f in fasta_files]
    return full_paths

def GF(name):
    """
    Command line args:
        1. Path to input fasta file
        2. Path to config file

    Constructs a Pipeline object to search for CRISPR-Transposon
    systems in the genome/config of interest.
    """

    genomic_data = name
    print(name)
    conf_file = "./config.yaml"

    # load contents of yaml conf file
    stream = open(conf_file, 'r')
    conf = yaml.load(stream)

    # create a directory to write results to, if it doesn't exist already
    if not os.path.exists(conf["out-dir"]):
        os.mkdir(conf["out-dir"])

    # results file will be called "<contig filename>_results.csv"
    job_id = os.path.basename(genomic_data)
    #outfile = os.path.join(conf["out-dir"], "{}_results.csv".format(job_id))

    p = Pipeline()
    for step in conf["steps"]:

        if step["type"] == "seed":
            p.add_seed_step(db=step["blast-db"], name=step["name"], e_val=step["e-val"], blast_type=step["blast-type"])

        elif step["type"] == "filter":
            p.add_filter_step(db=step["blast-db"], name=step["name"], e_val=step["e-val"], blast_type=step["blast-type"])

        elif step["type"] == "blast":
            p.add_blast_step(db=step["blast-db"], name=step["name"], e_val=step["e-val"], blast_type=step["blast-type"])

        else:
            p.add_crispr_step()

    p.run(data=genomic_data, job_id=job_id, output_directory=conf["out-dir"], min_prot_len=conf["min-prot-len"],
          span=conf["span"], gzip=conf["gzip"])

import pandas as pd

df = pd.read_csv('TypeIF_path')
all_fasta_files = df['paths'].tolist()
from multiprocessing import Pool
if __name__ == '__main__':

   # all_fasta_files = get_all_file_paths(path_to_input_dir)
    print(len(all_fasta_files))

    #p = Pool(50)
    p = Pool(70)
    p.map(GF, all_fasta_files)


import sys
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import json

driver = webdriver.Chrome()

#driver = webdriver.Chrome()
driver.get('https://mops.twse.com.tw/mops/web/t100sb07_1')

#输入股票代码
#
# stockid="2330"
stockid = ["6560 ", "6561 "]

newData=[]

for var,index in enumerate(stockid):

    driver.find_element_by_id('co_id').send_keys(index)#找id=email
    driver.find_element_by_id('co_id').send_keys(Keys.ENTER)
    time.sleep(1) #这很重要

    soup = BeautifulSoup(driver.page_source, 'html.parser')
    table=soup.find('table', class_ = 'hasBorder')

    table_all=table.find_all('td')

    table_data=[]

    for i in range(0,len(table_all)):
        table_data.append(table_all[i].text)

    tablec_dict = {
                #召开法人说明会日期：
                "date": table_data[1].replace('\n',''),
                #召开法人说明会地点：
                "location": table_data[3].replace('\n',''),

                #法人说明会择要讯息：：
                "import_info": table_data[5].replace('\n',''),
                #法人说明会简报内容中文：：
                "chinese_file": table_data[8].replace('\n',''),
                #法人说明会简报中文内容连结：：
                "chinese_href":'https://mops.twse.com.tw/nas/STR/'+table_data[8].replace('\n',''),
                #法人说明会简报内容英文：：
                "english_file": table_data[10].replace('\n',''),

                #法人说明会简报英文内容连结：：
                "english_href":'https://mops.twse.com.tw/nas/STR/'+table_data[10].replace('\n',''),

                #法人说明会相关资讯：
                "info": table_data[13].replace('\n',''),

                #影音连结：
                "video": table_data[14].replace('\n','').replace('\xa0',''),

                #其他应叙明事项：
                "other": table_data[16],

                }

    newData += [tablec_dict]
    driver.find_element_by_id('co_id').clear()

driver.quit()
#driver.close()

# newData= json.dumps(newData)
# print(newData)

#https://mops.twse.com.tw/mops/nas/STR/


from setuptools import setup, find_packages
from pip._internal.req import parse_requirements

install_requires = parse_requirements('requirements.txt', session='hack')
install_requires = [str(ir.requirement) for ir in install_requires]

setup(
    name='btbinance',
    version='1.0',
    description='Backtrader Binance Websocket broker',
    url='https://github.com/AwesomeTrading/BTBinance.git',
    author='Santatic',
    license='Private',
    packages=find_packages(include=['btbinance', 'btbinance.*']),
    install_requires=install_requires,
)


from utils.preprocessing import Preprocessing
import math

class DocumentSim(object):
	def __init__(self):
		self.text = Preprocessing()

	def TFs(self, sentences):

		tfs = {}
		for sent in sentences:
			wordFreqs = sent.getWordFreqs()

			for word in wordFreqs.keys():
				if tfs.get(word, 0) != 0:
					tfs[word] = tfs[word] + wordFreqs[word]
				else:
					tfs[word] = wordFreqs[word]
		return tfs

	def TFw(self, word, sentence):
		return sentence.getWordFreqs().get(word, 0)

	def IDFs(self, sentences):

		N = len(sentences)
		idfs = {}
		words = {}
		w2 = []

		for sent in sentences:
			for word in sent.getStemmedWords():
				if sent.getWordFreqs().get(word, 0) != 0:
					words[word] = words.get(word, 0) + 1

		for word in words:
			n = words[word]
			try:
				w2.append(n)
				idf = math.log10(float(N) / n)
			except ZeroDivisionError:
				idf = 0

			idfs[word] = idf

		return idfs

	def IDF(self, word, idfs):
		return idfs[word]

	def sim(self, sentence1, sentence2, idfs):

		numerator = 0
		denom1 = 0
		denom2 = 0

		for word in sentence2.getStemmedWords():
			numerator += self.TFw(word, sentence2) * self.TFw(word, sentence1) * self.IDF(word, idfs) ** 2

		for word in sentence1.getStemmedWords():
			denom2 += (self.TFw(word, sentence1) * self.IDF(word, idfs)) ** 2

		for word in sentence2.getStemmedWords():
			denom1 += (self.TFw(word, sentence2) * self.IDF(word, idfs)) ** 2

		try:
			return numerator / (math.sqrt(denom1) * math.sqrt(denom2))

		except ZeroDivisionError:
			return float("-inf")


from datetime import date
from functools import wraps
from logging import info
from typing import List
from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from starlette import status
from src.currency.dals import CurrencyDAL
from settings.database import async_session_maker
from src.currency.models import Currency
from src.currency.schemas import ShowCurrency, LastPriceCurrency

def check_currency_in_db(func):
    """Checking if the currency is in the database"""

    @wraps(func)
    async def wrapper(*args, **kwargs):
        get_currency = await CurrencyAction.get_currency(
            args[0], args[1]
        )
        if get_currency is not None:
            return await func(*args, **kwargs)

        info(f'Currency {args[0]} is not in the database')

        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail='The currency ticker doesn\'t exist yet'
        )

    return wrapper

class CurrencyAction:
    """Actions on currency"""

    @staticmethod
    async def create_currency(**kwargs) -> Currency:
        """
        Method to create a new Currency in the database
        :param kwargs: The keyword arguments
        :return: The created Currency
        """

        async with async_session_maker() as session:
            async with session.begin():
                currency_dal = CurrencyDAL(session)
                new_currency = await currency_dal.create_currency(
                    currency_ticker=kwargs.get('currency_ticker'),
                    current_price=kwargs.get('current_price'),
                    unix_time=kwargs.get('unix_time')
                )
                return new_currency

    @staticmethod
    @check_currency_in_db
    async def get_all_by_currency(
            currency_ticker: str, db: AsyncSession
    ) -> List[ShowCurrency] | None:
        """
        Retrieve all saved data for the specified currency
        :param currency_ticker: currency
        :param db: AsyncSession
        :return: The list of currencies
        """
        async with db as session:
            async with session.begin():
                currency_dal = CurrencyDAL(session)
                all_currency = await currency_dal.get_all_by_currency(
                    currency_ticker
                )
                return [
                    ShowCurrency(
                        id=currency.id,
                        currency_ticker=currency.currency_ticker,
                        current_price=currency.current_price,
                        unix_time=currency.unix_time,
                        created_at=currency.created_at
                    ) for currency in all_currency
                ]

    @staticmethod
    async def get_currency(
            currency_ticker: str, db: AsyncSession) -> Currency | None:
        """
        Get currency by name
        :param currency_ticker: currency ticker name
        :param db: AsyncSession
        :return: Currency or None
        """
        async with db as session:
            async with session.begin():
                currency_dal = CurrencyDAL(session)
                get_currency = await currency_dal.get_currency(currency_ticker)
                return get_currency

    @staticmethod
    @check_currency_in_db
    async def get_last_price_currency(
            currency_ticker: str, db: AsyncSession) -> LastPriceCurrency | None:
        """
        Returns the last price currency
        :param currency_ticker: currency ticker name
        :param db: AsyncSession
        :return: last price currency
        """
        async with db as session:
            async with session.begin():
                currency_dal = CurrencyDAL(session)

                last_price = await currency_dal.get_last_price_currency(
                    currency_ticker
                )
                return LastPriceCurrency(
                    currency_ticker=last_price.currency_ticker,
                    last_price=last_price.current_price
                )

    @staticmethod
    @check_currency_in_db
    async def get_price_curr_filter_by_date(
            currency_ticker: str, db: AsyncSession, timestamp: date

    ) -> ShowCurrency | None:
        """
        Getting the price of a currency with a filter by date
        :param currency_ticker: currency ticker name
        :param timestamp: date of the currency
        :param db: AsyncSession
        :return: Currency or None
        """
        async with db as session:
            async with session.begin():
                currency_dal = CurrencyDAL(session)

                currency = await currency_dal.get_price_curr_filter_by_date(
                    currency_ticker, timestamp
                )
                return ShowCurrency(
                    id=currency.id,
                    currency_ticker=currency.currency_ticker,
                    current_price=currency.current_price,
                    unix_time=currency.unix_time,
                    created_at=currency.created_at)


import tensorflow as tf
from keras.models import Sequential, load_model
from keras.layers.core import Dense, Activation, Dropout
from keras.utils import np_utils
from keras.callbacks import CSVLogger, TensorBoard, ModelCheckpoint, EarlyStopping
import pickle
import gzip
import pandas as pd
import numpy as np
import gc
from datetime import date, datetime

from sklearn.utils import shuffle
import sklearn, random
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from keras.models import clone_model
from keras.models import load_model

import scipy.io as si
import argparse
import time

## Dir
runDir = "/xdisk/manojgopale/AES/dataCollection/"

np.random.seed(9)
scaler = StandardScaler()
def process_inputs (data):
	#data = pd.read_csv(dataPath, header=None)
	dataShuffle = shuffle(data)
	x_data_shuffle = dataShuffle[:,0:-1]
	y_data = dataShuffle[:,-1]
	x_data = scaler.fit_transform(x_data_shuffle)
	return x_data, y_data

## config: the configuration for which data is to be generated
## trainSize (4000) : Number of power traces per key to take
def getData(config, trainSize):

	## runDir
	runDir = "/xdisk/manojgopale/AES/dataCollection/"

	## Count for saving files
	count = 0

	## Number of keys to save in one train file
	partLen = 128
	keySize = 256

	## Create output
	inter_y = np.zeros([trainSize,1])
	inter_y[1999] = 1 ## Settting the 2000th data to be '1', which is the the AES trace

	for key in range(keySize):
		## Get 3 random samples for adding train, dev and test to the data
		index = random.sample(range(500), 3)

		## Load each key data for creating the training and testing files
		matStr = runDir + "/matResult/" +  config + "/value" + str(key) + ".mat"
		print("key=%s\nmatStr=%s\n" %(key, matStr))

		## Create intermediate datasets which will be post processed to get data
		## split into 1361 power traces
		interTrain = si.loadmat(matStr)["power"][index[0]]
		interDev =   si.loadmat(matStr)["power"][index[1]]
		interTest =  si.loadmat(matStr)["power"][index[2]]

		## Create data sets, this splits the complete data in 4000 rows with 1361 in each row
		## Each is a wondow of 1361, with step of 1
		interTrainData = interTrain[np.array([range(i, i+1361) for i in range(trainSize)])]
		interDevData =   interDev[np.array([range(i, i+1361) for i in range(trainSize)])]
		interTestData =  interTest[np.array([range(i, i+1361) for i in range(trainSize)])]

		##Append the output's at 2000th location for correct traces
		interTrainData = np.concatenate((interTrainData, inter_y), axis=1)
		interDevData =   np.concatenate((interDevData, inter_y), axis=1)
		interTestData =  np.concatenate((interTestData, inter_y), axis=1)

		## Append inter*Data to coressponding full data.
		if key == 0 or key%partLen == 0:
			fullTrain = interTrainData[:]
			fullDev = interDevData[:]
			fullTest = interTestData[:]
			print("loop1: key=%s" %(key))

		elif key%partLen == partLen-1:
			fullTrain = np.concatenate((fullTrain, interTrainData), axis=0)
			fullDev =   np.concatenate((fullDev, interDevData), axis=0)
			fullTest =  np.concatenate((fullTest, interTestData), axis=0)

			## Save files to csv, so as to use it for portability analysis
			trainSave = runDir + "/processedData/run_1_per_key/data/" + config + "/train_" + str(count) + ".csv"
			devSave   = runDir + "/processedData/run_1_per_key/data/" + config + "/dev_" + str(count) + ".csv"
			testSave  = runDir + "/processedData/run_1_per_key/data/" + config + "/test_" + str(count) + ".csv"

			##Save files
			np.savetxt(trainSave, fullTrain, fmt="%10.5f", delimiter=",")
			np.savetxt(devSave, fullTrain, fmt="%10.5f", delimiter=",")
			np.savetxt(testSave, fullTrain, fmt="%10.5f", delimiter=",")

			## Increment the count variable
			count = count + 1

			print("loop2: key=%s" %(key))
			print("\nSaved data to\n%s\n%s\n%s" %(trainSave, devSave, testSave))

		else:
			fullTrain = np.concatenate((fullTrain, interTrainData), axis=0)
			fullDev =   np.concatenate((fullDev, interDevData), axis=0)
			fullTest =  np.concatenate((fullTest, interTestData), axis=0)
			print("loop3: key=%s" %(key))

		## Clear variables
		interTrain = None
		interDev = None
		interTest = None
		interTrainData = None
		interDevData = None
		interTestData = None

	##Create final datasets
	x_train, y_train = process_inputs(fullTrain)
	x_dev, y_dev = process_inputs(fullDev)
	x_test, y_test = process_inputs(fullTest)

	##Clear fullData variables
	fullTrain = None
	fullDev = None
	fullTest = None

	print("\nCleared fullData variables\n")

	print("Shapes of datasets\nx_train=%s, y_train=%s\nx_dev=%s, y_dev=%s\nx_test=%s, y_test=%s"
	       %(x_train.shape, y_train.shape, x_dev.shape, y_dev.shape, x_test.shape, y_test.shape))

	return (x_train, y_train), (x_dev, y_dev), (x_test, y_test)

class Classifier:
	def __init__(self, resultDir: str, modelName: str, x_train, y_train, x_dev, y_dev, x_test, y_test, hiddenSize):
		""" Initialize parameters and sequential model for training
		"""
		self.resultDir = resultDir
		self.modelName = modelName
		self.x_train = x_train
		self.x_dev = x_dev
		self.x_test = x_test
		self.y_train = y_train
		self.y_dev = y_dev
		self.y_test = y_test
		self.hiddenSize = hiddenSize

		self.dropOut = 0.2

		self.model = Sequential()

		self.model.add(Dense(self.hiddenSize, input_shape=(1361,)))
		self.model.add(Activation('relu'))

		self.model.add(Dropout(self.dropOut))
		self.model.add(Dense(1))
		self.model.add(Activation('sigmoid'))

		self.model.compile(loss='binary_crossentropy', metrics=['binary_accuracy'], optimizer='adam')
		print("Model summary\n")
		print(self.model.summary())

	def train(self, batchSize):
		""" Train the model with the training data
		batchSize : batch size during trainig
		"""

		Epochs = 100

		logFile = self.resultDir + '/' + self.modelName +'.log'
		csv_logger = CSVLogger(logFile, append=True, separator="\t")

		earlyStop = EarlyStopping(monitor='val_binary_accuracy', patience=5, mode='auto', verbose=1, restore_best_weights=True)

		#filePath = self.resultDir + '/' + self.modelName + '_checkPoint_best_model.hdf5'
		### This file will include the epoch number when it gets saved.
		#repeatingFile = self.resultDir + '/' + self.modelName +'_{epoch:02d}_epoch_acc_{accVal:.2f}.hdf5'
		### By default the every_10epochs will save the model at every 10 epochs
		#checkPoint = newCallBacks.ModelCheckpoint_every_10epochs(filePath, repeatingFile, self.x_test, self.y_test , monitor='val_categorical_accuracy', verbose=1, save_best_only=True, every_10epochs=True)

		##Class weights for class imbalanced data
		## https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
		y = self.y_train.astype(int)

		sampleSize = len(y)
		classNum = len(set(y))

		weight_0 = sampleSize/(classNum * np.bincount(y)[0])
		weight_1 = sampleSize/(classNum * np.bincount(y)[1])

		classWeight = {0: weight_0, 1: weight_1}

		self.history = self.model.fit(self.x_train, self.y_train, batch_size= batchSize, epochs=Epochs, verbose=1, shuffle= True, validation_data=(self.x_dev, self.y_dev), class_weight=classWeight, callbacks=[csv_logger, earlyStop])

	def evaluate(self):
		""" Evaluate the model on itself
		"""

		self.model_score = self.model.evaluate(self.x_test, self.y_test, batch_size=2048)
		print("%s score = %f\n" %(self.modelName, self.model_score[1]))
		return self.model_score

	def saveModel(self):
		""" Save the model
		"""
		saveStr = self.resultDir + '/' + self.modelName +'1HL_' + str(self.hiddenSize) + '_' + str(self.history.epoch[-1]+1) + 'epochs_' + str(self.dropOut).replace('.', 'p') + 'Dropout_' + '{0:.2f}'.format(self.model_score[1]*100).replace('.', 'p') + '.h5'
		print("Saving model to\n%s\n" %(saveStr))
		self.model.save(saveStr)


# mPlane Protocol Reference Implementation
# tStat component code
#
# (c) 2013-2014 mPlane Consortium (http://www.ict-mplane.eu)
#               Author: Edion TEGO
# mSLAcert-Agent-V-1.0.0
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see <http://www.gnu.org/licenses/>.
#-------------------------------------------------------------------------------
# Find proccess and kill it if you cannot bind port
# ps aux | grep -i iperf
# sudo kill proccess_ID
#

import threading
from datetime import datetime
from time import sleep
import time
import mplane.model
import mplane.scheduler
import mplane.utils
from urllib3 import HTTPSConnectionPool
from urllib3 import HTTPConnectionPool
from socket import socket
import ssl
import argparse
import sys
import re
import json
import subprocess
import collections
from datetime import datetime, timedelta
from ipaddress import ip_address
import tornado.web
import tornado.ioloop
from subprocess import Popen
import urllib3
urllib3.disable_warnings()

DEFAULT_IP4_NET = "192.168.1.0/24"
DEFAULT_SUPERVISOR_IP4 = '127.0.0.1'
DEFAULT_SUPERVISOR_PORT = 8888
REGISTRATION_PATH = "register/capability"
SPECIFICATION_PATH = "show/specification"
RESULT_PATH = "register/result"

DUMMY_DN = "Dummy.Distinguished.Name"

print("    ###########################################################################################################");
print("    ###$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$##");
print("    ###$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$     $$$$$$$$  $$$$$$       $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$##");
print("    ##$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$  $$$  $$$$$$  $$$$$  $$$$$  $$$$$$  $$$$$$$$$$$$$$    $$$$$$$$##");
print("    ##$$$$$$$$$$$$$      $$$$$$$$$$$$$$$$$$$$$  $$$$$  $$$$  $$$$  $$$$$$$  $$$$$        $$$$$$  $$$$  $$$$$$##");
print("    ##$$$$$$$$$$   ;$$$$   $$$$$$       $$$$$$  $$$$  $$$$$  $$$$$$$$$   $  $$$$$  $$$$$  $$$$  $$$$$  $$$$$$##");
print("    ##$$$$$$$$   $$$$$$$$  $$$$   $$$$$  $$$$$  $$  $$$$$$$  $$$$$$$  $$$$  $$$$$  $$$$$  $$$$        $$$$$$$##");
print("    ##$$$$$$   $$$$$$$$$$!      $$$$$$$   $$$$   $$$$$$$$$$  $$$$$  $$$$$$  $$$$$  $$$$$  $$$$  $$$$$$$$$$$$$##");
print("    ##$$$$   $$$$$$$$$$$$$$  $$$$$$$$$$$  $$$$  $$$$$$$$$$$  $$$$  $$$$$    $$$$$  $$$$$  $$$$  $$$$$  $$$$$$##");
print("    ##$$$  $$$$$$$$$$$$$$$$$$$$$$$$$$$$$  $$$$  $$$$$$$$$$$  $$$$$       $  $$$$$  $$$$$  $$$$$       $$$$$$$##");
print("    ###$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$##");
print("    ##$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$##");
print("    ###$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$##");
print("    ###$$$$$$$$_____________________&______________________&_____________&_______________$$$$$$$$$$$$$$$$$$$$##");
print("    ###$$$$$$$$$___| mSLAcert probe| TCP & UDP Agent verification and certification|_______$$$$$$$$$$$$$$$$$$##");
print("    ###########################################################################################################");

"""
Implements mSLA-Agent proxy for integration into
the mPlane reference implementation.
(capability push, specification pull)

"""
commands = [
    'echo Starteded on	$(date)	-------------------->> ./results/mslaTCP.receive.client.txt',
    'iperf -s -p 5001 -i 1 -f k >> ./results/mslaTCP.receive.client.txt &',
    'pid5001=$!',
#    'echo Ended on	$date	------------------------------->> ./mslaTCP.receive.client.txt',
    'echo Starteded on	$(date)	-------------------->> ./results/mslaUDP.receive.client.txt',
    'iperf -s -u -p 5002 -i 1 -f  k >> ./results/mslaUDP.receive.client.txt &',
    'pid5002=$!',
    'ps aux | grep -i iperf',
#    'echo Ended on	$date	------------------------------->> ./mslaUDP.receive.client.txt',
]
# run in parallel and store locally the result
processes = [Popen(cmd, shell=True) for cmd in commands]

def sla_AGENT_capability(ip4addr):
    cap = mplane.model.Capability(label="msla-AGENT-Probe-ip4", when = "now ... future / 1s")
    cap.add_parameter("source.ip4",ip4addr)
    cap.add_parameter("destination.ip4")
    cap.add_result_column("mSLA.udpCapacity.download.iperf")
    return cap

def services(ip4addr = None):
    services = []
    if ip4addr is not None:
        services.append(mSLAcert_Agent_Service(sla_AGENT_capability(ip4addr)))
    return services

class mSLAcert_Agent_Service(mplane.scheduler.Service):
    def __init__(self, cap):
        # verify the capability is acceptable
        if not ((cap.has_parameter("source.ip4") or
                 cap.has_parameter("source.ip6")) and
                (cap.has_parameter("destination.ip4") or
                 cap.has_parameter("destination.ip6")) and
                (cap.has_result_column("mSLA.udpCapacity.download.iperf"))):
            raise ValueError("capability not acceptable")
        super(mSLAcert_Agent_Service, self).__init__(cap)

    def run(self, spec, check_interrupt):
         # unpack parameters
        period = spec.when().period().total_seconds()
        duration = spec.when().duration().total_seconds()
        if duration is not None and duration > 0:
            count = int(duration / period)
        else:
            count = None

        agent = []
        a = time.time()
        time.sleep(count)
        report_i_am_agent = -1001
        b = time.time()

        # derive a result from the specification
        res = mplane.model.Result(specification=spec)
        # put actual start and end time into result
        res.set_when(mplane.model.When(a , b))
        # are we returning aggregates or raw numbers?
        if res.has_result_column("mSLA.udpCapacity.download.iperf"):
                res.set_result_value("mSLA.udpCapacity.download.iperf", report_i_am_agent)
                #os.system("scp ./results/mslaTCP.receive.client.txt USER@Repository:/repository/temp/")
                #os.system("scp ./results/mslaUDP.receive.client.txt USER@Repository:/repository/temp/")
        return res


from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import logging
import math

from ezdxf.math import Vec2, UCS, NULLVEC
from ezdxf.lldxf import const
from ezdxf.entities import DimStyleOverride, Dimension
from .dim_base import (
    BaseDimensionRenderer,
    get_required_defpoint,
    compile_mtext,
)

if TYPE_CHECKING:
    from ezdxf.eztypes import GenericLayoutType

__all__ = ["OrdinateDimension"]
logger = logging.getLogger("ezdxf")

class OrdinateDimension(BaseDimensionRenderer):
    # Required defpoints:
    # defpoint = origin (group code 10)
    # defpoint2 = feature location (group code 13)
    # defpoint3 = end of leader (group code 14)
    # user text location is ignored (group code 11) and replaced by default
    # location calculated by the ezdxf renderer:

    def __init__(
        self,
        dimension: Dimension,
        ucs: Optional[UCS] = None,
        override: Optional[DimStyleOverride] = None,
    ):
        # The local coordinate system is defined by origin and the
        # horizontal_direction in OCS:
        self.origin_ocs: Vec2 = get_required_defpoint(dimension, "defpoint")
        self.feature_location_ocs: Vec2 = get_required_defpoint(
            dimension, "defpoint2"
        )
        self.end_of_leader_ocs: Vec2 = get_required_defpoint(
            dimension, "defpoint3"
        )
        # Horizontal direction in clockwise orientation, see DXF reference
        # for group code 51:
        self.horizontal_dir = -dimension.dxf.get("horizontal_direction", 0.0)
        self.rotation = math.radians(self.horizontal_dir)
        self.local_x_axis = Vec2.from_angle(self.rotation)
        self.local_y_axis = self.local_x_axis.orthogonal()
        self.x_type = bool(  # x-type is set!
            dimension.dxf.get("dimtype", 0) & const.DIM_ORDINATE_TYPE
        )
        super().__init__(dimension, ucs, override)

        # Measurement directions can be opposite to local x- or y-axis
        self.leader_vec_ocs = self.end_of_leader_ocs - self.feature_location_ocs
        leader_x_vec = self.local_x_axis.project(self.leader_vec_ocs)
        leader_y_vec = self.local_y_axis.project(self.leader_vec_ocs)
        try:
            self.measurement_direction: Vec2 = leader_x_vec.normalize()
        except ZeroDivisionError:
            self.measurement_direction = Vec2(1, 0)
        try:
            self.measurement_orthogonal: Vec2 = leader_y_vec.normalize()
        except ZeroDivisionError:
            self.measurement_orthogonal = Vec2(0, 1)

        if not self.x_type:
            self.measurement_direction, self.measurement_orthogonal = (
                self.measurement_orthogonal,
                self.measurement_direction,
            )

        self.update_measurement()
        if self.tol.has_limits:
            self.tol.update_limits(self.measurement.value)

        # Text width and -height is required first, text location and -rotation
        # are not valid yet:
        self.text_box = self.init_text_box()

        # Set text location and rotation:
        self.measurement.text_location = self.get_default_text_location()
        self.measurement.text_rotation = self.get_default_text_rotation()

        # Update text box location and -rotation:
        self.text_box.center = self.measurement.text_location
        self.text_box.angle = self.measurement.text_rotation
        self.geometry.set_text_box(self.text_box)

        # Update final text location in the DIMENSION entity:
        self.dimension.dxf.text_midpoint = self.measurement.text_location

    def get_default_text_location(self) -> Vec2:
        if self.x_type:
            text_vertical_shifting_dir = -self.local_x_axis
        else:
            text_vertical_shifting_dir = self.local_y_axis

        # user text location is not supported and ignored:
        return (
            self.end_of_leader_ocs
            + self.measurement_orthogonal * (self.text_box.width * 0.5)
            + text_vertical_shifting_dir
            * self.measurement.text_vertical_distance()
        )

    def get_default_text_rotation(self) -> float:
        # user text rotation is not supported and ignored:
        return (90.0 if self.x_type else 0.0) + self.horizontal_dir

    def update_measurement(self) -> None:
        feature_location_vec: Vec2 = self.feature_location_ocs - self.origin_ocs
        # ordinate measurement is always absolute:
        self.measurement.update(
            self.local_x_axis.project(feature_location_vec).magnitude
            if self.x_type
            else self.local_y_axis.project(feature_location_vec).magnitude
        )

    def get_defpoints(self) -> list[Vec2]:
        return [
            self.origin_ocs,
            self.feature_location_ocs,
            self.end_of_leader_ocs,
        ]

    def transform_ucs_to_wcs(self) -> None:
        """Transforms dimension definition points into WCS or if required into
        OCS.
        """

        def from_ucs(attr, func):
            point = dxf.get(attr, NULLVEC)
            dxf.set(attr, func(point))

        dxf = self.dimension.dxf
        ucs = self.geometry.ucs
        from_ucs("defpoint", ucs.to_wcs)
        from_ucs("defpoint2", ucs.to_wcs)
        from_ucs("defpoint3", ucs.to_wcs)
        from_ucs("text_midpoint", ucs.to_ocs)

        # Horizontal direction in clockwise orientation, see DXF reference
        # for group code 51:
        dxf.horizontal_direction = -ucs.to_ocs_angle_deg(self.horizontal_dir)

    def render(self, block: GenericLayoutType) -> None:
        """Main method to create dimension geometry of basic DXF entities in the
        associated BLOCK layout.

        Args:
            block: target BLOCK for rendering

        """
        super().render(block)
        self.add_ordinate_leader()
        measurement = self.measurement
        if measurement.text:
            if self.geometry.supports_dxf_r2000:
                text = compile_mtext(measurement, self.tol)
            else:
                text = measurement.text
            self.add_measurement_text(
                text, measurement.text_location, measurement.text_rotation
            )
        self.geometry.add_defpoints(self.get_defpoints())

    def add_ordinate_leader(self) -> None:
        # DXF attributes from first extension line not from dimension line!
        attribs = self.extension_lines.dxfattribs(1)
        # The ordinate leader is normal to the measurement direction.
        # leader direction and text direction:
        direction = self.measurement_orthogonal
        leg_size = self.arrows.arrow_size * 2.0
        #            /---1---TEXT
        # x----0----/
        # d0 = distance from feature location (x) to 1st upward junction
        d0 = direction.project(self.leader_vec_ocs).magnitude - 2.0 * leg_size

        start0 = (
            self.feature_location_ocs + direction * self.extension_lines.offset
        )
        end0 = self.feature_location_ocs + direction * max(leg_size, d0)
        start1 = self.end_of_leader_ocs - direction * leg_size
        end1 = self.end_of_leader_ocs
        if self.measurement.vertical_placement != 0:
            end1 += direction * self.text_box.width

        self.add_line(start0, end0, dxfattribs=attribs)
        self.add_line(end0, start1, dxfattribs=attribs)
        self.add_line(start1, end1, dxfattribs=attribs)

    def add_measurement_text(
        self, dim_text: str, pos: Vec2, rotation: float
    ) -> None:
        """Add measurement text to dimension BLOCK.

        Args:
            dim_text: dimension text
            pos: text location
            rotation: text rotation in degrees

        """
        attribs = self.measurement.dxfattribs()
        self.add_text(dim_text, pos=pos, rotation=rotation, dxfattribs=attribs)


import socket
from datetime import datetime
from time import time, sleep
from numpipe import config
import matplotlib.pyplot as plt
import matplotlib as mpl

DEFAULT_DELAY = config.get_config()['notifications']['delay_default']

def get_bot_token():
    """get bot token from config file"""
    return config.get_config()['notifications']['telegram']['token']

def get_chat_id():
    """get chat ID from config file"""
    return config.get_config()['notifications']['telegram']['chat_id']

def notifications_active():
    """check whether notifications can be sent based on the config file"""
    return get_bot_token() and get_chat_id()

def generate_time_str(time):
    """convert time (in seconds) to a text string"""
    hours = int(time // 60**2)
    minutes = int((time - hours*60**2) // 60)
    seconds = int((time - hours*60**2 - minutes*60) // 1)

    ret = ''
    if hours:
        unit = 'hr' if hours == 1 else 'hrs'
        ret += f'{hours} {unit} '
    if minutes:
        unit = 'min' if minutes == 1 else 'mins'
        ret += f'{minutes} {unit} '

    if not (hours and minutes):
        unit = 'sec' if seconds == 1 else 'secs'
        ret += f'{seconds} {unit}'

    return ret.strip()

def check_idle_matplotlib(delay=DEFAULT_DELAY, check_every=.5):
    """
    Check if the user is idle based on matplotlib plot interactions (mouse move, key press, window interactions)

    Arguments:
        delay         time (in seconds) to check before declaring idle
        check_every   time (in seconds) between interaction checks
    """
    nfigures_before = len(plt.get_fignums())
    if not nfigures_before:
        raise RuntimeError('cannot check for user idleness if there are no figures')

    mouse_moved = False
    key_pressed = False
    def on_mouse_movement(event):
        nonlocal mouse_moved
        mouse_moved = True

    def on_key_press(event):
        nonlocal key_pressed
        key_pressed = True

    fig = plt.figure(nfigures_before)
    cid = fig.canvas.mpl_connect('motion_notify_event', on_mouse_movement)
    cid = fig.canvas.mpl_connect('key_press_event', on_key_press)

    t_start = time()
    while time() - t_start < 1:
        sleep(.01)
        if not plt.get_fignums():
            return

    backend = mpl.get_backend()
    qt_backends = ('Qt4Agg', 'Qt5Agg')
    if backend in qt_backends:
        x0 = fig.canvas.manager.window.x()
        y0 = fig.canvas.manager.window.y()
        w0 = fig.canvas.manager.window.size()

    t_start = time()
    while time() - t_start < delay:
        sleep(check_every)
        if len(plt.get_fignums()) != nfigures_before:
            return False
        if mouse_moved:
            return False
        if key_pressed:
            return False

        if backend in qt_backends:
            if not fig.canvas.manager.window.isActiveWindow():
                return False

            x = fig.canvas.manager.window.x()
            y = fig.canvas.manager.window.y()
            w = fig.canvas.manager.window.size()

            if x != x0 or y != y0:
                return False

            if w != w0:
                return False

    return True

def send_message(message):
    """send a text message"""
    from telegram import Bot, ParseMode
    bot = Bot(token=get_bot_token())
    bot.send_message(chat_id=get_chat_id(), text=message, parse_mode=ParseMode.MARKDOWN)

def send_message_from(message, filename):
    """send a text message with """
    from telegram import Bot, ParseMode

    host = socket.gethostname()
    message = f'*{message}*' + f'\n_{host}:{filename}_'

    bot = Bot(token=get_bot_token())
    bot.send_message(chat_id=get_chat_id(), text=message, parse_mode=ParseMode.MARKDOWN)

def send_finish_message(filename, njobs, time, num_exceptions):
    """send a text message summarizing the jobs that ran

    Arguments:
        filename     name of the python file
        njobs        number of jobs ran
        time         runtime of the jobs
        num_exceptions   number of jobs that threw exceptions
    """
    host = socket.gethostname()
    time_str = generate_time_str(time)
    tab = '    '

    if num_exceptions:
        status = f'{num_exceptions}/{njobs} failures'
    else:
        status = 'success'

    date = datetime.now().strftime("%H:%M %d-%m-%Y")

    text = f'''`Simulation finished:
{tab}filename___{filename}.py
{tab}status_____{status}
{tab}host_______{host}
{tab}njobs______{njobs}
{tab}runtime____{time_str}
{tab}date_______{date}`
'''
    send_message(text)

def send_images(filename, exempt=[]):
    """send images (from matplotlib)

    Arguments:
        filename     name of the python file (without .py extension)
        exempt       (optional) a list of figure numbers to not send
    """
    if not plt.get_fignums():
        return

    from telegram import Bot, ChatAction, InputMediaPhoto
    from io import BytesIO
    bot = Bot(token=get_bot_token())
    chat_id = get_chat_id()

    send_action = lambda: bot.send_chat_action(chat_id=chat_id, action=ChatAction.UPLOAD_PHOTO)
    send_action()
    media = []

    num_figs = 0
    t_start = time()
    for i in plt.get_fignums():
        fig = plt.figure(i)
        if fig.number in exempt:
            continue

        caption = f'{filename}-fig{i}'

        bio = BytesIO()
        bio.name = f'fig{i}.png'
        fig.savefig(bio, format='png')
        bio.seek(0)
        media.append(InputMediaPhoto(bio, caption=caption))
        plt.close(fig)

        num_figs += 1
        if num_figs == 10:
            num_figs = 0
            bot.send_media_group(chat_id, media=media)
            media = []

        if time() - t_start > 7:
            send_action()
            t_start = time()

    if num_figs:
        bot.send_media_group(chat_id, media=media)

def send_videos(anims):
    """send a set of animations

    Arguments:
        anims    list of lists of the animations (each embedded list must share the same figure)
    """
    if not anims:
        return

    from telegram import Bot
    import tempfile
    plt.close('all')

    bot = Bot(token=get_bot_token())
    chat_id = get_chat_id()

    with tempfile.TemporaryDirectory() as direc:
        for i,anim_list in enumerate(anims):
            plt.close(anim_list[0]._fig)
            filepath = f'{direc}/vid{i}.mp4'
            send_animation(bot, chat_id, anim_list, filepath)
            bot.send_video(chat_id, video=open(filepath, 'rb'))

def send_animation(bot, chat_id, anim_list, filepath, *args, **kwargs):
    """send a single animation for a given bot and chat_id

    Arguments:
        bot         telegram bot
        chat_id     telgram chat id
        anim_list   list of animations (belonging to the same figure)
        filepath    filepath the animation will be saved to
        *args       additional arguments to pass to anim.save
        **kwargs    additional key-word arguments to pass to anim.save
    """
    from telegram import ChatAction
    anim  = anim_list[0]
    send_action = lambda: bot.send_chat_action(chat_id=chat_id, action=ChatAction.UPLOAD_VIDEO)

    store_func = anim._func
    t_start = time()
    send_action()
    def wrapper(*args):
        nonlocal t_start
        if time() - t_start > 7:
            send_action()
            t_start = time()
        return store_func(*args)
    anim._func = wrapper

    anim.save(filepath, extra_anim=anim_list[1:], *args, **kwargs)
    anim._func = store_func

def send_notifications(notifications, delay=DEFAULT_DELAY, check_idle=True, idle=False):
    """send a collection of notifications

    Arguments:
        notifications      list of functions to call that send notifications (no arguments)
        delay              time (in seconds) to check if the user is idle before notifying
        check_idle         whether or not to check if the user is idle (default: True)
        idle               whether or not the user is idle now (default: False)

    """
    if not notifications_active():
        return

    if check_idle:
        idle = check_idle_matplotlib(delay=delay)

    if idle:
        for notification in notifications:
            notification()


# change the categorical variables to dummies
def from_categorical_to_dummies(CV_data, var):
    import pandas as pd
    import numpy as np

    df = pd.get_dummies(CV_data[var])
    ## eliminate the variable with minimum variance
    l = df.std()
    #l.idxmin() #kjkrfgld_UhSVl
    df = df.drop(columns = l.idxmin())
    #columns and tranformation
    col = list(df.columns)
    names = np.repeat(var, len(col), axis=0)
    col = paste(names, col, sep = '_')
    df.columns = col

    # Join the dummy variables to the main dataframe
    df_new = pd.concat([CV_data, df], axis=1)
    df_new[col] = df_new[col].apply(pd.to_numeric, downcast='float')

    CV_data = df_new
    CV_data = CV_data.drop(columns = var)
    return(CV_data)

# paste function: used to concatenate any type of variables
import functools
def reduce_concat(x, sep=""):
    return functools.reduce(lambda x, y: str(x) + sep + str(y), x)

def paste(*lists, sep=" ", collapse=None):
    result = map(lambda x: reduce_concat(x, sep=sep), zip(*lists))
    if collapse is not None:
        return reduce_concat(result, sep=collapse)
    return list(result)

# group variables by target
def group_var_target(dataset, target, var):
    g = dataset.groupby(var)
    x = g[target].value_counts(normalize=True)
    x = x.reset_index(var)
    x.columns = [var, 'count']
    x = x.reset_index(target)
    return x

# fill numeric na's
def fill_numeric_nas(dataset, k):
    #conda install -c brittainhard fancyimpute
    import numpy as np
    import pandas as pd
    from fancyimpute import KNN, MICE
    dataset['poor'] = dataset['poor'].astype('str')
    df_numeric = dataset.select_dtypes(include=[np.float])
    df_int = dataset.select_dtypes(include=[np.uint8])
    df_str = dataset.select_dtypes(include=[np.object])
    col = df_numeric.columns
    df_filled = pd.DataFrame(KNN(k, verbose=False).complete(df_numeric))
    #df_filled = pd.DataFrame(MICE(n_imputations=200, impute_type='col', verbose=False).complete(df_numeric.as_matrix()))
    df_filled.columns = col
    df = pd.concat([df_filled, df_str], axis=1)
    df = pd.concat([df, df_int], axis=1)
    return df

# correlations

# result can be used w/ seaborn's heatmap
def compute_correlation_matrix(df, method='pearson'):
    import pandas as pd
    corr_mat = df.corr()
    return corr_mat

def prepare_data(dataset, coll):
   import numpy as np
   # MANAGE OUTLIERS
   dataset = fix_extreme_values(dataset)
   # ELIMINATE VARIABLES To avoid multicolineality
   #
   # insert values in categorical variables
   dataset.kjkrfgld.fillna('MISSING',inplace = True)
   dataset.bpowgknt.fillna('MISSING',inplace = True)
   dataset.raksnhjf.fillna('MISSING',inplace = True)
   dataset.vwpsxrgk.fillna('MISSING',inplace = True)
   #fill numeric na's
   dataset = fill_numeric_nas(dataset, 3)
   # include interactions
   dataset['inter_1'] = dataset['yfmzwkru'] * dataset['omtioxzz']
   dataset['inter_2'] = dataset['yfmzwkru'] * dataset['tiwrsloh']
   dataset['inter_3'] = dataset['yfmzwkru'] * dataset['weioazcf']
   dataset['inter_4'] = dataset['omtioxzz'] * dataset['tiwrsloh']
   dataset['inter_5'] = dataset['omtioxzz'] * dataset['weioazcf']
   dataset['inter_6'] = dataset['tiwrsloh'] * dataset['weioazcf']
   # one-hot-encodign
   dataset_complete = from_categorical_to_dummies(dataset, 'kjkrfgld')
   dataset_complete = from_categorical_to_dummies(dataset_complete, 'bpowgknt')
   dataset_complete = from_categorical_to_dummies(dataset_complete, 'raksnhjf')
   dataset_complete = from_categorical_to_dummies(dataset_complete, 'vwpsxrgk')

   cols = ['omtioxzz', 'tiwrsloh', 'weioazcf', 'yfmzwkru',  'inter_3',
'inter_4', 'inter_6','kjkrfgld_GVYCk', 'kjkrfgld_MISSING', 'kjkrfgld_qzGkS', 'kjkrfgld_tnDpM', 'bpowgknt_MISSING', 'raksnhjf_qTmDg', 'raksnhjf_rXCdD', 'vwpsxrgk_ARuYG', 'vwpsxrgk_MISSING', 'vwpsxrgk_QRKWz',  'vwpsxrgk_bUhyU', 'vwpsxrgk_puHHm', 'vwpsxrgk_qcizG', 'vwpsxrgk_yKWYC'] #'omtioxzz', 'tiwrsloh', 'weioazcf',

  # filter variables
   if len(coll) <= 1:
      dataset_complete = dataset_complete.drop(cols, axis=1)
   else:
      dataset_complete = dataset_complete.loc[:, coll]
      dataset_complete = dataset_complete.fillna(0)

   d = {'False' : '0', 'True' : '1'}
   dataset_complete['poor'] = dataset_complete['poor'].map(d)
   dataset_complete['poor'] = dataset_complete['poor'].astype('int')
   return dataset_complete

def fix_extreme_values(df_numeric):
   # Fix the extreme values
   x = df_numeric['yfmzwkru'].nlargest(2)
   x = pd.DataFrame(x)
   out_max = x.max()
   next_value = x.min()
   df_numeric.loc[df_numeric['yfmzwkru'] >= out_max[0], 'yfmzwkru'] = next_value[0]

   x = df_numeric['omtioxzz'].nsmallest(2)
   x = pd.DataFrame(x)
   out_min = x.min()
   next_value = x.max()
   df_numeric.loc[df_numeric['omtioxzz'] == out_min[0], 'omtioxzz'] = next_value[0]
   return df_numeric

import matplotlib.pyplot as plt
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix'):

    import matplotlib.pyplot as plt
    import numpy as np
    import itertools
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap= plt.cm.Blues ) #, cmap=cmap
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

######################## -------------------   CHI-SQUARE --------------------------------------------------------------

import pandas as pd
import numpy as np
import scipy.stats as stats
from scipy.stats import chi2_contingency

class ChiSquare:
    def __init__(self, dataframe):
        self.df = dataframe
        self.p = None #P-Value
        self.chi2 = None #Chi Test Statistic
        self.dof = None

        self.dfObserved = None
        self.dfExpected = None

    def _print_chisquare_result(self, colX, alpha):
        result = ""
        if self.p<alpha:
            result="{0} is IMPORTANT for Prediction".format(colX)
        else:
            result="{0} is NOT an important predictor. (Discard {0} from model)".format(colX)

        print(result)

    def TestIndependence(self,colX,colY, alpha=0.05):
        X = self.df[colX].astype(str)
        Y = self.df[colY].astype(str)

        self.dfObserved = pd.crosstab(Y,X)
        chi2, p, dof, expected = stats.chi2_contingency(self.dfObserved.values)
        self.p = p
        self.chi2 = chi2
        self.dof = dof

        self.dfExpected = pd.DataFrame(expected, columns=self.dfObserved.columns, index = self.dfObserved.index)

        self._print_chisquare_result(colX,alpha)


import sqlite3
from Dominio.perro import Perro

import jsonpickle

class PersistenciaPerros():

    def __init__(self):
        self.con = sqlite3.connect("la_tienda_de_santiago")

    def connect_perro(self):
        self.__crear_tabla_perro()

    def __crear_tabla_perro(self):
        try:
            cursor = self.con.cursor()
            query = "CREATE TABLE PERRO(codigoPer text primary key, nombre text," \
                    " edad int, raza text," \
                    " precio float,colorPelage text)"
            cursor.execute(query)
        except sqlite3.OperationalError as ex:
            pass

    def guardar_perro(self, perro: Perro):
        cursor = self.con.cursor()
        query = "insert into PERRO(codigoPer ,nombre ," \
                " edad ,raza,precio ,colorPelage ) values(" \
                f" ?,?,?,?,?,?)"
        cursor.execute(query, (str(perro.codigoPer), perro.nombre, perro.edad,
                               perro.raza,perro.precio, perro.colorPelage))
        self.con.commit()

    @classmethod
    def save_json_perro(cls, perro):
        text_open = open("files/perro/" + str(perro.codigoPer) + '.json', mode='w')
        json_gui = jsonpickle.encode(perro)
        text_open.write(json_gui)
        text_open.close()

    @classmethod
    def load_json_perro(cls, file_name):
        text_open = open("files/perro/" + file_name, mode='r')
        json_gui = text_open.readline()
        perro = jsonpickle.decode(json_gui)
        text_open.close()
        return perro

    @classmethod
    def load_json(cls, file_name):
        text_open = open("files/" + file_name, mode='r')
        json_gui = text_open.readline()
        mascota = jsonpickle.decode(json_gui)
        text_open.close()
        return mascota


#Joshua Hsin ID# 13651420
import json
from json2html import *
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from urllib.request import urlopen
import ssl
import os
import re
import sys
import _pickle as pickle
##https://docs.python.org/3/library/pickle.html
from collections import defaultdict
from nltk.stem import PorterStemmer
#https://pythonprogramming.net/stemming-nltk-tutorial/
import warnings
import math

warnings.filterwarnings("ignore", category=UserWarning, module='bs4')
ps = PorterStemmer()
key = dict() #key dictionary with urls
word_dict = defaultdict(dict) #word_dictionary

drop = open("dropoff.txt", "wb")
drop2 = open("dropoff2.txt", "wb")
drop3 = open("dropoff3.txt", "wb")
drop_final_list = [open("a.txt", "wb"), open("b.txt", "wb"), open("c.txt", "wb"), open("d.txt", "wb"), open("e.txt", "wb"), open("f.txt", "wb"),
                   open("g.txt", "wb"), open("h.txt", "wb"), open("i.txt", "wb"), open("j.txt", "wb"), open("k.txt", "wb"), open("l.txt", "wb"),
                   open("m.txt", "wb"), open("n.txt", "wb"), open("o.txt", "wb"), open("p.txt", "wb"), open("q.txt", "wb"), open("r.txt", "wb"),
                   open("s.txt", "wb"), open("t.txt", "wb"), open("u.txt", "wb"), open("v.txt", "wb"), open("w.txt", "wb"), open("x.txt", "wb"),
                   open("y.txt", "wb"), open("z.txt", "wb"), open("extra.txt", "wb")]
stop_words = ['a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an','and', 'any', 'are','aren\'t', 'as'
              , 'at', 'be', 'because', 'been', 'before', 'being', 'below','between', 'both', 'but', 'by', 'can\'t',
              'cannot', 'could', 'couldn\'t', 'did', 'didn\'t', 'do', 'does', 'doesn\'t', 'doing', 'don\'t', 'down', 'during',
              'each', 'few', 'for', 'from', 'further', 'had', 'hadn\'t', 'has', 'hasn\'t', 'have', 'haven\'t', 'having', 'he',
              'he\'d', 'he\'ll', 'he\'s', 'her', 'here', 'here\'s', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'how\'s',
              'i', 'i\'d', 'i\'ll', 'i\'m', 'i\'ve', 'if', 'in', 'into', 'is', 'isn\'t', 'it', 'it\'s', 'its', 'itself', 'let\'s',
              'me', 'more', 'most', 'mustn\'t', 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or',
              'other', 'ought', 'our', 'ours']
doc_num = 1

def drop_word(d, word, value):
    #dump a word dictionary into appropriate file
    global doc_num
    for x in d[word].keys():
        if(d[word][x] > 0):
            d[word][x] *= math.log(doc_num/len(d[word].keys()))
    pickle.dump(d, drop_final_list[value])

def merge1(drop, a):
    #merge the rest of one file
    global doc_num
    finish = False
    while not finish:
        try:
            a_word = list(a.keys())[0]
            for x in a[a_word].keys():
                a[a_word][x] *= math.log(doc_num/len(a[a_word].keys()))
            if(0 <= (ord(a_word[0]) - 97) <= 25):
                pickle.dump(a,drop_final_list[ord(a_word[0]) - 97])
            else:
                pickle.dump(a,drop_final_list[-1])
            a = pickle.load(drop)
        except EOFError:
            finish = True
    return

def merge2(drop, drop2, a, b):
    #merge two files together
    a_word = list(a.keys())[0]
    b_word = list(b.keys())[0]
    finish = False
    while not finish:
        if(a_word < b_word):
            try:
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(a,a_word, value)
                else:
                    drop_word(a,a_word, -1)
                a = pickle.load(drop)
                a_word = list(a.keys())[0]
            except EOFError:
                merge1(drop2, b)
                finish = True
        elif(a_word == b_word):
            d = defaultdict(dict)
            d[a_word] = {**a[a_word], **b[b_word]}
            value = ord(a_word[0]) - 97
            if(0 <= value <= 25):
                drop_word(d,a_word, value)
            else:
                drop_word(d,a_word, -1)
            try:
                a = pickle.load(drop)
                a_word = list(a.keys())[0]
                try:
                    b = pickle.load(drop)
                    b_word = list(b.keys())[0]
                except EOFError:
                    merge1(drop, a)
                    finish = True
            except EOFError:
                try:
                    b = pickle.load(drop)
                    merge1(drop2, b)
                    finish = True
                except EOFError:
                    finish = True
        elif(a_word > b_word):
            #print(a,b)
            try:
                value = ord(b_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(b,b_word, value)
                else:
                    drop_word(b,b_word, -1)
                b = pickle.load(drop2)
                b_word = list(b.keys())[0]
            except EOFError:
                merge1(drop,a)
                finish = True
    return

def merge():
    #merge all 3 files together
    drop = open('dropoff.txt', 'rb')
    drop2 = open('dropoff2.txt', 'rb')
    drop3 = open('dropoff3.txt', 'rb')
    a = pickle.load(drop)
    b = pickle.load(drop2)
    c = pickle.load(drop3)
    a_word = list(a.keys())[0]
    b_word = list(b.keys())[0]
    c_word = list(c.keys())[0]
    finish = False
    while not finish:
        #compare all three files to determine which has the lowest value word
        #dump the lowest value word into the appropriate file (a - z and extra)
        #if files have the same word, merge word dicts, then dump word_dict
        if(a_word < b_word):
            if(a_word < c_word):
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(a,a_word, value)
                else:
                    drop_word(a,a_word, -1)
                try:
                    a = pickle.load(drop)
                    a_word = list(a.keys())[0]
                except EOFError:
                    merge2(drop2, drop3, b, c)
                    finish = True
            elif(a_word == c_word):
                d = defaultdict(dict)
                d[a_word] = {**a[a_word], **c[c_word]}
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,a_word, value)
                else:
                    drop_word(d,a_word, -1)
                try:
                    a = pickle.load(drop)
                    a_word = list(a.keys())[0]
                    try:
                        c = pickle.load(drop3)
                        c_word = list(c.keys())[0]
                    except EOFError:
                        merge2(drop, drop2, a, b)
                        finish = True
                except EOFError:
                    try:
                        c = pickle.load(drop3)
                        merge2(drop2, drop3, b, c)
                        finish = True
                    except EOFError:
                        merge1(drop2, b)
                        finish = True
            elif(c_word < a_word):
                #print('3',a_word,b_word,c_word)
                value = ord(c_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(c,c_word, value)
                else:
                    drop_word(c,c_word, -1)
                try:
                    c = pickle.load(drop3)
                    c_word = list(c.keys())[0]
                except EOFError:
                    merge2(drop, drop2, a, b)
                    finish = True
        elif(a_word == b_word):
            if(b_word < c_word):
                d = defaultdict(dict)
                d[a_word] = {**a[a_word], **b[b_word]}
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,a_word, value)
                else:
                    drop_word(d,a_word, -1)
                try:
                    a = pickle.load(drop)
                    a_word = list(a.keys())[0]
                    try:
                        b = pickle.load(drop2)
                        b_word = list(b.keys())[0]
                    except EOFError:
                        merge2(drop, drop3, a, c)
                        finish = True
                except EOFError:
                    try:
                        b = pickle.load(drop2)
                        merge2(drop2, drop3, b, c)
                        finish = True
                    except EOFError:
                        merge1(drop3, c)
                        finish = True
            elif(b_word == c_word):
                d = defaultdict(dict)
                d[a_word] = {**a[a_word], **b[b_word], **c[c_word]}
                value = ord(a_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,a_word, value)
                else:
                    drop_word(d,a_word, -1)
                try:
                    a = pickle.load(drop)
                    try:
                        b = pickle.load(drop2)
                        try:
                            c = pickle.load(drop3)
                        except EOFError:
                            merge2(drop, drop2, a, b)
                            finish = True
                    except EOFError:
                        try:
                            c = pickle.load(drop3)
                            merge2(drop, drop2, a, c)
                            finish = True
                        except EOFError:
                            merge1(drop,a)
                            finish = True
                except EOFError:
                    try:
                        b = pickle.load(drop2)
                        try:
                            c = pickle.load(drop3)
                            merge2(drop2, drop3, b, c)
                            finish = True
                        except EOFError:
                            merge1(drop3, b)
                            finish = True
                    except EOFError:
                        try:
                            c = pickle.load(drop3)
                            merge1(drop3, c)
                            finish = True
                        except EOFError:
                            finish = True
                a_word = list(a.keys())[0]
                b_word = list(b.keys())[0]
                c_word = list(c.keys())[0]
            elif(b_word > c_word):
                value = ord(c_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(c,c_word, value)
                else:
                    drop_word(c,c_word, -1)
                try:
                    c = pickle.load(drop3)
                    c_word = list(c.keys())[0]
                except EOFError:
                    merge2(drop, drop2, a, b)
                    finish = True
        elif(b_word < a_word):
            if(b_word < c_word):
                value = ord(b_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(b,b_word, value)
                else:
                    drop_word(b,b_word, -1)
                try:
                    b = pickle.load(drop2)
                    b_word = list(b.keys())[0]
                except EOFError:
                    try:
                        c = pickle.load(drop3)
                        merge2(drop, drop3, a, c)
                        finish = True
                    except EOFError:
                        merge1(drop, a)
                        finish = True
            elif(b_word == c_word):
                d = defaultdict(dict)
                d[b_word] = {**b[b_word], **c[c_word]}
                value = ord(b_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(d,b_word, value)
                else:
                    drop_word(d,b_word, -1)
                try:
                    b = pickle.load(drop2)
                    b_word = list(b.keys())[0]
                    try:
                        c = pickle.load(drop3)
                        c_word = list(c.keys())[0]
                    except:
                        merge2(drop, drop2, a, b)
                        finish = True
                except EOFError:
                    try:
                        c = pickle.load(drop3)
                        merge2(drop, drop3, a, c)
                        finish = True
                    except EOFError:
                        merge1(drop, a)
                        finish = True
            elif(b_word > c_word):
                value = ord(c_word[0]) - 97
                if(0 <= value <= 25):
                    drop_word(c,c_word, value)
                else:
                    drop_word(c,c_word, -1)
                try:
                    c = pickle.load(drop3)
                    c_word = list(c.keys())[0]
                except EOFError:
                    merge2(drop, drop2, a, b)
                    finish = True
    return

def drop_off3(word_dict):
    #drop off current word_dict into drop3 file
    global drop3
    for word in sorted(word_dict.keys()):
        a = defaultdict(dict)
        a[word] = word_dict[word]
        for x,y in a[word].items():
            if(a[word][x] > 0):
                a[word][x] = (1 + math.log(a[word][x], 10))
        pickle.dump(a, drop3)
    return

def drop_off2(word_dict):
    #drop off current word_dict into drop2 file
    global drop2
    for word in sorted(word_dict.keys()):
        a = defaultdict(dict)
        a[word] = word_dict[word]
        for x,y in a[word].items():
            if(a[word][x] > 0):
                a[word][x] = (1 + math.log(a[word][x], 10))
        pickle.dump(a, drop2)
    return

def drop_off(word_dict):
    #drop off current word_dict into drop file
    global drop
    for word in sorted(word_dict.keys()):
        a = defaultdict(dict)
        a[word] = word_dict[word]
        for x,y in a[word].items():
            if(a[word][x] > 0):
                a[word][x] = (1 + math.log(a[word][x], 10))
        pickle.dump(a, drop)
    return

def short_word(word):
    #if word is <= 1 character, not a or i, and not a number return empty word
    if(len(word) <= 1 and (word != 'a' and word != 'i' and word != 'A' and word != 'I' and word not in ['0','1','2','3','4','5','6','7','8','9'])):
        word = ''
    elif(len(word) > 20):
        word = ''
    return word

def check(word):
    #check word
    if(re.match(r"([.]{0})([\-]{1,})([.]{0})$", word) or re.match(r"(.)*(https://)(.)*", word) or re.match(r"(.)*(http://)(.)*", word)):
        #get rid of words with multiple dashes and urls
        return ''
    if(re.match(r"([0-9]{1,2})(:{1})([0-5]{1}[0-9]{1})(am|pm)$", word)):
        return ''
    if(re.match(r"([0-9]{1,2})(st|nd|rd|th{1})$", word)):
        return word
    if(re.match(r"(.)*(.edu)$", word) and word != 'uci.edu' and word != 'ics.edu' and word != 'ics.uci.edu'):
        #return emails as is
        return word
    if(re.match(r"(([A-Za-z]{1})(\.{1})){2,}", word)):
        #Get rid of extraneous parts of words if acronym in word, then stem
        word = re.sub(r"[^a-z|^A-Z]", '', word)
        return word
    word = re.sub(r"[^a-z|^A-Z|^\'|^\-]", ' ', word)
    if(re.match(r"([\']{3})([A-Za-z\-]+)([\']{3})$", word)):
        #if quotations around word, get rid of them
        word = word[3:-3]
        word = ps.stem(word)
        return word
    if(re.match(r"([A-Za-z\-]+)([\']{3})$", word)):
        #if quotations after word, get rid of them
        word = word[:-3]
        word = ps.stem(word)
        return word
    if(re.match(r"([\']{3})([A-Za-z\-]+)$", word)):
        #if quotations before word, get rid of them
        word = word[3:]
        word = ps.stem(word)
        return word
    if(re.match(r"([\']{1})([A-Za-z\-]+)([\']{1})$", word) or re.match(r"([\-]{1})([A-Za-z\']+)([\-]{1})$",word)):
        #if quotations or dashes around word, get rid of them
        word = word[1:-1]
        word = ps.stem(word)
        return word
    if(re.match(r"([A-Za-z\-]+)([\']{1})$", word) or re.match(r"([A-Za-z\']+)([\-]{1})$", word)):
        #if quotations or dashes after word, get rid of them
        word = word[:-1]
        word = ps.stem(word)
        #word = "".join(word.split())
        return word
    if(re.match(r"([\']{1})([A-Za-z\-]+)$", word) or re.match(r"([\-]{1})([A-Za-z\']+)$", word)):
        #if quotations or dashes before word, get rid of them
        word = word[1:]
        word = ps.stem(word)
        return word
    if(word[-2] != "\'"):
        word = ps.stem(word)
    return word

def insert(word, value):
    #insert words and values into word_dict
    #if word already exists, increment value of appropriate document for word
    global word_dict
    global doc_num
    if word != '':
        if ' ' in word:
            words = word.split()
            for i in words:
                i = check(i)
                if i != '':
                    if(len(i) != 1 or i in ['0','1','2','3','4','5','6','7','8','9']):
                        if i not in word_dict.keys():
                            word_dict[i][doc_num] = value
                        else:
                            if(doc_num not in word_dict[i].keys()):
                                word_dict[i][doc_num] = value
                            else:
                                word_dict[i][doc_num] += value
        else:
            if(word not in word_dict.keys()):
                word_dict[word][doc_num] = value
            else:
                if(doc_num not in word_dict[word].keys()):
                    word_dict[word][doc_num] = value
                else:
                    word_dict[word][doc_num] += value
    return word_dict

def parse(soup, tag, value):
    #parse through the json file words
    global word_dict
    a = soup.find_all(tag)
    for x in range(len(a)):
        i = a[x].text
        for word2 in i.split():
            word = check(word2.lower())
            word_dict = insert(word, value)
    return word_dict

def index():
    #index words into inverted index
    global key
    global doc_num
    global word_dict
    directory_num = 1

    ssl._create_default_https_context = ssl._create_unverified_context
    #https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error
    for subdir, dirs, files in os.walk('DEV'):
        #https://stackoverflow.com/questions/19587118/iterating-through-directories-with-python
        print(str(subdir))
        for filename in files:
            if(str(os.path.join(subdir, filename)).endswith('.DS_Store')):
                #ignore extraneous files (for Mac)
                pass
            else:
                #load files, put url into key dictionary
                a = open(str(os.path.join(subdir, filename)), 'r')
                b = json.load(a)
                key[doc_num] = str(b['url'])
                #https://pypi.org/project/json2html/
                try:
                    #find tags with words and index
                    soup = BeautifulSoup(b['content'], 'lxml')
                    #https://www.crummy.com/software/BeautifulSoup/bs4/doc/
                    if(soup.find('title') != None):
                        word_dict = parse(soup, 'title', 1.5)
                    if(soup.find('p') != None):
                        word_dict = parse(soup, 'p', 1)
                    if(soup.find('h1') != None):
                        word_dict = parse(soup, 'h1', 1.5)
                    if(soup.find('h2') != None):
                        word_dict = parse(soup, 'h2', 1.5)
                    if(soup.find('h3') != None):
                        word_dict = parse(soup, 'h3', 1.5)
                    if(soup.find('h4') != None):
                        word_dict = parse(soup, 'h4', 1)
                    if(soup.find('h5') != None):
                        word_dict = parse(soup, 'h5', 1)
                    if(soup.find('h6') != None):
                        word_dict = parse(soup, 'h6', 1)
                    if(soup.find('b') != None):
                        word_dict = parse(soup, 'b', 1.5)
                    if(soup.find('strong') != None):
                        word_dict = parse(soup, 'strong', 1.5)
                    if(soup.find('i') != None):
                        word_dict = parse(soup, 'i', 1.5)
                    if(soup.find('u') != None):
                        word_dict = parse(soup, 'u', 1.5)
                    doc_num += 1
                except:
                    pass
        if(directory_num == 29):
            drop_off(word_dict)
            global drop
            drop.close()
            word_dict.clear()
        elif(directory_num == 59):
            drop_off2(word_dict)
            global drop2
            drop2.close()
            word_dict.clear()
        elif(directory_num == 89):
            drop_off3(word_dict)
            global drop3
            drop3.close()
            merge()
            break
        directory_num += 1
    #close files
    drop.close()
    drop2.close()
    drop3.close()
    for x in range(len(drop_final_list)):
        drop_final_list[x].close()

    for x in range(0, 26):
        drop_final_list[x] = open(chr(x + 97) + '.txt', 'rb')
    drop_final_list[-1] = open('extra.txt', 'rb')

if __name__ == '__main__':
    index()
    print('done')


# Specialized ion charge state class for STREAM

import matplotlib.pyplot as plt
from DREAM.Output.IonState import IonState as DREAMIonState

class IonState(DREAMIonState):

    def __init__(self, name, Z, Z0, data, grid, output, attr=list()):
        """
        Constructor.
        """
        super().__init__(name=name, Z=Z, Z0=Z0, data=data, grid=grid, output=output, attr=attr)

    def plotBalance(self, ax=None, show=None):
        """
        Generates a plot showing the impact of the various terms in the
        ion rate equation (if saved).
        """
        if 'stream' not in self.output.other:
            raise Exception("The 'stream' category of other quantities was not saved to this output.")

        genax = ax is None

        if genax:
            ax = plt.axes()

            if show is None:
                show = True

        qty = {
            'ionrateequation_posIonization': 'Positive ionization',
            'ionrateequation_negIonization': 'Negative ionization',
            'ionrateequation_posRecombination': 'Positive recombination',
            'ionrateequation_negRecombination': 'Negative recombination',
            'ionrateequation_posChargeExchange': 'Positive charge-exchange',
            'ionrateequation_negChargeExchange': 'Negative charge-exchange'
        }

        nm = self.name.partition('-')[0]
        offset = self.output.eqsys.n_i.getIonOffset(nm, self.Z0)

        stream = self.output.other.stream
        for term, desc in qty.items():
            trm = stream[term][:,offset,0]
            trm = trm.reshape((trm.size,))
            ax.plot(self.grid.t[1:], trm, label=desc)

        ax.legend()

        if show:
            plt.show(block=False)

        return ax


from gtts import gTTS
import googletrans
from ui import Ui_MainWindow
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMessageBox
import sys
from playsound import playsound
import os
from threading import Thread
import PyPDF2

def ReverseDict(d):
    key_dict = []
    value_dict = []
    for i in d:
        value_dict.append(i)
        key_dict.append(d.get(i))

    return dict(zip(key_dict, value_dict))

def Capitilization(s):
	place = s[1:]
	s = s[0].upper() + place
	return s

class GttsWindow(QtWidgets.QMainWindow):

	def __init__(self,*args,**kwargs):
		super().__init__(*args,**kwargs)
		self.langs = ReverseDict(googletrans.LANGUAGES)
		self.ui = Ui_MainWindow()
		self.ui.setupUi(self)

		for i in self.langs:
			self.ui.Language_Selector.addItem(Capitilization(i))

		self.ui.Convert.clicked.connect(self.ConvertFile)
		self.ui.ReadPDFButton.clicked.connect(self.ReadFromPDFFile)
		self.ui.ReadFileButton.clicked.connect(self.ReadFromTextFile)

	def ConvertFile(self):

		msg = QMessageBox()
		msg.setWindowTitle("gTTs Error")
		msg.setIcon(QMessageBox.Critical)
		msg.setStandardButtons(QMessageBox.Close)
		msg.setDefaultButton(QMessageBox.Close)
		icon = QtGui.QIcon()
		icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
		msg.setWindowIcon(icon)

		if(str(self.ui.textEdit.toPlainText())!=""):
			print("here")
			tts = gTTS(str(self.ui.textEdit.toPlainText()) , lang=self.langs.get(self.ui.Language_Selector.currentText().lower()))
			print("here")
			self.audiopath = str(QtWidgets.QFileDialog.getSaveFileName(self,"Save MP3 File" , filter="MP3 Files (*.mp3)")[0])
			self.audiopath =  self.audiopath.replace(r"/" , chr(92))
			if(self.audiopath != ""):
				self.__SaveThread(tts)
				m = QMessageBox()
				m.setWindowTitle("gTTs Preview")
				m.setText(f"Do you want to preview '{str(self.audiopath.split(chr(92))[len(self.audiopath.split(chr(92)))-1])}' ?")
				m.setIcon(QMessageBox.Question)
				m.setStandardButtons(QMessageBox.No | QMessageBox.Yes)
				m.setDefaultButton(QMessageBox.No)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.buttonClicked.connect(self.PreviewSound)
				m.exec_()
			else:
				m = QMessageBox()
				m.setWindowTitle("gTTs Error")
				m.setText(f"File not Saved")
				m.setIcon(QMessageBox.Critical)
				m.setStandardButtons(QMessageBox.Close)
				m.setDefaultButton(QMessageBox.Close)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.buttonClicked.connect(self.PreviewSound)
				m.exec_()
		else:
			msg.setText("No Text Inputted Into Input Field.")
			msg.exec_()

	def PreviewSound(self,pt):
		if(pt.text() == "&Yes"):
			if("tts.mp3" in list(os.listdir())):
				os.remove("tts.mp3")
			with open(str(self.audiopath) , "rb") as g:
				with open("tts.mp3" , "wb") as h:
					h.write(g.read())
					h.close()
					g.close()
			playsound("tts.mp3", False)
		else:
			pass

	def ReadFromTextFile(self):
		path = str(QtWidgets.QFileDialog.getOpenFileName(self , caption="Choose Text File" , filter="Text files (*.txt)")[0]).replace(r"/" , chr(92))
		try:
			with open(path , "r") as f:
				self.ui.textEdit.setText(f.read())
				f.close()
		except:
				m = QMessageBox()
				m.setWindowTitle("gTTs Error")
				m.setText("Text File Not Chosen.")
				m.setIcon(QMessageBox.Critical)
				m.setStandardButtons(QMessageBox.Close)
				m.setDefaultButton(QMessageBox.Close)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.exec_()

	def ReadFromPDFFile(self):
		path = str(QtWidgets.QFileDialog.getOpenFileName(self , caption="Choose PDF File" , filter="PDF files (*.pdf)")[0]).replace(r"/" , chr(92))
		try:
			otpt = """

			"""
			with open(path, "rb") as pdf_file:
				pdf_reader = PyPDF2.PdfFileReader(pdf_file)
				for i in range(0,pdf_reader.getNumPages()):
					otpt = otpt + pdf_reader.getPage(i).extractText()+"\n"
			self.ui.textEdit.setText(otpt)
		except:
				m = QMessageBox()
				m.setWindowTitle("gTTs Error")
				m.setText("PDF File Not Chosen.")
				m.setIcon(QMessageBox.Critical)
				m.setStandardButtons(QMessageBox.Close)
				m.setDefaultButton(QMessageBox.Close)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.exec_()

	def __SaveThread(self,tts_object):
		print("saving")
		tts_object.save(self.audiopath)

if __name__ == "__main__":
	app = QtWidgets.QApplication(sys.argv)
	ui = GttsWindow()
	ui.show()
	sys.exit(app.exec_())


from stylelib import *
import pandas as pd
import json
import matplotlib.patches as mpatches

C_BASE = "BLUE"
C_PROPOSE = "ORANGE"

def get_pattern_patches(labels, patterns):
    patches = []
    for i, l in enumerate(labels):
        p = mpatches.Patch(facecolor=(1, 1, 1, 0), edgecolor='black',alpha=.99,hatch=patterns[i],label=l)
        patches.append(p)
    return patches

def get_color_patches(labels, colors):
    patches = []
    for i, l in enumerate(labels):
        p = mpatches.Patch(facecolor=colors[i], edgecolor='black',alpha=.99, label=l)
        patches.append(p)
    return patches

def reformat_line(line):
    if "-nan" in line:
        return line.replace("-nan", "\"-nan\"")
#     line = line.split("{")[1].split("}")[0]
#     for token in line.split(","):
#         var = token.split(":")[0]
#         if "\"" not in var:
#             print("missing double quote in {}".format(var))
#             line = line.replace(var, "\"{}\"".format(var.strip()))
    return line

def apply_mask(df, mask):
    for m in mask:
        if len(m) == 3:
            df = df[df[m[0]] != m[1]]
        else:
            if isinstance(m[1], list):
                # format [operation, value], e.g. ["le", 0]
                if m[1][0] == "le":
                    df = df[df[m[0]] <= m[1][1]]
                elif m[1][0] == "ge":
                    df = df[df[m[0]] >= m[1][1]]
                elif m[1][0] == "lt":
                    df = df[df[m[0]] < m[1][1]]
                elif m[1][0] == "gt":
                    df = df[df[m[0]] > m[1][1]]
                elif m[1][0] == "eq":
                    df = df[df[m[0]] == m[1][1]]
                elif m[1][0] == "ne":
                    df = df[df[m[0]] != m[1][1]]
            else:
                df = df[df[m[0]] == m[1]]
    return df