import datetime
import time
import sys

# parse user-defined parameters
input_log_file = sys.argv[1]
input_tex_file = sys.argv[2]
output_tex_file = sys.argv[3]

NUM_INTERVALS = 10
MAX_TIME = float(sys.argv[4])

# define some constants
BATCH_PROC_TASK = '@batch_proc_task'
REGULARIZING_TASK = '@regularizing_task'
NORMALIZING_TASK = '@normalizing_task'
MERGING_TASK = '@merging_task'
MERGING_INCREMENTS_TASK = '@merging_increments_task'

GLOBAL_START = 'Log file created at:'
START_BATCH_PROC = 'Processor: start processing batch'
COMPLETE_BATCH_PROC = 'Processor: complete processing batch'
START_REGULARIZING = 'MasterComponent: start regularizing'
COMPLETE_REGULARIZING = 'MasterComponent: complete regularizing'
START_NORMALIZING = 'MasterComponent: start normalizing'
COMPLETE_NORMALIZING = 'MasterComponent: complete normalizing'
START_MERGING = 'MasterComponent: start merging models'
COMPLETE_MERGING = 'MasterComponent: complete merging models'
START_MERGING_INCREMENTS = 'MasterComponent: start merging processor increments'
COMPLETE_MERGING_INCREMENTS = 'MasterComponent: complete merging processor increments'

START_TASK_TYPE = {
    START_BATCH_PROC: BATCH_PROC_TASK,
    START_REGULARIZING: REGULARIZING_TASK,
    START_NORMALIZING: NORMALIZING_TASK,
    START_MERGING: MERGING_TASK,
    START_MERGING_INCREMENTS: MERGING_INCREMENTS_TASK,
}

COMPLETE_TASK_TYPE = {
    COMPLETE_BATCH_PROC: BATCH_PROC_TASK,
    COMPLETE_REGULARIZING: REGULARIZING_TASK,
    COMPLETE_NORMALIZING: NORMALIZING_TASK,
    COMPLETE_MERGING: MERGING_TASK,
    COMPLETE_MERGING_INCREMENTS: MERGING_INCREMENTS_TASK,
}

class Task(object):
    def __init__(self, task_type=None, start_time=None, complete_time=None, batch_name=None, model_name=None, model_index=None, rhs=None):
        self.task_type = task_type
        self.start_time = start_time
        self.complete_time = complete_time
        self.batch_name = batch_name
        self.model_name = model_name
        self.model_index = model_index

        if rhs is not None:
            self.task_type = rhs.task_type
            self.start_time = rhs.start_time
            self.complete_time = rhs.complete_time
            self.batch_name = rhs.batch_name
            self.model_name = rhs.model_name
            self.model_index = rhs.model_index

    def __str__(self):
        return '{0}\n {1}\n {2}\n {3}\n {4}\n {5}\n'.format(self.task_type, self.start_time, self.complete_time,
                                                      self.batch_name, self.model_name, self.model_index)

def parse_log(log_file_name):

    def _make_comparator(less_than):
        def compare(x, y):
            if less_than(x, y):
                return -1
            elif less_than(y, x):
                return 1
            else:
                return 0
        return compare

    def _read_global_start_time(line):
        line_list = line.split()

        month, day = int(line_list[4][5: 7]), int(line_list[4][8: ])
        hour, minute, second = int(line_list[5][0: 2]), int(line_list[5][3: 5]), int(line_list[5][6: ])

        return time.mktime(datetime.datetime(year=2000, month=month, day=day, hour=hour,
                                             minute=minute, second=second).timetuple())

    def _read_task_info(line, global_start_time, read_batch):
        line_list = line.split()

        process_name = line_list[2]
        month, day, hour = int(line_list[0][1: 3]), int(line_list[0][3: ]), int(line_list[1][0: 2])
        minute, second, m_second = int(line_list[1][3: 5]), int(line_list[1][6: 8]), float('0.{}'.format(line_list[1][9: ]))

        current_time = time.mktime(datetime.datetime(year=2000, month=month, day=day, hour=hour,
                                                     minute=minute, second=second).timetuple()) + m_second
        batch_name = None
        if read_batch:
            batch_name = line_list[8]

        model_name = None
        if len(line_list) == 12:
            model_name = line_list[11]

        return process_name, current_time - global_start_time, batch_name, model_name

    max_time_stamp = -1
    min_time_stamp = None
    models_list = []
    main_thread_name = None
    use_merger = False
    process_info = {}  # dict, key --- process name, value --- list of tasks
    with open(log_file_name, 'r') as fin:
        incomplete_tasks = {}
        global_start_time = None
        for line in fin:
            if GLOBAL_START in line:
                global_start_time = _read_global_start_time(line)

            else:
                for task_name, task_type in START_TASK_TYPE.iteritems():
                    if task_name in line:
                        read_batch = (task_type == BATCH_PROC_TASK)

                        if task_type == MERGING_INCREMENTS_TASK:
                            use_merger = True

                        process_name, start_time, batch_name, model_name = _read_task_info(line,
                                                                                global_start_time,
                                                                                read_batch=read_batch)
                        task_name = '{0}-{1}'.format(task_type, process_name)

                        if min_time_stamp is None:
                            min_time_stamp = start_time

                        model_index = None
                        if model_name is not None and (len(models_list) == 0 or model_name != models_list[-1]):
                            models_list.append(model_name)

                        model_index = len(models_list) - 1
                        incomplete_tasks[task_name] = Task(task_type, start_time, -1, batch_name, model_name, model_index)
                        break

                for task_name, task_type in COMPLETE_TASK_TYPE.iteritems():
                    if task_name in line:
                        read_batch = (task_type == BATCH_PROC_TASK)
                        process_name, complete_time, batch_name, model_name = _read_task_info(line,
                                                                                global_start_time,
                                                                                read_batch=read_batch)
                        task_name = '{0}-{1}'.format(task_type, process_name)
                        main_thread_name = process_name  # this would be 'main' in the last task

                        if (task_name not in incomplete_tasks) or\
                             (read_batch and incomplete_tasks[task_name].batch_name != batch_name and\
                                             incomplete_tasks[task_name].model_name != model_name) or\
                              (complete_time < incomplete_tasks[task_name].start_time):
                            raise IOError('Given log-file is incorrect')

                        incomplete_tasks[task_name].complete_time = complete_time

                        if complete_time > max_time_stamp:
                            max_time_stamp = complete_time

                        if process_name not in process_info:
                            process_info[process_name] = []

                        process_info[process_name].append(Task(rhs=incomplete_tasks[task_name]))
                        del incomplete_tasks[task_name]
                        break

    for process_name in process_info.keys():
        process_info[process_name].sort(cmp=_make_comparator(lambda x, y: x.start_time > y.start_time), reverse=True)

    return process_info, models_list, max_time_stamp, min_time_stamp, main_thread_name, use_merger

process_info, models_list, max_time_stamp, min_time_stamp, main_thread_name, use_merger = parse_log(input_log_file)
for _, tasks in process_info.iteritems():
    for task in tasks:
        task.start_time -= min_time_stamp
        task.complete_time -= min_time_stamp

max_time_stamp -= min_time_stamp

# write info to latex file
coef = NUM_INTERVALS / max_time_stamp
line_len_value = 5.7 #5.4 - 0.35 * (len(process_info.keys()) - 1)  # incorrect!
with open(input_tex_file, 'r') as fin:
    with open(output_tex_file, 'w') as fout:
        for line in fin:
            if '%%%%% START_MARKER_1 %%%%%' not in line and\
               '%%%%% START_MARKER_2 %%%%%' not in line and\
               '%%%%% START_MARKER_3 %%%%%' not in line:
                fout.write(line)
            elif '%%%%% START_MARKER_1 %%%%%' in line:
                value = MAX_TIME / (10.0 * NUM_INTERVALS) * coef
                fout.write('-- ++(0,-0.1) -- ++(0,0.05)\n')
                for _ in xrange(4):
                    fout.write('-- ++({},0) ++(0,-0.02) -- ++(0,0.02)\n'.format(value))
                fout.write('-- ++({},0)  -- ++(0,-0.035) -- ++(0,0.035)\n'.format(value))
                for _ in xrange(5):
                    fout.write('-- ++({},0) ++(0,-0.02) -- ++(0,0.02)\n'.format(value))
                fout.write('-- ++(0,-0.05) -- ++(0,0.1)\n')
            elif '%%%%% START_MARKER_2 %%%%%' in line:
                iter = MAX_TIME / (1.0 * NUM_INTERVALS)
                for i in [int(iter * i) for i in xrange(NUM_INTERVALS + 1)]:
                    fout.write('\draw[dotted] (t_cur) +(0,-.1) node[above=0.05cm] {{\\tiny {}s}};\n'.format(i))
                    fout.write('\clki\n')
            else:
                fout.write('\\begin{{wave}}{{{0}}}{{10}}{{{1:.2f}}}{{{2}}}\n'.format(len(process_info.keys()), max_time_stamp, line_len_value))
                process_names_list = process_info.keys()
                process_names_list.remove(main_thread_name)
                process_names_list = [main_thread_name] + process_names_list

                for p_i, p_name in enumerate(process_names_list):
                    name_to_print = 'Main' if not use_merger else 'Merger'
                    if p_i > 0:
                        name_to_print = 'Proc-{}'.format(p_i)
                    process_str = '\\nextwave{{{}}} '.format(name_to_print)
                    process_str += '\\Wait{{ }}{{{}}}'.format(process_info[p_name][0].start_time * coef)
                    for idx, _ in enumerate(process_info[p_name]):
                        if idx != len(process_info[p_name]) - 1:
                            val_1 = (process_info[p_name][idx].complete_time - process_info[p_name][idx].start_time) * coef
                            val_2 = (process_info[p_name][idx + 1].start_time - process_info[p_name][idx].complete_time) * coef

                            if process_info[p_name][idx].task_type == BATCH_PROC_TASK:
                                process_str += '\\ProcBatch{0}[]{{{1}}} \\Wait{{ }}{{{2}}} '.format(
                                    'One' if process_info[p_name][idx].model_index % 2 else 'Two', val_1, val_2)
                            elif process_info[p_name][idx].task_type == REGULARIZING_TASK:
                                process_str += '\\Regularization[]{{{0}}} \\Wait{{ }}{{{1}}} '.format(val_1, val_2)
                            elif process_info[p_name][idx].task_type == NORMALIZING_TASK:
                                process_str += '\\Normalization[]{{{0}}} \\Wait{{ }}{{{1}}} '.format(val_1, val_2)
                            elif process_info[p_name][idx].task_type == MERGING_TASK:
                                process_str += '\\Merge[]{{{0}}} \\Wait{{ }}{{{1}}} '.format(val_1, val_2)
                            elif process_info[p_name][idx].task_type == MERGING_INCREMENTS_TASK:
                                process_str += '\\MergeInc[]{{{0}}} \\Wait{{ }}{{{1}}} '.format(val_1, val_2)
                        else:
                            val = (process_info[p_name][idx].complete_time - process_info[p_name][idx].start_time) * coef
                            if process_info[p_name][idx].task_type == BATCH_PROC_TASK:
                                process_str += '\\ProcBatch{0}[]{{{1}}} '.format(
                                    'One' if process_info[p_name][idx].model_index % 2 else 'Two', val)
                            elif process_info[p_name][idx].task_type == REGULARIZING_TASK:
                                process_str += '\\Regularization[]{{{0}}} '.format(val)
                            elif process_info[p_name][idx].task_type == NORMALIZING_TASK:
                                process_str += '\\Normalization[]{{{0}}} '.format(val)
                            elif process_info[p_name][idx].task_type == MERGING_TASK:
                                process_str += '\\Merge[]{{{0}}} '.format(val)
                            elif process_info[p_name][idx].task_type == MERGING_INCREMENTS_TASK:
                                process_str += '\\MergeInc[]{{{0}}} '.format(val)

                    fout.write('{}\n'.format(process_str))


import pandas as pd
import csv
import argparse

def findPredictById(row,data_pred,task):
	val = data_pred[data_pred['idtwitter'] == row.idtwitter]
	print(val)
	if task == 'pos' :
		return val['pred_pos'].values[0]
	if task == 'neg' :
		return val['pred_neg'].values[0]
	return -1

if __name__ == "__main__":

	parser = argparse.ArgumentParser()
	parser.add_argument('--gold', type=str, help="gold file for evaluate sistem", default='./data/test_gold.csv')
	parser.add_argument('--predictPos', type=str, help="output dir", default='./res_bert_try_opos.csv')
	parser.add_argument('--predictNeg', type=str, help="Path of csv for train ", default='./res_bert_try_oneg.csv')
	parser.add_argument('--runName', type=str, help="name for run ", default='001')

	args = parser.parse_args()
	gold = args.gold
	predPos = args.predictPos
	predNeg = args.predictNeg
	data_data_gold = pd.read_csv(gold,sep='\t')
	data_pred = pd.read_csv(predPos,sep='|')
	data_pred.pred_pos.astype(str)

	print(data_pred.head())
	data_pred_neg = pd.read_csv(predNeg,sep='|')
	data_pred_neg.pred_neg.astype(str)

	data_data_gold['pred_pos'] = data_data_gold.apply(lambda row: findPredictById(row,data_pred,'pos'), axis=1)
	data_data_gold['pred_neg'] = data_data_gold.apply(lambda row: findPredictById(row,data_pred_neg,'neg'), axis=1)
	print(data_data_gold.head())

	data_to_save = data_data_gold[['idtwitter','subj','pred_pos','pred_neg','iro','lpos','lneg','top']]

	data_to_save.idtwitter = data_to_save.idtwitter.astype(str)
	data_to_save.subj = data_to_save.subj.astype(str)
	data_to_save.pred_pos = data_to_save.pred_pos.astype(str)
	data_to_save.pred_neg = data_to_save.pred_neg.astype(str)
	data_to_save.iro = data_to_save.iro.astype(str)
	data_to_save.lpos = data_to_save.lpos.astype(str)
	data_to_save.lneg = data_to_save.lneg.astype(str)
	data_to_save.top = data_to_save.top.astype(str)
	data_to_save.to_csv(r'./res_bert_to_evaluete_'+args.runName+'.csv',header=None,index=None, sep=',', mode='a', quotechar='"',quoting=csv.QUOTE_NONNUMERIC)


#/usr/bin/env
"""A simple async web crawler"""

import asyncio
import argparse
import logging
import sys
import json
import re
from typing import Dict
from urllib import parse

import aiohttp
import async_timeout
import html5lib
from bs4 import BeautifulSoup
from pprint import pformat

from utils import get_parent_url

parser = argparse.ArgumentParser(description='Run web crawler asynchronously')
parser.add_argument('--url', type=str, required=True, help='Source URL to start the crawler from')
parser.add_argument('--workers', type=int, default=5, help='Number of workers to crawl')
parser.add_argument('--verbose', action='store_true', help='Detailed output to stdout')
parser.add_argument('--save', action='store_true', help='Save to crawler_recipes.json file')

logging.basicConfig(stream=sys.stdout, format="%(message)s")
log = logging.getLogger()
log.setLevel(logging.INFO)

crawled_urls = set()

def update_queue(queue: asyncio.Queue, soup: BeautifulSoup, regexp: str, parent_url: str) -> None:
  """Update queue with new links

  Args:
    queue (asyncio.Queue): The asyncio queue that the work has to be done on
    soup (BeautifulSoup): BeautifulSoup object for the page
    regex (str): Reg exp to validate recipe URL
    parent_url (str): Parent URL
  """

  for link in soup.find_all('a', href=True):
    new_link = parse.urljoin(parent_url, link['href'])
    if re.search(regexp, new_link) and get_parent_url(new_link) == parent_url and new_link not in crawled_urls:
      queue.put_nowait(new_link)

def process_recipe(recipe: Dict, verbose: bool = False, save: bool = False) -> None:
  """Process recipe dictionary based on verbose printing option or saving to JSON

  Args:
    recipe (Dict): Recipe information
    verbose (bool, optional): Defaults to False. Flag to log out to terminal
    save (bool, optional): Defaults to False. Flag to save to crawler_recipes.json file
  """

  if not recipe:
    pass
  if verbose:
    log.info(pformat(recipe))
  elif save:
    with open('crawler_recipes.json', 'a+') as f:
      f.write(json.dumps(recipe) + '\n')

def parse_page_soup(url: str, soup: bytes) -> Dict:
  """Parse the recipe page soup to get required recipe information

  Args:
    url (str): Recipe URL from which the page bytes are fetched
    soup (BeautifulSoup): BeautifulSoup object for the page

  Returns:
    Dict: Recipe information
  """
  try:
    recipe_name = soup.select('h1')[0].text
    recipe_photo = soup.select('div.recipe-details-image > img')[0]['src']
    ingredients = [' '.join([t.text for t in li.find_all('span', recursive=False)]).replace(
        u'\xa0', u' ').strip() for li in soup.select('div.recipe-ingredients li')]
    half_star_tags = soup.select('#reviews span.half-star')
    full_stars_tags = soup.select('a.recipe-details-rating span.full-star')
    half_star = float(half_star_tags[0]['data-star-number']) if half_star_tags else 0.0
    full_stars = float(full_stars_tags[-1]['data-star-number']) if full_stars_tags else 0.0
    ratings = full_stars + (0.5 if full_stars < 5 and half_star else 0.0)
    reviews = int(soup.select('#reviews span')[0].text.strip('()'))
    cook_time = ' '.join(t.text for t in soup.select(
        'div.recipe-summary-item.unit span')).replace(u'\xa0', u' ')
    serve_tags = soup.select('div.servings input')
    serve = int(serve_tags[0]['value']) if serve_tags else 1
    return dict(recipeName=recipe_name,
                recipePhoto=recipe_photo,
                ingredients=ingredients,
                ratings=ratings,
                reviews=reviews,
                cookTime=cook_time,
                serve=serve)
  except:
    return {}

async def get_page_bytes(url: str) -> bytes:
  """Get page in bytes from URL

  Args:
    url (str): URL to fetch the page from

  Returns:
    bytes: Page in bytes
  """

  async with aiohttp.ClientSession() as session:
    async with session.get(url) as response:
      if response.status == 200:
        page_bytes = await response.read()
        return page_bytes
      return None

async def crawl(queue: asyncio.Queue,
                parent_url: str,
                verbose: bool = False,
                save: bool = False,
                regex: str = r'/recipe/.+') -> None:
  """Crawls URLs in the queue while updating the queue itself with URLs

  Args:
    queue (asyncio.Queue): The asyncio queue that the work has to be done on
    parent_url (str): Parent URL
    verbose (bool, optional): Defaults to False. Flag to log out to terminal
    save (bool, optional): Defaults to False. Flag to save to crawler_recipes.json file
    regex (str, optional): Defaults to r'/recipe/.+'. Reg exp to validate recipe URL
  """

  global crawled_urls
  try:
    while True:
      url = await queue.get()
      if url not in crawled_urls:
        crawled_urls.add(url)
        page_bytes = await get_page_bytes(url)
        if page_bytes:
          soup = BeautifulSoup(page_bytes.decode('utf-8'), 'html5lib')
          recipe = parse_page_soup(url, soup)
          process_recipe(recipe, verbose, save)
          update_queue(queue, soup, regex, parent_url)
      queue.task_done()
  except asyncio.CancelledError:
    log.debug('Error: The worker has been cancelled; perhaps on purpose')
    pass

def main() -> None:
  """Parses commandline arguments, starts async IO events loop and creates workers to work on queue
  """

  args = parser.parse_args()
  parent_url = get_parent_url(args.url)
  if not parent_url:
    raise Exception('Unsupported URL provided')
  if not args.verbose and args.save:
    with open('crawler_recipes.json', 'w'):
      pass
  loop = asyncio.get_event_loop()
  queue = asyncio.Queue()
  queue.put_nowait(args.url)
  workers = [asyncio.ensure_future(crawl(queue, parent_url, verbose=args.verbose, save=args.save)) for _ in range(args.workers)]
  loop.run_until_complete(queue.join())
  for w in workers:
    w.cancel()
  loop.close()

if __name__ == '__main__':
  main()


import time
from typing import List, Type, Union

from acme.jax import savers as acme_savers
from chex import dataclass

from mava.callbacks import Callback
from mava.components import Component
from mava.components.updating.parameter_server import ParameterServer
from mava.core_jax import SystemParameterServer
from mava.utils.checkpointing_utils import update_to_best_net
from mava.wrappers import SaveableWrapper

"""Checkpointer component for Mava systems."""

@dataclass
class CheckpointerConfig:
    checkpoint_minute_interval: float = 5
    restore_best_net: Union[str, None] = None

class Checkpointer(Component):
    def __init__(
        self,
        config: CheckpointerConfig = CheckpointerConfig(),
    ):
        """Component for checkpointing system variables."""
        self.config = config

    def on_parameter_server_init(self, server: SystemParameterServer) -> None:
        """Create the system checkpointer.

        Args:
            server: SystemParameterServer.

        Returns:
            None.
        """
        saveable_parameters = SaveableWrapper(server.store.parameters)
        old_trainer_steps = server.store.parameters["trainer_steps"].copy()
        server.store.system_checkpointer = acme_savers.Checkpointer(
            object_to_save=saveable_parameters,  # must be type saveable
            directory=server.store.experiment_path,
            add_uid=False,
            time_delta_minutes=0,
        )

        # Check if the checkpointer restored the network parameters
        # and if the user wants the network with the best performance.
        if (old_trainer_steps != server.store.parameters["trainer_steps"]) and (
            self.config.restore_best_net is not None
        ):
            update_to_best_net(server, self.config.restore_best_net)

        server.store.last_checkpoint_time = time.time()
        server.store.checkpoint_minute_interval = self.config.checkpoint_minute_interval

    def on_parameter_server_run_loop_checkpoint(
        self, server: SystemParameterServer
    ) -> None:
        """Intermittently checkpoint the server parameters.

        Args:
            server: SystemParameterServer.

        Returns:
            None.
        """
        if (
            time.time() - server.store.last_checkpoint_time
            > self.config.checkpoint_minute_interval * 60 + 1
        ):
            server.store.system_checkpointer.save()
            server.store.last_checkpoint_time = time.time()

    @staticmethod
    def name() -> str:
        """Static method that returns component name."""
        return "checkpointer"

    @staticmethod
    def required_components() -> List[Type[Callback]]:
        """List of other Components required in the system for this Component to function.

        ParameterServer required to set up server.store.parameters
        and server.store.experiment_path.

        Returns:
            List of required component classes.
        """
        return [ParameterServer]


import sqlite3
from sqlite3 import Error

def createConnection(dbFile):
    """
        create a database connection to the SQLite database
        specified by db_file
    """
    conn = None
    try:
        conn = sqlite3.connect(dbFile)
    except Error as e:
        print(e)
    return conn


#! usr/bin/python3.5
import numpy as np
import PCApackage as pac
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from sklearn.decomposition import PCA
import os

os.system('wget ftp://ftp.ncbi.nlm.nih.gov/geo/datasets/GDS6nnn/GDS6248/soft/GDS6248.soft.gz')
os.system('gunzip GDS6248.soft.gz')
os.system('tail -n +141 GDS6248.soft > GDS6248.softer') #getting rid of the redundant lines
os.system('rm GDS6248.soft')
os.system('head -n -1 GDS6248.softer > GDS6248.soft') #one last redundant line
os.system('rm GDS6248.softer')

#In the following loop I'm keeping the float values while skipping the strings by setting the ValueError exception
temp = []
with open('GDS6248.soft') as f:
	for l in f:
		temp2=[]
		for x in l.split()[2:]:
			try:
				temp2.append(float(x))
			except ValueError:
				pass
		temp.append(temp2)

X=np.array(temp)
Color = ['m' for x in range(3)] + ['c' for x in range(24)] + ['r' for x in range(24)] #Color scheme for the samples

plt.figure()
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0.5, hspace=0.5)

green = mpatches.Patch(color='m',label='baseline')
cyan = mpatches.Patch(color='c',label='normal diet')
red = mpatches.Patch(color='r',label='high fat diet')

Y=pac.PCA(X,k=2, F=False)[0]
plt.subplot(2,2,1)
plt.title('After Linear PCA')
plt.scatter(Y[0], Y[1], c=Color)
plt.legend(bbox_to_anchor=(1, -.1), loc=2, borderaxespad=0.,handles=[green, cyan, red], prop={'size':10})
Y=pac.PPCA(X, 2)
plt.subplot(2,2,2)
plt.title('After PPCA')
plt.scatter(Y[0], Y[1], c=Color)

Y=pac.PPCA(X, M=2, variance=False)
plt.subplot(2,2,3)
plt.title('After PPCA, zero sigma')
plt.scatter(Y[0], Y[1], c=Color)

my_pca=PCA(n_components=2)
Y = my_pca.fit_transform(X.T).T
plt.subplot(2,2,4)
plt.title('After Sklearn PCA')
plt.scatter(Y[0], Y[1], c=Color)
plt.show()


from django import forms

from tinymce.widgets import TinyMCE
from .models import Subject,InfoList

class InfoListForm(forms.ModelForm):
    def __init__(self,department,year,*args,**kwargs):
        super(InfoListForm,self).__init__(*args,**kwargs)
        self.fields['subject'].queryset = Subject.objects.filter(department=department,year=year)

    class Meta:
        model = InfoList
        fields = ['subject','title','content']
        widgets = {
            'content':TinyMCE(attrs={'cols': 80, 'rows': 15})
        }

class SubjectForm(forms.ModelForm):
    class Meta:
        model = Subject
        fields = ['subject']


"""
@brief      test log(time=4s)
@author     Xavier Dupre
"""
import os
import unittest
import warnings
import logging
from io import StringIO
from docutils.parsers.rst import directives
from pyquickhelper.pycode import get_temp_folder, ExtTestCase
from pyquickhelper.helpgen import rst2html
from pyquickhelper.helpgen.sphinxm_custom_app import CustomSphinxApp
from pyquickhelper.sphinxext.sphinximages.sphinxtrib.images import ImageDirective

class TestImageExtension(ExtTestCase):

    def setUp(self):
        logger = logging.getLogger('gdot')
        logger.disabled = True

    def test_post_parse_sn(self):
        directives.register_directive("image", ImageDirective)

    def test_thumbnail(self):
        from docutils import nodes as skip_

        content = """
                    test a directive
                    ================

                    before

                    .. thumbnail:: http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/_static/project_ico.png
                        :width: 10
                        :height: 20
                        :download: 1

                    after

                    this code shoud appear
                    """.replace("                    ", "")
        content = content.replace('u"', '"')

        logger2 = logging.getLogger("video")

        log_capture_string = StringIO()
        ch = logging.StreamHandler(log_capture_string)
        ch.setLevel(logging.DEBUG)
        logger2.addHandler(ch)
        with warnings.catch_warnings(record=True):
            html = rst2html(content,  # fLOG=fLOG,
                            writer="custom", keep_warnings=True,
                            directives=None)

        warns = log_capture_string.getvalue()
        if warns:
            raise Exception(warns)

        t1 = "this code shoud not appear"
        if t1 in html:
            raise Exception(html)

        t1 = "this code shoud appear"
        if t1 not in html:
            raise Exception(html)

        t1 = "_images"
        if t1 not in html:
            raise Exception(html)

        t1 = "linkedin"
        if t1 in html:
            raise Exception(html)

        temp = get_temp_folder(__file__, "temp_sphinx_thumbnail")
        with open(os.path.join(temp, "out_image.html"), "w", encoding="utf8") as f:
            f.write(html)

    def test_sphinx_ext_thumbnail_html(self):
        temp = get_temp_folder(__file__, "temp_sphinx_ext_thumbnail_html")
        src_ = os.path.join(temp, "..", "data", "image")

        app = CustomSphinxApp(src_, temp)
        app.build()

        index = os.path.join(temp, "index.html")
        self.assertExists(index)
        img = os.path.join(temp, '_images', 'im.png')
        self.assertExists(img)

if __name__ == "__main__":
    unittest.main()


import os.path
import time
from PIL import Image
from threading import Thread
import cv2
import os
from flask import Flask,request,render_template
from datetime import date
from datetime import datetime
import numpy as np
import pandas as pd

#### Defining Flask App
app = Flask(__name__)

#### Saving Date today in 2 different formats
datetoday = date.today().strftime("%m_%d_%y")
datetoday2 = date.today().strftime("%d-%B-%Y")

#### Initializing VideoCapture object to access WebCam
face_detector = cv2.CascadeClassifier('static/haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0)

#### If these directories don't exist, create them
if not os.path.isdir('Attendance'):
    os.makedirs('Attendance')
if not os.path.isdir('datasets'):
    os.makedirs('datasets')
if f'Attendance-{datetoday}.csv' not in os.listdir('Attendance'):
    with open(f'Attendance/Attendance-{datetoday}.csv','w') as f:
        f.write('Name,Roll,Time')

#### get a number of total registered users
def totalreg():
    return len(os.listdir('datasets'))

# -------------- image labesl ------------------------

def getImagesAndLabels(path):
    # create empth face list
    faces = []
    # create empty ID list
    Ids = []
    for folder in os.listdir(path):
        # get the path of all the files in the folder
        folder_path = os.path.join(path,folder)
        imagePaths = [os.path.join(folder_path, f) for f in os.listdir(folder_path)]
        # now looping through all the image paths and loading the Ids and the images
        for imagePath in imagePaths:
            # loading the image and converting it to gray scale
            pilImage = Image.open(imagePath).convert('L')
            # Now we are converting the PIL image into numpy array
            imageNp = np.array(pilImage, 'uint8')
            # getting the Id from the image
            Id = int(os.path.split(imagePath)[-1].split(".")[1])
            # extract the face from the training image sample
            faces.append(imageNp)
            Ids.append(Id)
    return faces, Ids

#### Extract info from today's attendance file in attendance folder
def extract_attendance():
    df = pd.read_csv(f'Attendance/Attendance-{datetoday}.csv')
    names = df['Name']
    rolls = df['Roll']
    times = df['Time']
    l = len(df)
    return names,rolls,times,l

#### Add Attendance of a specific user
def add_attendance(id):
    for name in os.listdir('datasets'):
        if name.split('_')[1] == id:
            username = name.split('_')[0]
            userid = name.split('_')[1]
            current_time = datetime.now().strftime("%H:%M:%S")

            df = pd.read_csv(f'Attendance/Attendance-{datetoday}.csv')
            if int(userid) not in list(df['Roll']):
                with open(f'Attendance/Attendance-{datetoday}.csv', 'a') as f:
                    f.write(f'\n{username},{userid},{current_time}')

### A function which trains the model on all the faces available in faces folder
def train_model():
    # ----------- train images function ---------------
    recognizer = cv2.face.LBPHFaceRecognizer_create()
    harcascadePath = "static/haarcascade_frontalface_default.xml"
    detector = cv2.CascadeClassifier(harcascadePath)
    faces, Id = getImagesAndLabels("datasets")
    Thread(target=recognizer.train(faces, np.array(Id))).start()
    # Below line is optional for a visual counter effect
    Thread(target=counter_img("datasets")).start()
    recognizer.save("static/Trainner.yml")
    print("All Images")

# Optional, adds a counter for images trained (You can remove it)
def counter_img(path):
    imgcounter = 1
    imagePaths = [os.path.join(path, f) for f in os.listdir(path)]
    for imagePath in imagePaths:
        print(str(imgcounter) + " Images Trained", end="\r")
        time.sleep(0.008)
        imgcounter += 1

def get_name_from_ID(id):
    for folder in os.listdir('datasets'):
        name, str_id = folder.split('_')
        if int(str_id) == id:
            return name
#-------------------------start recognition ---------------------------------
def face_recognition():  # generate frame by frame from camera
    def draw_boundary(img, classifier, scaleFactor, minNeighbors, color, clf):
        gray_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        features = classifier.detectMultiScale(gray_image, scaleFactor, minNeighbors)

        for (x, y, w, h) in features:
            cv2.rectangle(img, (x, y), (x + w, y + h), color, 2)
            id, pred = clf.predict(gray_image[y:y + h, x:x + w])
            confidence = int(100 * (1 - pred / 300))
            add_attendance(str(id))
            s = " id: " + str(id)
            name = get_name_from_ID(id) + s
            if confidence > 70:
                cv2.putText(img, name, (x, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.8, color, 1, cv2.LINE_AA)
            else:
                cv2.putText(img, "UNKNOWN", (x, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 1, cv2.LINE_AA)

    def recognize(img, clf, faceCascade):
        draw_boundary(img, faceCascade, 1.1, 10, (255, 255, 0), clf)

    faceCascade = cv2.CascadeClassifier("static/haarcascade_frontalface_default.xml")
    clf = cv2.face.LBPHFaceRecognizer_create()
    clf.read("static/Trainner.yml")

    wCam, hCam = 500, 400

    cap = cv2.VideoCapture(0)
    cap.set(3, wCam)
    cap.set(4, hCam)

    while True:
        ret, img = cap.read()
        recognize(img, clf, faceCascade)
        cv2.imshow('Attendance',img)
        key = cv2.waitKey(1)
        if key == 27:
            break

################## ROUTING FUNCTIONS #########################

#### Our main page
@app.route('/')
def home():
    names,rolls,times,l = extract_attendance()
    return render_template('home.html',names=names,rolls=rolls,times=times,l=l,totalreg=totalreg(),datetoday2=datetoday2)

#### This function will run when we click on Take Attendance Button
@app.route('/start',methods=['GET'])
def start():
    if 'Trainner.yml' not in os.listdir('static'):
        return render_template('home.html',totalreg=totalreg(),datetoday2=datetoday2,mess='There is no trained model in the static folder. Please add a new face to continue.')

    face_recognition()
    cap.release()
    cv2.destroyAllWindows()
    names,rolls,times,l = extract_attendance()
    return render_template('home.html',names=names,rolls=rolls,times=times,l=l,totalreg=totalreg(),datetoday2=datetoday2)

@app.route('/add',methods=['GET','POST'])
def add():
    name = request.form['newusername']
    Id = request.form['newuserid']
    userimagefolder = 'datasets/' + name + '_' + str(Id)
    if not os.path.isdir(userimagefolder):
        os.makedirs(userimagefolder)
    cam = cv2.VideoCapture(0)
    harcascadePath = 'static/haarcascade_frontalface_default.xml'
    detector = cv2.CascadeClassifier(harcascadePath)
    sampleNum = 0
    maxSample = 50

    while (True):
        ret, img = cam.read()
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        faces = detector.detectMultiScale(gray, 1.3, 5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE)
        for (x, y, w, h) in faces:
            cv2.rectangle(img, (x, y), (x + w, y + h), (10, 159, 255), 2)
            # incrementing sample number
            sampleNum = sampleNum + 1
            # saving the captured face in the dataset folder TrainingImage
            cv2.imwrite(userimagefolder + '/' + name + "." + str(Id) + '.' + str(sampleNum) + ".jpg", gray[y:y + h, x:x + w])
            # display the frame
            cv2.imshow('frame', img)
        # wait for 100 miliseconds
        if cv2.waitKey(100) & 0xFF == ord('q'):
            break
        # break if the sample number is more than 100
        elif sampleNum >= maxSample:
            break
    cam.release()
    cv2.destroyAllWindows()
    print('Training Model')
    train_model()
    names, rolls, times, l = extract_attendance()
    return render_template('home.html', names=names, rolls=rolls, times=times, l=l, totalreg=totalreg(), datetoday2=datetoday2)

if __name__ == '__main__':
    app.run(debug=True)


#!/usr/bin/env python
# coding=utf-8

"""Helper functions"""

import re
import bert
import jieba
import logging
import feather
import operator
import unicodedata
import numpy as np
import pandas as pd
import tensorflow as tf
import multiprocessing as mp
from functools import partial
from dbfread import DBF
from opencc import OpenCC
from bert import tokenization
from keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder

# disable warnings in jieba packages
jieba.setLogLevel(logging.CRITICAL)

def normalize_full_width(text):
    """
    a function to normalize full width characters
    """
    return unicodedata.normalize('NFKC', text)

def sep_chi_eng(text):
    """
    a function to separate adjacent Chinese and English words from each other with a space character
    e.g. sep_chi_eng("apple手機iphone") = "apple 手機 iphone"
    """
    text = re.sub("([a-z])([\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC])", "\\1 \\2", text)
    text = re.sub("([\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC])([a-z])", "\\1 \\2", text)
    return text

def segment_chi(text):
    """
    a function to segment Chinese text using jieba
    e.g. segment_chi("蘋果手機") = "蘋果 手機"
    """
    return ' '.join(jieba.cut(text, cut_all=False))

def case_lower(text):
    """
    a function to convert English letters to lower case
    """
    return text.lower()

def remove_punct(text):
    """
    a function to remove punctuation
    """
    return re.sub("[^\w\s]", " ", text)

def detect_chi(text):
    """
    a function to return True if text containing at least 1 Chinese character, False otherwise.
    """
    return bool(re.search(u"[\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC]", text))

class ModifiedWordPieceTokenizer(object):
    """
    a text tokenizer for performing modified Word Piece segmentation. The tokenizer is modified based on BERT's.
    It preserves the segmented Chinese instead of converting some of them to the single and special [UKN] token.

    Package required: bert, re
    """

    def __init__(self, bert_vocab_file, do_lower_case):
        self.bert_vocab_file = bert_vocab_file
        self.do_lower_case = do_lower_case
        self.tokenizer = bert.tokenization.FullTokenizer(vocab_file=self.bert_vocab_file,
                                                         do_lower_case=self.do_lower_case)

    def tokenize(self, text):
        """
        tokenize the input text

        INPUT:
            text: string
        OUTPUT:
            string
        """
        wordPieces = self.tokenizer.tokenize(text)
        tokens = text.lower().split()

        i = 0  # index for tokens
        j = 0  # index for wordPieces
        string = ''  # for storing the concatenation of wordPieces
        output = []  # empty list to store the English wordPieces and concatenated Chinese vocabularies

        for wordPiece in wordPieces:
            wordPiece = re.sub('##', '',
                               wordPiece)  # substitute the special "##" symbol for concatenation of wordPieces later on
            if wordPiece == '[UNK]':
                wordPiece = '難'  # substitute the [UNK] character for Chinese to one single arbitray Chinese word 難
            string += wordPiece  # add the wordPiece to string

            # match base on comparing length of token and string but not at textual level due to presence of [UKN]
            # character/ the arbitray character is created such that the unknown Chinese word still remain a word
            # length of 1
            if len(string) == len(tokens[i]):
                if bool(re.search(u"[\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC]|[UNK]", string)):
                    # append token instead of string due to possible presence of unknown
                    # Chinese character in string
                    output.append(tokens[i])
                else:
                    # append English wordPieces
                    output.append(wordPieces[j])
                i += 1  # after matching 1 token, increase the token index by 1
                string = ''  # reset the string for comparison in the next token
            elif not bool(re.search(u"[\u2E80-\u2FD5\u3400-\u4DBF\u4E00-\u9FCC]", wordPiece)):
                # append English wordPieces
                output.append(wordPieces[j])

            j += 1  # next wordPieces

        return ' '.join(output)

def text_cleaning(text_array, word_tokenization, bert_vocab_file, do_lower_case, num_threads=1):
    """
    a function to clean text, including
    - Normalize full width characters
    - Convert all English letters to lower case
    - Translate Chinese simplified characters to traditional Chinese
    - Separate Chinese and English tokens from each other
    - Segment Chinese text
    - remove punctuation
    - perform the special Word Piece Segmentation if specified by the argument "word_tokenization"

    INPUT:
        text_array: list/numpy array/pandas Series
        word_tokenization: method of word segmentation, either split by "space" or "word_piece" tokenization
        bert_vocab_file: string --path to the .txt file of vocabularies for modified Word Piece Tokenizer
        do_lower_case: boolean --whether the modified Word Piece Tokenizer converts English letters to lower case or not
        num_threads: int --number of CPU processors for performing the text cleaning
    OUTPUT:
        text_array: a list of string (cleaned text)
    """

    with mp.Pool(processes=num_threads) as pool:

        print("   Normalizing full-width characters...")
        text_array = pool.map(normalize_full_width, text_array)

        print("   Converting English letters to lower case...")
        text_array = pool.map(case_lower, text_array)

        print("   Translating Simplified Chinese to Traditional...")
        bool_contain_chi = pool.map(detect_chi, text_array)
        text_array = np.array(text_array)
        cc = OpenCC('s2t')
        text_array[bool_contain_chi] = list(map(lambda x: cc.convert(x), text_array[bool_contain_chi]))

        print("   Separating Chinese amd English word tokens from each other...")
        text_array = pool.map(sep_chi_eng, text_array)

        print("   Segmenting Chinese vocabularies...")
        text_array = np.array(text_array)
        dtype_0 = str(text_array.dtype)
        dtype_len_0 = int(re.findall("[0-9]+", dtype_0)[0])

        if sum(bool_contain_chi):
            replacement = pool.map(segment_chi, text_array[bool_contain_chi])
            replacement = np.array(replacement)
            dtype_1 = str(replacement.dtype)
            dtype_len_1 = int(re.findall("[0-9]+", dtype_1)[0])

            # change the type of text_array to that of the replacement
            # in order to avoid some of the characters in the segmented text being trimmed
            if dtype_len_0 > dtype_len_1:
                text_array = text_array.astype(dtype_0)
            else:
                text_array = text_array.astype(dtype_1)

            text_array[bool_contain_chi] = replacement

        print("   Removing punctuation...")
        text_array = pool.map(remove_punct, text_array)

        if word_tokenization == "word_piece":
            print("   Word Piece Segmentation...")
            tokenizer = ModifiedWordPieceTokenizer(bert_vocab_file=bert_vocab_file,
                                                   do_lower_case=do_lower_case)
            text_array = pool.map(tokenizer.tokenize, text_array)
        elif word_tokenization == "space":
            return text_array

    return text_array

def tokens_to_index(token_list, vocab_dict):
    """
    convert word tokens to number index

    INPUT:
        token_list: a list of string tokens
        vocab_dict: a dictionary with tokens(key)-integer(value) pairs
    OUTPUT:
        index_list: a list of integer indices
    """
    index_list = []
    for x in token_list:
        try:
            index_list.append(vocab_dict[x])
        except:
            index_list.append(1)  # index for oov tokens
    return index_list

def padding(index_list, max_len):
    """
    pad 0 to the index_list

    INPUT:
        index_list: an integer list with length <= max_len
        max_len: integer --maximum number of entries in the list
    OUTPUT:
        an integer list with length == max_len
    """
    return [0] * (max_len - len(index_list)) + index_list

def truncate_text(token_list, max_len):
    """
    cutting tokens that exceeds the max_len for each input sentence

    INPUT:
        token_list: a list of string with variable length
        max_len: maximum number of tokens in the text
    OUTPUT:
        a list of string with length <= max_len
    """
    return token_list[:max_len]

def get_one_hot_encoding(label_lists, labels):
    """
    a function to return class labels in one hot encoding format
    INPUT:
        label_lists: list of integer labels for the samples
        labels: a list of all possible classification classes
    OUTPUT:
        y: 2-dimensional list, specifically a one-hot matrix -- shape of [N, len(labels)]
    """
    le = LabelEncoder()
    le = le.fit(labels)
    y = le.transform(label_lists)
    y = to_categorical(y, num_classes=len(labels))
    return y

def split_text(text):
    """a function to split sentence into a list of tokens based on space characters."""
    return text.split()

def is_valid_label(label, all_labels):
    """a function to return whether the input "label" exist in the all_labels, which is a list of string"""
    return label in all_labels

def label_validation(df, col_label, all_labels, num_threads=1):
    """
    a function to separate a data frame into two, one contains records with valid labels, one without

    INPUT:
        df: a pandas dataframe which the column "TEXT" and "LABEL" must exist
        col_label: name of label column in the dataframe
        labels: a list of all available labels of the classification task
        num_threads: number of CPU processors for performing the subtasks
    OUTPUT:
        df_valid: a pandas dataframe which all records have valid labels
        df_invalid: a pandas dataframe which all records have invalid labels
    """
    with mp.Pool(processes=num_threads) as pool:
        bool_valid_label = pool.map(partial(is_valid_label, all_labels=all_labels), df[col_label])
        bool_invalid_label = pool.map(operator.not_, bool_valid_label)

        df_valid = df.loc[bool_valid_label, :]
        df_invalid = df.loc[bool_invalid_label, :]

    return df_valid, df_invalid

def is_uninformative(index_list, max_len):
    """
    a boolean function to determine whether an input text is informative or not.
    Since the index 0 is used for padding and the index 1 is used for out-of-vocabulary tokens, when all token indices
    for an input text is either 0 or 1 only (equivalently small than or equal to 1), the text is
    unlikely to be reasonably predicted by the model.

    INPUT:
        index_list: a list of indices
        max_len:  maximum number of tokens in each text
    OUTPUT:
        True or False

    """
    return bool(sum([e <= 1 for e in index_list]) == max_len)

def get_wxy(des_lists, vocab_dict, max_len, labels, num_threads=1, label_lists=None, label_to_weight_map=None):
    """
    a function to get the x, y of input data frame, weight and boolean lists

    INPUT:
        des_lists: a list/numpy array/pandas Series of N texts
        vocab_dict: a dictionary with tokens(key)-integer(value) pairs
        max_len: maximum number of tokens in each text
        labels: a list of all possible classification classes
        num_threads: int --number of CPU processors for performing the subtasks
        label_lists: a list/numpy array/pandas Series of labels each input text belongs to
        label_weight_map: a map with label as the key and the corresponding weight as value

    OUTPUT:
        x_out_valid: 2-dimensional numpy array -- shape of [N-U, max_len], where U is the number of uninformative
                                               -- samples. The uninformative samples are excluded for training.
        y_out_valid: 2-dimensional numpy array, specifically a one-hot matrix -- shape of [N-U, len(labels)]
        weight_valid: an numpy array of weight for informative samples only, length of array is N-U
        bool_informative: a boolean list of length N, True means informative and vice versa
        bool_uninformative: a boolean list of length N, True means uninformative and vice versa
    """

    with mp.Pool(processes=num_threads) as pool:

        token_lists = pool.map(split_text, des_lists)
        truncated_token_lists = pool.map(partial(truncate_text, max_len=max_len), token_lists)
        truncated_index_lists = pool.map(partial(tokens_to_index, vocab_dict=vocab_dict), truncated_token_lists)
        x = pool.map(partial(padding, max_len=max_len), truncated_index_lists)
        x_out = np.array(x)

        bool_uninformative = pool.map(partial(is_uninformative, max_len=max_len), x_out)
        bool_informative = pool.map(operator.not_, bool_uninformative)

        x_out_valid = x_out[bool_informative]

        if label_lists is not None:
            y = get_one_hot_encoding(label_lists, labels)
            y_out = np.array(y)
            y_out_valid = y_out[bool_informative]

        else:
            y_out_valid = None

        if label_to_weight_map is not None:
            informative_labels = np.array(label_lists)[bool_informative]
            weight_valid = np.array(list(map(lambda x: label_to_weight_map[x], informative_labels)))
        else:
            weight_valid = None

    return x_out_valid, y_out_valid, weight_valid, bool_informative, bool_uninformative

def focal_loss(gamma=2., alpha=0.25):
    gamma = float(gamma)
    alpha = float(alpha)

    def focal_loss_fixed(y_true, y_pred):
        """Focal loss for multi-classification
        FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)
        Notice: y_pred is probability after softmax
        gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper
        d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)
        Focal Loss for Dense Object Detection
        https://arxiv.org/abs/1708.02002

        Arguments:
            y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls]
            y_pred {tensor} -- model's output, shape of [batch_size, num_cls]

        INPUT:
            gamma {float} -- (default: {2.0})
            alpha {float} -- (default: {0.25})

        OUTPUT:
            [tensor] -- focal loss value.
        """
        epsilon = 1.e-9
        y_true = tf.convert_to_tensor(y_true, tf.float32)
        y_pred = tf.convert_to_tensor(y_pred, tf.float32)

        model_out = tf.add(y_pred, epsilon)
        ce = tf.multiply(y_true, -tf.log(model_out))
        weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma))
        fl = tf.multiply(alpha, tf.multiply(weight, ce))
        reduced_fl = tf.reduce_max(fl, axis=1)
        return tf.reduce_mean(reduced_fl)

    return focal_loss_fixed

def load_data_file(file_path, FLAGS):
    """
    a function load the feather file and perform basic validation checking

    INPUT:
        file_path: string --path to the data file
        FLAGS: model flags
    OUTPUT:
        df: a pandas data frame which the columns "TEXT" and "LABEL" must exist.
    """

    # import the data file as pandas dataframe
    if bool(re.search(".feather", file_path.lower())):
        df = feather.read_dataframe(file_path)
    elif bool(re.search(".csv", file_path.lower())):
        df = pd.read_csv(file_path, dtype={'TEXT': str, 'LABEL': str})
    elif bool(re.search(".xlsx", file_path.lower())):
        df = pd.read_excel(file_path, dtype={'TEXT': str, 'LABEL': str})
    elif bool(re.search(".dbf", file_path.lower())):
        text = DBF(file_path, char_decode_errors="ignore")  # set "ignore_missing_memofile=True" in absence of memo file
        df = pd.DataFrame(iter(text))
    else:
        raise ValueError("Fail to open the file {}, the acceptable data file format \
        include .feather, .csv, .txt, .xlsx and .dbf only".format(file_path))

    # basic validation checking
    if "TEXT" not in df.columns:
        raise ValueError("The column `TEXT` is missing from the input data file: {}.".format(file_path))
    # only in "do_train" or "do_eval" mode the column "LABEL" is required
    if FLAGS.do_train or FLAGS.do_eval:
        if "LABEL" not in df.columns:
            raise ValueError("The column `LABEL` is missing from the input data file: {}.".format(file_path))

    return df

def get_cr_distr(prediction, col_label, col_pred, col_probit):
    """
    get the distribution of classification rate

    INPUT:
        prediction: a pandas dataframe with at least three columns, one for the groud truth label of the prediction,
                    another for the predicted label as well as one for the predicted probability of the predicted label
        col_label: column name of the ground truth label
        col_pred:  column name of the predicted label
        col_predict: column name of the predicted probability of the predicted label
    OUTPUT:
        distr_table: a pandas dataframe that shows the distribution of the classification rate and the accuracy
    """

    bins = np.arange(0, 1.1, 0.1)
    prediction["Probability"] = pd.cut(prediction[col_probit], bins)
    prediction["True/False"] = prediction[col_label] == prediction[col_pred]
    prediction["Proportion (%)"] = 1

    distr_table = prediction.loc[:, ["Probability", 'True/False', 'Proportion (%)']].groupby(["Probability", 'True/False'])
    distr_table = (distr_table.count()/prediction.shape[0])*100
    distr_table = distr_table.unstack()
    distr_table["Proportion (%)", "Total"] = distr_table["Proportion (%)"][True] + distr_table["Proportion (%)"][False]
    distr_table["Classification rate (%)"] = 100 * (distr_table["Proportion (%)"][True] / \
                                             (distr_table["Proportion (%)"][True] + distr_table["Proportion (%)"][False]))
    return distr_table


from ctfdist.paths import *
from ctfdist.descent import *
from ctfdist.helper_functions import *
from sklearn.linear_model import LogisticRegression
from scripts.toy_helper import *
from scripts.plot_helper import *

# setup
random_seed = 2338
random_state = np.random.RandomState(random_seed)
outcome_column_name = 'y'
group_column_name = 's'
minority_group = 0
majority_group = 1

train_size = 0.30
test_size = 0.20
audit_size = 1.0 - train_size - test_size

# generate full dataset
ps = 0.5
coefs = np.array([5.0, -2.5, -2.5])

param_min = {
    'coefs': coefs,
    'group_label': minority_group,
    'px': [0.9, 0.2, 0.2],
    'n_samples': 50000,
    }

param_maj = {
    'coefs': coefs,
    'group_label': majority_group,
    'px': [0.1, 0.5, 0.5],
    'n_samples': 50000,
    }

df_min, h_min = generate_toy_dataset(**param_min)
df_maj, h_maj = generate_toy_dataset(**param_maj)
df = df_min.append(df_maj)

#### Data Splitting ####

df_train, df_audit, df_test = train_test_audit_split(df = df,
                                                     train_size = train_size,
                                                     test_size = test_size,
                                                     audit_size = audit_size,
                                                     group_column_name = group_column_name,
                                                     outcome_column_name = outcome_column_name,
                                                     random_state = random_state)

#### Models ####

#handle for p(S = 1 | X = x) -- model to get true outcomes for minority group
hs_min = h_min['get_px']
hs_maj = h_maj['get_px']
predict_group = lambda x: np.multiply(ps, hs_maj(x)) / (np.multiply(ps, hs_maj(x)) + np.multiply(1.0 - ps, hs_min(x)))

# handle for p(Y = 1 | X = x) -- model to get true outcomes for minority group:
get_py_min = h_min['get_py']
get_py_maj = h_maj['get_py']
predict_ytrue = lambda x: get_py_min(x).flatten()
predict_ytrue_maj = lambda x: get_py_maj(x).flatten()

# train predictive model
X, Y, S = df_to_XYS(df_train, minority_group = minority_group)
f_hat = LogisticRegression(penalty = 'l2',  C = 100, fit_intercept = False)
f_hat.fit(X = X, y = Y)
pos_class_idx = np.flatnonzero(f_hat.classes_ > 0)
predict_prob = lambda x: f_hat.predict_proba(x)[:, pos_class_idx].flatten()
predict_yhat = lambda x: 2.0 * np.greater(predict_prob(x), 0.5) - 1.0

#### Distribution Descent Experiment ####

script_metric_weights = {
    'cal': {'cal': 1.0},
    'fpr': {'fpr': 1.0},
    'fnr': {'fnr': 1.0},
    'fdr': {'fdr': 1.0},
    'da_out': {'da_out': 1.0},
    'da_in': {'da_in': 1.0},
    'da_mix': {'da_out': 0.8, 'da_in': 0.2},
    }

data_name = 'toy'
metric_name = 'fpr'
metric_weights = script_metric_weights[metric_name]
figure_name = '%s/%s_descent_%s.pdf' % (results_dir, data_name, metric_name)

### SANITY CHECK PERFORMANCE
np.mean(S == np.greater(predict_group(X), 0.5))

train_stats_df = get_performance_metrics(df = df_train,
                                         data_type = 'train',
                                         predict_yhat = predict_yhat,
                                         predict_prob = predict_prob)

audit_stats_df = get_performance_metrics(df = df_audit,
                                         data_type = 'audit',
                                         predict_yhat = predict_yhat,
                                         predict_prob = predict_prob)

test_stats_df = get_performance_metrics(df = df_test,
                                        data_type = 'test',
                                        predict_yhat = predict_yhat,
                                        predict_prob = predict_prob)

train_stats_df[train_stats_df['s'] >= 0][['s', metric_name]]

# Fit Counterfactual Distribution
ctf_args = {
    'df': df_audit,
    'metric_weights': metric_weights,
    #
    'predict_yhat': predict_yhat,
    'predict_group':predict_group,
    'predict_ytrue': predict_ytrue,
    'predict_ytrue_maj': predict_ytrue_maj,
    #
    'outcome_column_name': outcome_column_name,
    'group_column_name':group_column_name,
    'minority_group':minority_group,
    #
    'random_state':random_state,
    }

ctf_dist = CounterfactualDistribution(**ctf_args)
df_audit_min, df_audit_maj = split_by_group(df_audit)
results = ctf_dist.fit(df_min_test = df_audit_min, df_maj_test = df_audit_maj, max_iterations = 1e2, step_size = 0.1)
f, _ = plot_descent_profile(results)
f.savefig(figure_name)

# Build Preprocessor
pp = ctf_dist.build_preprocessor(df_audit)
stats_df = get_performance_metrics(df = df_test,
                                   data_type = 'test',
                                   pp = pp,
                                   predict_yhat = predict_yhat,
                                   predict_prob = predict_prob,
                                   minority_group = minority_group)

stats_df[stats_df['s'] >= 0][['s', metric_name, 'dist_type']]


import torch

from xumi.ml.models import Seq2Seq

def test_models():
    n_classes = 100

    source = torch.randint(low=0, high=n_classes, size=(20, 16))
    target = torch.randint(low=0, high=n_classes, size=(20, 32))

    s2s = Seq2Seq(out_vocab_size=n_classes, pad_idx=0)

    out = s2s((source, target))

    assert list(out.size()) == [20, 32, 100]


import sys
import requests
from bs4 import BeautifulSoup
from konlpy.tag import Okt
from collections import Counter, OrderedDict
import matplotlib
import matplotlib.pyplot as plt

BASE_URL = "https://www.joongang.co.kr/search?keyword="
SEARCH_URL = "&page="
get_words1 = ''
font_name = 'Malgun Gothic'

def get_words(key_word, page_range):
    title = []
    result_words = ''

    for page in range(page_range):
        current_page = page
        crawling_url_list = BASE_URL + key_word + SEARCH_URL + str(page_range)

        response = requests.get(crawling_url_list)
        soup = BeautifulSoup(response.text, 'lxml')
        total_titles = soup.select('h2.headline > a')
        total_contents = soup.select('div.card_body > p')

        for total_title in total_titles:
            title.append(total_title.string.replace('\n', ''))
        for total_content in total_contents:
            title.append(total_content.string.replace('\n', '').replace('\xa0', ''))
        result_words = ' '.join(s for s in title)

    return result_words

def wordcount(words):
    print('data load...')

    result = {}
    engine = Okt()
    data = words
    all_nouns = engine.nouns(data)
    nouns = [n for n in all_nouns if len(n) > 1]

    global count, by_num

    count = Counter(nouns)
    by_num = OrderedDict(sorted(count.items(), key=lambda t: t[1], reverse=True))
    word = [i for i in by_num.keys()]
    number = [i for i in by_num.values()]

    for w, n in zip(word, number):
        result[w] = n
    print('words count finish.')
    return result

def top_n(by_num):
    print("searching top20..")

    rank = count.most_common(20)
    global top
    top = dict(rank)

def topn_vis_bar(top):
    print("creating graph")
    fig = plt.gcf()
    fig.set_size_inches(15, 10)
    matplotlib.rc('font', family=font_name, size=20)
    plt.title('기사에 많이 나온 단어 Top 20', fontsize=35)
    plt.xlabel('기사에 나온 단어', fontsize=30)
    plt.ylabel('기사에 나온 단어의 개수', fontsize=30)
    plt.bar(top.keys(), top.values(), color='#FFA7A7')
    plt.savefig('top_words.jpg')
    plt.show()
    print('- top_words.jpg가 저장되었습니다 \n')

def main(argv):  # argument values
    # 명령 라인에 주어진 모든 값을 받아서 sys.argv 리스트에 넣음
    if len(argv) != 3:  # 입력한 데이터가 3개가 아닐 때
        print('사용법을 모르시는군요')
        print('사용법 : python [모듈이름][키워드][가져올 페이지 숫자]')
        return

    key_word = argv[1]
    page_range = int(argv[2])
    # argv[0] = 모듈의 파일명

    words = get_words(key_word, page_range)
    words_count = wordcount(words)
    top_n(words_count)
    topn_vis_bar(top)

if __name__ == '__main__':
    main(sys.argv)


from _config import MAX_LENGTH
from torch.utils.data import Dataset
from collections import namedtuple, Counter
import random
import json
import os

Pair = namedtuple('Pair', ('utterance', 'response', 'rating', 'conversation_id', 'opening_line'))
numeric_ratings = {'Poor':0, 'Not Good':1, 'Passable':2, 'Good':3, 'Excellent':4}

def standardise_sentence(sentence):
    sentence = sentence.replace(',',' , ').replace('.',' . ').replace('?',' ? ').replace('!',' ! ').replace('(',' ( ').replace(')',' ) ').replace('%',' % ')
    while '  ' in sentence:
        sentence = sentence.replace('  ',' ')
    return sentence.lower()

def load_alexa_pairs(fname='train.json', dir='./data/amazon'):
    pairs = []
    data = json.loads(open(os.path.join(dir, fname), 'r').read())
    for c_id, conversation in data.items():
        first = True
        for utterance, response in zip(conversation['content'][:-1],conversation['content'][1:]):
            u_sentence, r_sentence = standardise_sentence(utterance['message']), standardise_sentence(response['message'])
            if response['turn_rating'] not in [''] and len(u_sentence.split(' ')) < MAX_LENGTH and len(r_sentence.split(' ')) < MAX_LENGTH:
                pairs.append(Pair(utterance=u_sentence,
                                  response=r_sentence,
                                  rating=numeric_ratings[response['turn_rating']],
                                  conversation_id=c_id,
                                  opening_line=first))
                first = False
    return pairs

class AlexaDataset(Dataset):
    """Amazon Alexa Conversations dataset."""

    def __init__(self, json=None, dir='./data/amazon', rare_word_threshold=0):
        """
        Args:
            json (string): Name of json file to load
            dir (string): Directory where json is stored
            rare_word_threshold: Remove pairs which contain words that appear less than or equal to threshold
        """
        self.data = []
        self.rare_word_threshold = rare_word_threshold
        if json is not None:
            self.add_pairs_from_json(json, dir)
        else:
            for f in [f for f in os.listdir(dir) if f.endswith('.json')]:
                print("Added {} to dataset".format(f))
                self.add_pairs_from_json(f, dir)

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        return self.data[index]

    @property
    def conversation_ids(self):
        return set([p.conversation_id for p in self.data])

    @property
    def opening_lines(self):
        return [self.get_conversation(c)[0].utterance for c in self.conversation_ids]

    def balance_data(self):
        min_obs = 10000000
        for r in numeric_ratings.values():
            obs = len([p for p in self.data if p.rating==r])
            min_obs = obs if obs < min_obs else min_obs
        new_data = []
        for r in numeric_ratings.values():
            r_data = [p for p in self.data if p.rating==r]
            if len(r_data) >= min_obs:
                r_data = r_data[:min_obs]
                new_data += r_data
        self.data = new_data
        self.ids = list(set([p.conversation_id for p in self.data]))

    def add_pairs_from_json(self, json, dir='./data/amazon'):
        self.data += load_alexa_pairs(json, dir)
        self._trim_rare_words(self.rare_word_threshold)
        self.ids = list(set([p.conversation_id for p in self.data]))

    def random_opening_line(self):
        return random.choice([p.utterance for p in self.data if p.opening_line])

    def get_conversation(self, id):
        return [p for p in self.data if p.conversation_id == id]

    def random_conversation(self):
        id = 0
        while id == 0:
            id = random.choice(self.ids)
        return [p for p in self.data if p.conversation_id == id]

    def _rare_words(self, threshold=3):
        count = Counter()
        for pair in self.data:
            for word in set(pair.utterance.split(' ')):
                count[word] += 1
        rare_words = set([word for word, cnt in count.items() if cnt <= threshold])
        return rare_words

    def _trim_rare_words(self, threshold=3):
        rare_words = self._rare_words(threshold)
        new_data = []
        for pair in self.data:
            words = pair.utterance.split(' ') + pair.response.split(' ')
            if set(words).isdisjoint(rare_words):
                new_data.append(pair)
        print('{} pairs trimmed, {} remain'.format(len(self.data) - len(new_data), len(new_data)))
        self.data = new_data

    def trimPairsToVocab(self, voc):
        keepPairs = []
        for pair in self.data:
            keep = True
            for word in pair.utterance.split(' ') + pair.response.split(' '):
                if word not in voc.word2index:
                    keep = False
            if keep:
                keepPairs.append(pair)
        print('=============={}=============='.format(len(self.data)-len(keepPairs)))
        self.data = keepPairs

    def add_scrambled_training_data(self, addition_rate=0.2, rating='Poor'):
        pairs = []
        for i in range(int(len(self.data) * addition_rate)):
            utterance = random.choice(self.data).utterance
            response = random.choice(self.data).response
            pairs.append(Pair(utterance=utterance,
                              response=response,
                              rating=numeric_ratings[rating],
                              conversation_id='0',
                              opening_line=False))
        self.data += pairs

if __name__ == '__main__':
    data = AlexaDataset(rare_word_threshold=2)
    print(len(data))
    print(data[0])
    c = data.get_conversation('t_bde29ce2-4153-4056-9eb7-f4ad710505fe')
    s = data.random_conversation()
    data.add_scrambled_training_data()


import logging
from datetime import datetime, date, time, timedelta

from flask import current_app

from app.data_analysis.helpers import get_daily_values_dict
from app.default.helpers import get_jobs
from app.default.models import Machine

def get_machine_quality(machine: Machine, time_start: datetime, time_end: datetime) -> float:
    """ Calculate the quality of machine output for calculating OEE"""
    jobs = get_jobs(time_start, time_end, machine=machine)
    total_machine_quantity_produced = 0
    total_machine_rejects_produced = 0
    for job in jobs:
        # I've already considered what happens if this job ran outside the requested start/end, and it would be ok,
        # All we can do is assume the job has same performance all the way through, so the ratio would still be the same
        rejects_qty = job.get_total_reject_quantity()
        good_qty = job.get_total_good_quantity()
        total_machine_quantity_produced += (good_qty + rejects_qty)
        total_machine_rejects_produced += rejects_qty
    if total_machine_quantity_produced == 0:
        return 0
    quality = (total_machine_quantity_produced - total_machine_rejects_produced) / total_machine_quantity_produced
    if quality > 1:
        logging.warning(f"Quality of >1 calculated for machine {machine.name} on {time_start.date()}")
    return quality

def get_daily_quality_dict(requested_date: date = None, human_readable=False):
    """ Return a dictionary with every machine's performance on the given date """
    quality_dict = get_daily_values_dict(get_machine_quality, requested_date)
    if human_readable:
        for k, v in quality_dict.items():
            v = v * 100
            quality_dict[k] = f"{round(v, 1)}%"
    return quality_dict


from collections import defaultdict
import time

def split_pair(pair):
    pair = pair.split(' -> ')
    return pair

def make_puzzle_inputs(file_path):
    with open(file_path) as file:
        input_text = file.read().splitlines()
        polymer = input_text[0]
        pair_list = input_text[2:]
        pair_list = list(map(split_pair, pair_list))
        return polymer, pair_list

def grow_polymer(polymer, pair_list, polymer_lookup_dict):
    new_polymer = ''
    updated_polymer_pair = ''
    for i, char in enumerate(polymer):
        try:
            polymer_pair = char + polymer[i + 1]
        except IndexError:
            new_polymer += char
            break
        if polymer_pair in polymer_lookup_dict.keys():
            updated_polymer_pair = polymer_lookup_dict[polymer_pair]
        else:
            for pair in pair_list:
                if pair[0] == polymer_pair:
                    updated_polymer_pair = polymer_pair[0] + pair[1]
                    polymer_lookup_dict[polymer_pair] = updated_polymer_pair
                    break
        new_polymer += updated_polymer_pair
    return new_polymer, polymer_lookup_dict

def grow_polymer_n_times(polymer, pair_list, polymer_lookup_dict, n):
    for i in range(n):
        print('NNC' in polymer)
        start_time = time.time()
        polymer, polymer_lookup_dict = grow_polymer(polymer, pair_list, polymer_lookup_dict)
        elapsed_time = time.time() - start_time
        print(f'grew polymer {i + 1} times in {round(elapsed_time, 4)} seconds')
        # occurance_dict = create_occurance_dict(polymer)
        # max_key, min_key = find_max_and_min_key(occurance_dict)
        # print(occurance_dict)
        # answer = calculate_answer(occurance_dict, min_key, max_key)
        # print(f'polymer is {len(polymer)} characters')
        # print(f'answer is {answer}')
    return polymer

def default_to_0():
    return 0

def create_occurance_dict(polymer):
    occurance_dict = defaultdict(default_to_0)
    for char in polymer:
        occurance_dict[char] = occurance_dict[char] + 1
    return dict(occurance_dict)

def find_max_and_min_key(occurance_dict):
    max_key = max(occurance_dict, key=occurance_dict.get)
    min_key = min(occurance_dict, key=occurance_dict.get)
    return max_key, min_key

def calculate_answer(occurance_dict, min_key, max_key):
    max = occurance_dict[max_key]
    min = occurance_dict[min_key]
    return max - min

polymer, pair_list = make_puzzle_inputs('example_1.txt')
polymer_lookup_dict = {}
polymer = grow_polymer_n_times(polymer, pair_list, polymer_lookup_dict, 17)
occurance_dict = create_occurance_dict(polymer)
print(occurance_dict)
max_key, min_key = find_max_and_min_key(occurance_dict)
answer = calculate_answer(occurance_dict, min_key, max_key)
print(f'answer = {answer}')


import os
import json
import requests
from src.config import get_config
from src.mssql_db import execute_sp, get_sp_first_result_set
from src.utils import format_date

config = get_config()

def _get_custom_attributes(custom_attributes):
    custom_attrs = json.dumps(custom_attributes)
    if custom_attrs == '{}':
        custom_attrs = None

    return custom_attrs

def fetch_api_rep_data():
    """ Return the Telecom API rep json data. """
    if config['IS_PRODUCTION']:
        return fetch_api_prod_rep_data()

    return fetch_api_test_rep_data()

def fetch_api_test_rep_data():
    """ Returns JSON payload containing the test data. """
    file_path = os.path.join(config['ROOT_DIR'], 'test_data.json')
    if os.path.exists(file_path) is False:
        raise FileExistsError(f"The test data file was not found.")

    with open(file_path) as fp:
        contents = fp.read()

    return json.loads(contents)

def fetch_api_prod_rep_data():
    """ Returns JSON payload containing the production data. """
    result = requests.get(config['TELECOM_REP_ENDPOINT'])
    if result.status_code != 200:
        result.raise_for_status()

    return result.json()

def start_process():
    """ Calls the stored procedure to start the process. """
    results = execute_sp(
        'UMA_TELECOM.PROCESS_API_DATA',
        {
            'MESSAGE': 'START'
        }
    )

    result = get_sp_first_result_set(results)
    if not result:
        return False

    return result['return_value']

def stop_process(master_load_id):
    """ Calls the stored procedure to stop the process. """
    results = execute_sp(
        'UMA_TELECOM.PROCESS_API_DATA',
        {
            'MESSAGE': 'FINISHED',
            'LOAD_HIST_PKID_IN': master_load_id
        }
    )

    result = get_sp_first_result_set(results)
    if not result:
        return False

    return result

def save_rep(rep, master_load_id):
    """ Saves the rep data. """
    results = execute_sp(
        'UMA_TELECOM.SAVE_D_REP',
        {
            'REP_ID': rep['$id'],
            'REP_userId': rep['userId'],
            'REP_homeSite': rep['homeSite'],
            'REP_firstName': rep['firstName'],
            'REP_lastName': rep['lastName'],
            'REP_displayName': rep['displayName'],
            'REP_ntDomainUser': rep['ntDomainUser'],
            'REP_extension': rep['extension'],
            'REP_outboundANI': rep['outboundANI'],
            'REP_id_LIST': rep['id'],
            'REP_customAttributes': _get_custom_attributes(rep['customAttributes']),
            'REP_dateAdded': format_date(parse_date(rep['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        },
        out_arg='rep_id'
    )

    result = get_sp_first_result_set(results)
    if not result:
        return False

    return result['rep_id']

def save_rep_skill(rep_id, rep_skill, master_load_id):
    """ Saves the rep skill. """
    execute_sp(
        'UMA_TELECOM.SAVE_D_REP_SKILL',
        {
            'D_REP_ID': rep_id,
            'REP_SKILL_ID': rep_skill['$id'],
            'REP_SKILL_displayName': rep_skill['displayName'],
            'REP_SKILL_proficiency': rep_skill['proficiency'],
            'REP_SKILL_desireToUse': rep_skill['desireToUse'],
            'REP_SKILL_id_ALTERNATE': rep_skill['id'],
            'REP_SKILL_dateAdded': format_date(parse_date(rep_skill['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        }
    )

def save_rep_role(rep_id, rep_role, master_load_id):
    """ Saves the rep role. """
    execute_sp(
        'UMA_TELECOM.SAVE_D_REP_ROLE',
        {
            'D_REP_ID': rep_id,
            'REP_ROLE_ID': rep_role['$id'],
            'REP_ROLE_roleId': rep_role['roleId'],
            'REP_ROLE_name': rep_role['name'],
            'REP_ROLE_id_ALTERNATE': rep_role['id'],
            'REP_ROLE_dateAdded': format_date(parse_date(rep_role['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        }
    )

def save_rep_workgroup(rep_id, rep_workgroup, master_load_id):
    """ Saves the rep workgroup. """
    execute_sp(
        'UMA_TELECOM.SAVE_D_REP_WORKGROUP',
        {
            'D_REP_ID': rep_id,
            'REP_WORKGROUP_ID': rep_workgroup['$id'],
            'REP_WORKGROUP_Name': rep_workgroup['name'],
            'REP_WORKGROUP_customAttributes': _get_custom_attributes(rep_workgroup['customAttributes']),
            'REP_WORKGROUP_id_ALTERNATE': rep_workgroup['id'],
            'REP_WORKGROUP_dateAdded': format_date(parse_date(rep_workgroup['dateAdded'])),
            'LOAD_HISTORY_PKID': master_load_id
        }
    )

def parse_date(value):
    if value:
        return value.split('.', 1)[0]

    return ''


import pyowm

owm = pyowm.OWM('67f0d157b2ae7466f46377e76d4c5670')  # You MUST provide a valid API key

# Have a pro subscription? Then use:
# owm = pyowm.OWM(API_key='your-API-key', subscription_type='pro')

# Search for current weather in London (Great Britain)
# observation = owm.weather_at_place('London,GB')
observation = owm.weather_at_place('Kyiv')
w = observation.get_weather()
print(w)                      # <Weather - reference time=2013-12-18 09:20,
                              # status=Clouds>


import atexit
import pickle
import socket
import threading
import time

import cv2
import numpy as np

class Server:
    def __init__(self):
        PORT = 18082
        IPADDRESS = '192.168.0.105'
        SERVER_ADDRESS = (IPADDRESS, PORT)  # print(socket.gethostbyname(socket.gethostname()))

        self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server.bind(SERVER_ADDRESS)
        self.recieve_length = 1024
        atexit.register(self.close)
        self.img_s = None
        self.rate = [0 for i in range(100)]

    def main(self):
        self.server.listen()
        while True:
            self.conn, addr = self.server.accept()
            print(f"[ ACK ]{addr} connect\tcurrent {threading.active_count()}")
            t = threading.Thread(target=self.handle_client)
            t.start()

    def handle_client(self):
        while True:
            msg = self.conn.recvfrom(100000000)
            if len(msg) is not None:
                print(f"\r Last recieved: {sum(self.rate)} out of {len(self.rate)}", end="\t")
                # img_s = cv2.imdecode(np.asarray(bytearray(msg), dtype='uint8'), cv2.IMREAD_COLOR)
                try:
                    img_s = pickle.loads(msg[0])
                    if img_s is None:
                        pass
                    else:
                        self.img_s = cv2.resize(img_s, (640, 480))
                        self.rate.pop(0)
                        self.rate.append(1)
                except pickle.UnpicklingError:
                    # print("数据包过大")
                    self.rate.pop(0)
                    self.rate.append(0)
                except ValueError:
                    print("unregistered extension code 121")
                    self.rate.pop(0)
                    self.rate.append(0)
                except EOFError:
                    print("Ran out of input")
                    self.rate.pop(0)
                    self.rate.append(0)
                    break
                except Exception as e:
                    print(e, "Bypass")
                    self.rate.pop(0)
                    self.rate.append(0)
                # print(type(data))
                # img_s = cv2.imdecode(msg, cv2.IMREAD_COLOR)
        self.close()

    def close(self):
        self.conn.close()
        self.server.close()
        cv2.destroyAllWindows()
        print("Closed")

if __name__ == '__main__':
    s = Server()
    threading.Thread(target=s.main).start()
    while sum(s.rate) < 1:
        time.sleep(0.1)
    print("Create window")
    while sum(s.rate) > 0:
        pic = s.img_s
        if np.max(pic) < 50:
            pic_max = np.max(pic)
            pic_min = np.min(pic)
            pic = pic - pic_min
            pic = pic * (1 / (pic_max - pic_min))
            # pic = np.array(pic).astype("float32")
            # pid = cv2.bilateralFilter(pic, 0, 100, 5)
        cv2.imshow("remote_pic", pic)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    s.close()
    exit(0)


import copy
from unittest.mock import patch

import numpy as np
import torch
from ax.core.data import Data
from ax.core.objective import MultiObjective, Objective
from ax.core.observation import Observation, ObservationData, ObservationFeatures
from ax.core.optimization_config import MultiObjectiveOptimizationConfig
from ax.core.outcome_constraint import ObjectiveThreshold
from ax.core.search_space import SearchSpace
from ax.core.types import ComparisonOp
from ax.exceptions.core import UnsupportedError
from ax.metrics.branin import BraninMetric, NegativeBraninMetric
from ax.modelbridge.registry import Models
from ax.plot.pareto_frontier import interact_multiple_pareto_frontier
from ax.plot.pareto_utils import (
    _extract_observed_pareto_2d,
    compute_posterior_pareto_frontier,
    get_observed_pareto_frontiers,
    infer_reference_point_from_experiment,
    to_nonrobust_search_space,
)

from ax.utils.common.testutils import TestCase
from ax.utils.testing.core_stubs import (
    get_branin_experiment,
    get_branin_experiment_with_multi_objective,
    get_experiment_with_observations,
    get_robust_search_space_environmental,
    get_search_space,
)

class ParetoUtilsTest(TestCase):
    def setUp(self) -> None:
        experiment = get_branin_experiment()
        experiment.add_tracking_metric(
            BraninMetric(name="m2", param_names=["x1", "x2"])
        )
        sobol = Models.SOBOL(experiment.search_space)
        a = sobol.gen(5)
        experiment.new_batch_trial(generator_run=a).run()
        self.experiment = experiment
        self.metrics = list(experiment.metrics.values())

    def testComputePosteriorParetoFrontierByTrial(self) -> None:
        # Experiments with batch trials must specify trial_index or data
        with self.assertRaises(UnsupportedError):
            compute_posterior_pareto_frontier(
                self.experiment,
                self.metrics[0],
                self.metrics[1],
                absolute_metrics=[m.name for m in self.metrics],
            )
        pfr = compute_posterior_pareto_frontier(
            self.experiment,
            self.metrics[0],
            self.metrics[1],
            trial_index=0,
            absolute_metrics=[m.name for m in self.metrics],
            num_points=2,
        )
        self.assertIsNone(pfr.arm_names)

    def testComputePosteriorParetoFrontierByData(self) -> None:
        # Experiments with batch trials must specify trial_index or data
        compute_posterior_pareto_frontier(
            self.experiment,
            self.metrics[0],
            self.metrics[1],
            data=self.experiment.fetch_data(),
            absolute_metrics=[m.name for m in self.metrics],
            num_points=2,
        )

    def testObservedParetoFrontiers(self) -> None:
        experiment = get_branin_experiment_with_multi_objective(
            with_batch=True, has_optimization_config=False, with_status_quo=True
        )

        # Optimization config is not optional
        with self.assertRaises(ValueError):
            get_observed_pareto_frontiers(experiment=experiment, data=Data())

        objectives = [
            Objective(
                metric=BraninMetric(
                    name="m1", param_names=["x1", "x2"], lower_is_better=True
                ),
                minimize=True,
            ),
            Objective(
                metric=NegativeBraninMetric(
                    name="m2", param_names=["x1", "x2"], lower_is_better=True
                ),
                minimize=True,
            ),
            Objective(
                metric=BraninMetric(
                    name="m3", param_names=["x1", "x2"], lower_is_better=True
                ),
                minimize=True,
            ),
        ]
        bounds = [0, -100, 0]
        objective_thresholds = [
            ObjectiveThreshold(
                metric=objective.metric,
                bound=bounds[i],
                relative=True,
                op=ComparisonOp.LEQ,
            )
            for i, objective in enumerate(objectives)
        ]
        objective = MultiObjective(objectives=objectives)
        optimization_config = MultiObjectiveOptimizationConfig(
            objective=objective,
            objective_thresholds=objective_thresholds,
        )
        experiment.optimization_config = optimization_config
        experiment.trials[0].run()

        # For the check below, compute which arms are better than SQ
        df = experiment.fetch_data().df
        df["sem"] = np.nan
        data = Data(df)
        sq_val = df[(df["arm_name"] == "status_quo") & (df["metric_name"] == "m1")][
            "mean"
        ].values[0]
        pareto_arms = sorted(
            df[(df["mean"] <= sq_val) & (df["metric_name"] == "m1")]["arm_name"]
            .unique()
            .tolist()
        )

        pfrs = get_observed_pareto_frontiers(experiment=experiment, data=data)
        # We have all pairs of metrics
        self.assertEqual(len(pfrs), 3)
        true_pairs = [("m1", "m2"), ("m1", "m3"), ("m2", "m3")]
        for i, pfr in enumerate(pfrs):
            self.assertEqual(pfr.primary_metric, true_pairs[i][0])
            self.assertEqual(pfr.secondary_metric, true_pairs[i][1])
            self.assertEqual(pfr.absolute_metrics, [])
            self.assertEqual(list(pfr.means.keys()), ["m1", "m2", "m3"])
            self.assertEqual(len(pfr.means["m1"]), len(pareto_arms))
            self.assertTrue(np.isnan(pfr.sems["m1"]).all())
            # pyre-fixme[6]: For 1st param expected `Sized` but got
            #  `Optional[List[Optional[str]]]`.
            self.assertEqual(len(pfr.arm_names), len(pareto_arms))
            arm_idx = np.argsort(pfr.arm_names)
            for i, idx in enumerate(arm_idx):
                name = pareto_arms[i]
                # pyre-fixme[16]: Optional type has no attribute `__getitem__`.
                self.assertEqual(pfr.arm_names[idx], name)
                self.assertEqual(
                    pfr.param_dicts[idx], experiment.arms_by_name[name].parameters
                )

        pfrs = get_observed_pareto_frontiers(
            experiment=experiment, data=data, arm_names=["0_1"]
        )
        for pfr in pfrs:
            # pyre-fixme[58]: `in` is not supported for right operand type
            #  `Optional[typing.List[typing.Optional[str]]]`.
            self.assertTrue("status_quo" in pfr.arm_names)

    def testPlotMultipleParetoFrontiers(self) -> None:
        experiment = get_branin_experiment_with_multi_objective(
            has_objective_thresholds=True,
        )
        sobol = Models.SOBOL(experiment.search_space)
        a = sobol.gen(5)
        experiment.new_batch_trial(generator_run=a).run()
        pfrs = get_observed_pareto_frontiers(experiment=experiment)
        pfrs2 = copy.deepcopy(pfrs)
        pfr_lists = {"pfrs 1": pfrs, "pfrs 2": pfrs2}
        self.assertIsNotNone(interact_multiple_pareto_frontier(pfr_lists))

    def test_extract_observed_pareto_2d(self) -> None:
        Y = np.array([[1.0, 2.0], [2.1, 1.0], [1.0, 1.0], [2.0, 2.0], [3.0, 0.0]])
        reference_point = (1.5, 0.5)
        minimize = False
        pareto = _extract_observed_pareto_2d(
            Y=Y, reference_point=reference_point, minimize=minimize
        )

        # first and last points are beyond reference point, third point is dominated
        self.assertTrue(np.array_equal(pareto, np.array([[2.1, 1.0], [2.0, 2.0]])))

        # different `minimize` in each direction
        minimize = (True, False)
        pareto = _extract_observed_pareto_2d(
            Y=Y, reference_point=reference_point, minimize=minimize
        )
        self.assertTrue(np.array_equal(pareto, np.array([[1.0, 2.0]])))

        # no `reference_point` provided
        minimize = False
        pareto = _extract_observed_pareto_2d(
            Y=Y, reference_point=None, minimize=minimize
        )
        self.assertTrue(
            np.array_equal(pareto, np.array([[3.0, 0.0], [2.1, 1.0], [2.0, 2.0]]))
        )

    def test_to_nonrobust_search_space(self) -> None:
        # Return non-robust search space as is.
        search_space = get_search_space()
        self.assertIs(to_nonrobust_search_space(search_space), search_space)
        # Prune environmental variables and distributions from RSS.
        rss = get_robust_search_space_environmental()
        transformed_ss = to_nonrobust_search_space(rss)
        # Can't use isinstance here since RSS is also a SearchSpace.
        self.assertEqual(transformed_ss.__class__, SearchSpace)
        self.assertEqual(transformed_ss.parameters, rss._parameters)
        self.assertEqual(
            transformed_ss.parameter_constraints, rss.parameter_constraints
        )

class TestInfereReferencePointFromExperiment(TestCase):
    def test_infer_reference_point_from_experiment(self) -> None:
        for constrained in (True, False):
            observations = [[-1.0, 1.0], [-0.5, 2.0], [-2.0, 0.5], [-0.1, 0.1]]
            if constrained:
                observations = [
                    o + [c] for o, c in zip(observations, [1.0, 0.5, 1.0, 1.0])
                ]
            # Getting an experiment with 2 objectives by the above observations.
            experiment = get_experiment_with_observations(
                observations=observations,
                minimize=True,
                scalarized=False,
                constrained=constrained,
            )

            inferred_reference_point = infer_reference_point_from_experiment(experiment)

            # The nadir point for this experiment is [-0.5, 0.5]. The function actually
            # deducts 0.1*Y_range from each of the objectives. Since the range for each
            # of the objectives is +/-1.5, the inferred reference point would
            # be [-0.35, 0.35].
            self.assertEqual(inferred_reference_point[0].op, ComparisonOp.LEQ)
            self.assertEqual(inferred_reference_point[0].bound, -0.35)
            self.assertEqual(inferred_reference_point[0].metric.name, "m1")
            self.assertEqual(inferred_reference_point[1].op, ComparisonOp.GEQ)
            self.assertEqual(inferred_reference_point[1].bound, 0.35)
            self.assertEqual(inferred_reference_point[1].metric.name, "m2")

    def test_infer_reference_point_from_experiment_shuffled_metrics(self) -> None:
        # Generating an experiment with given data.
        observations = [
            [-1.0, 1.0, 0.1],
            [-0.5, 2.0, 0.2],
            [-2.0, 0.5, 0.3],
            [-0.1, 0.1, 0.4],
        ]
        experiment = get_experiment_with_observations(
            observations=observations,
            minimize=True,
            scalarized=False,
            constrained=True,
        )

        # Constructing fake outputs for `get_pareto_frontier_and_configs` so that
        # the order of metrics `m1`, `m2` and `m3` are reversed.
        frontier_observations_shuffled = [
            Observation(
                features=ObservationFeatures(parameters={"x": 0.0, "y": 0.0}),
                data=ObservationData(
                    metric_names=["m3", "m2", "m1"],
                    means=np.array([0.1, 1.0, -1.0]),
                    covariance=np.diag(np.full(3, float("nan"))),
                ),
            ),
            Observation(
                features=ObservationFeatures(parameters={"x": 0.1, "y": 0.1}),
                data=ObservationData(
                    metric_names=["m3", "m2", "m1"],
                    means=np.array([0.2, 2.0, -0.5]),
                    covariance=np.diag(np.full(3, float("nan"))),
                ),
            ),
            Observation(
                features=ObservationFeatures(parameters={"x": 0.2, "y": 0.2}),
                data=ObservationData(
                    metric_names=["m3", "m2", "m1"],
                    means=np.array([0.3, 0.5, -2.0]),
                    covariance=np.diag(np.full(3, float("nan"))),
                ),
            ),
        ]
        f_shuffled = torch.tensor(
            [
                [0.1000, 1.0000, -1.0000],
                [0.2000, 2.0000, -0.5000],
                [0.3000, 0.5000, -2.0000],
            ],
            dtype=torch.float64,
        )
        obj_w_shuffled = torch.tensor([0.0, 1.0, -1.0], dtype=torch.float64)
        obj_t_shuffled = torch.tensor(
            [-torch.inf, -torch.inf, torch.inf], dtype=torch.float64
        )

        # Test the function with these shuffled output for
        # `get_pareto_frontier_and_configs`.
        with patch(
            "ax.plot.pareto_utils.get_pareto_frontier_and_configs",
            return_value=(
                frontier_observations_shuffled,
                f_shuffled,
                obj_w_shuffled,
                obj_t_shuffled,
            ),
        ):
            inferred_reference_point = infer_reference_point_from_experiment(experiment)

            self.assertEqual(inferred_reference_point[0].op, ComparisonOp.LEQ)
            self.assertEqual(inferred_reference_point[0].bound, -0.35)
            self.assertEqual(inferred_reference_point[0].metric.name, "m1")
            self.assertEqual(inferred_reference_point[1].op, ComparisonOp.GEQ)
            self.assertEqual(inferred_reference_point[1].bound, 0.35)
            self.assertEqual(inferred_reference_point[1].metric.name, "m2")


import pyglet
import numpy as np
import math
sqrt = math.sqrt

window_width = 900
window_height = 780
window_caption = "Triangle Fractal"

window = pyglet.window.Window(
    window_width,
    window_height,
    caption=window_caption
    )

pixels = []

def generate_equal_triangle(l):
    '''
          A         AB=BC=AC=l
          .         C(0;0), B(CB, 0),
       .  |  .      AH = sqrt(AC^2+CH^2) =
    C ____|____ B   = sqrt(l^2 - (l/2)^2) =  h
          H         A (l/2,AH)
    '''
    C = [0,0]
    B = [l,0]
    h = sqrt (l**2 - (l/2)**2)
    A = [int(l/2), int(h)]
    return [A,B,C]

def pixels_from_figure(fig):
    pixs = []
    for dot in fig:
        pixs += dot
    return pixs

def make_fractal(fig, start=[0,0], ndots=1000000):
    x = start[0]
    y = start[1]
    nvrtx = len(fig)
    pixels = []
    for i in range(ndots):
        vertex = int(np.random.rand()*nvrtx)
        x = (x+fig[vertex][0])/2
        y = (y+fig[vertex][1])/2
        pixels += [int(x), int(y)]
    return pixels

@window.event
def on_draw():
    global pixels
    if pixels == []:
        print ("Init figure")
        fig = generate_equal_triangle(window_width)
        pixels += pixels_from_figure(fig)
        print ("done")
        print ("Make fractal")
        pixels += make_fractal(fig)
        print ("done")

    npix=int(len(pixels)/2)
    window.clear()
    pyglet.graphics.draw(npix, pyglet.gl.GL_POINTS,
        ('v2i', pixels)
    )

pyglet.app.run()